public static void saveMapB(
final File file, final Map<String, byte[]> props, final String comment) {
HashMap<String, String> m = new HashMap<String, String>();
for (Map.Entry<String, byte[]> e : props.entrySet())
m.put(e.getKey(), UTF8.String(e.getValue()));
saveMap(file, m, comment);
}
public static ConcurrentHashMap<String, byte[]> loadMapB(final File f) {
ConcurrentHashMap<String, String> m = loadMap(f);
if (m == null) return null;
ConcurrentHashMap<String, byte[]> mb = new ConcurrentHashMap<String, byte[]>();
for (Map.Entry<String, String> e : m.entrySet())
mb.put(e.getKey(), UTF8.getBytes(e.getValue()));
return mb;
}
/**
* main - writes some data and checks the tables size (with time measureing)
*
* @param args
*/
public static void main(final String[] args) {
// open a file, add one entry and exit
final File f = new File(args[0]);
if (f.exists()) FileUtils.deletedelete(f);
try {
final Records t = new Records(f, 8);
final byte[] b = new byte[8];
t.add("01234567".getBytes(), 0);
t.add("ABCDEFGH".getBytes(), 0);
t.add("abcdefgh".getBytes(), 0);
t.add("--------".getBytes(), 0);
t.add("********".getBytes(), 0);
for (int i = 0; i < 1000; i++) t.add("++++++++".getBytes(), 0);
t.add("=======0".getBytes(), 0);
t.add("=======1".getBytes(), 0);
t.add("=======2".getBytes(), 0);
t.cleanLast(b, 0);
System.out.println(UTF8.String(b));
t.cleanLast(b, 0);
// t.clean(2, b, 0);
System.out.println(UTF8.String(b));
t.get(1, b, 0);
System.out.println(UTF8.String(b));
t.put(1, "AbCdEfGh".getBytes(), 0);
t.get(1, b, 0);
System.out.println(UTF8.String(b));
t.get(3, b, 0);
System.out.println(UTF8.String(b));
t.get(4, b, 0);
System.out.println(UTF8.String(b));
System.out.println("size = " + t.size());
// t.clean(t.size() - 2);
t.cleanLast();
final long start = System.currentTimeMillis();
long c = 0;
for (int i = 0; i < 100000; i++) {
c = t.size();
}
System.out.println(
"size() needs " + ((System.currentTimeMillis() - start) / 100) + " nanoseconds");
System.out.println("size = " + c);
t.close();
} catch (final IOException e) {
ConcurrentLog.logException(e);
}
}
@Override
public Document[] parse(
final DigestURL location,
final String mimeType,
final String charset,
final VocabularyScraper scraper,
final int timezoneOffset,
final InputStream source)
throws Parser.Failure, InterruptedException {
byte[] b = null;
try {
b = FileUtils.read(source);
} catch (final IOException e1) {
throw new Parser.Failure(e1.toString(), location);
}
final BDecoder bd = new BDecoder(b);
final BObject bo = bd.parse();
if (bo == null) throw new Parser.Failure("BDecoder.parse returned null", location);
if (bo.getType() != BType.dictionary)
throw new Parser.Failure("BDecoder object is not a dictionary", location);
final Map<String, BObject> map = bo.getMap();
final BObject commento = map.get("comment");
final String comment = (commento == null) ? "" : UTF8.String(commento.getString());
// Date creation = new Date(map.get("creation date").getInteger());
final BObject infoo = map.get("info");
final StringBuilder filenames = new StringBuilder(80);
String title = "";
if (infoo != null) {
final Map<String, BObject> info = infoo.getMap();
final BObject fileso = info.get("files");
if (fileso != null) {
final List<BObject> filelist = fileso.getList();
for (final BObject fo : filelist) {
final BObject patho = fo.getMap().get("path");
if (patho != null) {
final List<BObject> l = patho.getList(); // one file may have several names
for (final BObject fl : l) {
filenames.append(fl.toString()).append(" ");
}
}
}
}
final BObject nameo = info.get("name");
if (nameo != null) title = UTF8.String(nameo.getString());
}
if (title == null || title.isEmpty()) title = MultiProtocolURL.unescape(location.getFileName());
return new Document[] {
new Document(
location,
mimeType,
charset,
this,
null,
null,
singleList(title), // title
comment, // author
location.getHost(),
null,
null,
0.0d,
0.0d,
filenames.toString(),
null,
null,
null,
false,
new Date())
};
}
@Override
public Document[] parse(
final AnchorURL location,
final String mimeType,
final String charset,
final VocabularyScraper scraper,
final int timezoneOffset,
final InputStream source)
throws Parser.Failure, InterruptedException {
// check memory for parser
if (!MemoryControl.request(200 * 1024 * 1024, false))
throw new Parser.Failure(
"Not enough Memory available for pdf parser: " + MemoryControl.available(), location);
// create a pdf parser
PDDocument pdfDoc;
try {
Thread.currentThread().setPriority(Thread.MIN_PRIORITY); // the pdfparser is a big pain
// pdfDoc = PDDocument.load(source);
final PDFParser pdfParser = new PDFParser(source);
pdfParser.setTempDirectory(new File(System.getProperty("java.io.tmpdir")));
pdfParser.parse();
pdfDoc = pdfParser.getPDDocument();
} catch (final IOException e) {
throw new Parser.Failure(e.getMessage(), location);
} finally {
Thread.currentThread().setPriority(Thread.NORM_PRIORITY);
}
if (pdfDoc.isEncrypted()) {
try {
pdfDoc.openProtection(new StandardDecryptionMaterial(""));
} catch (final BadSecurityHandlerException e) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted (1): " + e.getMessage(), location);
} catch (final IOException e) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted (2): " + e.getMessage(), location);
} catch (final CryptographyException e) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted (3): " + e.getMessage(), location);
}
final AccessPermission perm = pdfDoc.getCurrentAccessPermission();
if (perm == null || !perm.canExtractContent()) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted and cannot be decrypted", location);
}
}
// extracting some metadata
PDDocumentInformation info = pdfDoc.getDocumentInformation();
String docTitle = null,
docSubject = null,
docAuthor = null,
docPublisher = null,
docKeywordStr = null;
Date docDate = new Date();
if (info != null) {
docTitle = info.getTitle();
docSubject = info.getSubject();
docAuthor = info.getAuthor();
docPublisher = info.getProducer();
if (docPublisher == null || docPublisher.isEmpty()) docPublisher = info.getCreator();
docKeywordStr = info.getKeywords();
try {
if (info.getModificationDate() != null) docDate = info.getModificationDate().getTime();
} catch (IOException e) {
}
// unused:
// info.getTrapped());
}
info = null;
if (docTitle == null || docTitle.isEmpty()) {
docTitle = MultiProtocolURL.unescape(location.getFileName());
}
if (docTitle == null) {
docTitle = docSubject;
}
String[] docKeywords = null;
if (docKeywordStr != null) {
docKeywords = docKeywordStr.split(" |,");
}
Collection<AnchorURL>[] pdflinks = null;
Document[] result = null;
try {
// get the links
pdflinks = extractPdfLinks(pdfDoc);
// get the fulltext (either per document or for each page)
final PDFTextStripper stripper = new PDFTextStripper(StandardCharsets.UTF_8.name());
if (individualPages) {
// this is a hack which stores individual pages of the source pdf into individual index
// documents
// the new documents will get a virtual link with a post argument page=X appended to the
// original url
// collect text
int pagecount = pdfDoc.getNumberOfPages();
String[] pages = new String[pagecount];
for (int page = 1; page <= pagecount; page++) {
stripper.setStartPage(page);
stripper.setEndPage(page);
pages[page - 1] = stripper.getText(pdfDoc);
// System.out.println("PAGE " + page + ": " + pages[page - 1]);
}
// create individual documents for each page
assert pages.length == pdflinks.length
: "pages.length = " + pages.length + ", pdflinks.length = " + pdflinks.length;
result = new Document[Math.min(pages.length, pdflinks.length)];
String loc = location.toNormalform(true);
for (int page = 0; page < result.length; page++) {
result[page] =
new Document(
new AnchorURL(
loc
+ (loc.indexOf('?') > 0 ? '&' : '?')
+ individualPagePropertyname
+ '='
+ (page
+ 1)), // these are virtual new pages; we cannot combine them with '#'
// as that would be removed when computing the urlhash
mimeType,
StandardCharsets.UTF_8.name(),
this,
null,
docKeywords,
singleList(docTitle),
docAuthor,
docPublisher,
null,
null,
0.0f,
0.0f,
pages == null || page > pages.length ? new byte[0] : UTF8.getBytes(pages[page]),
pdflinks == null || page >= pdflinks.length ? null : pdflinks[page],
null,
null,
false,
docDate);
}
} else {
// collect the whole text at once
final CharBuffer writer = new CharBuffer(odtParser.MAX_DOCSIZE);
byte[] contentBytes = new byte[0];
stripper.setEndPage(3); // get first 3 pages (always)
writer.append(stripper.getText(pdfDoc));
contentBytes = writer.getBytes(); // remember text in case of interrupting thread
if (pdfDoc.getNumberOfPages() > 3) { // spare creating/starting thread if all pages read
stripper.setStartPage(4); // continue with page 4 (terminated, resulting in no text)
stripper.setEndPage(Integer.MAX_VALUE); // set to default
// we start the pdf parsing in a separate thread to ensure that it can be terminated
final PDDocument pdfDocC = pdfDoc;
final Thread t =
new Thread() {
@Override
public void run() {
Thread.currentThread().setName("pdfParser.getText:" + location);
try {
writer.append(stripper.getText(pdfDocC));
} catch (final Throwable e) {
}
}
};
t.start();
t.join(3000); // pdfbox likes to forget to terminate ... (quite often)
if (t.isAlive()) t.interrupt();
}
contentBytes = writer.getBytes(); // get final text before closing writer
Collection<AnchorURL> pdflinksCombined = new HashSet<AnchorURL>();
for (Collection<AnchorURL> pdflinksx : pdflinks)
if (pdflinksx != null) pdflinksCombined.addAll(pdflinksx);
result =
new Document[] {
new Document(
location,
mimeType,
StandardCharsets.UTF_8.name(),
this,
null,
docKeywords,
singleList(docTitle),
docAuthor,
docPublisher,
null,
null,
0.0f,
0.0f,
contentBytes,
pdflinksCombined,
null,
null,
false,
docDate)
};
}
} catch (final Throwable e) {
// close the writer (in finally)
// throw new Parser.Failure(e.getMessage(), location);
} finally {
try {
pdfDoc.close();
} catch (final Throwable e) {
}
}
// clear resources in pdfbox. they say that is resolved but it's not. see:
// https://issues.apache.org/jira/browse/PDFBOX-313
// https://issues.apache.org/jira/browse/PDFBOX-351
// https://issues.apache.org/jira/browse/PDFBOX-441
// the pdfbox still generates enormeous number of object allocations and don't delete these
// the following Object are statically stored and never flushed:
// COSFloat, COSArray, COSInteger, COSObjectKey, COSObject, COSDictionary,
// COSStream, COSString, COSName, COSDocument, COSInteger[], COSNull
// the great number of these objects can easily be seen in Java Visual VM
// we try to get this shit out of the memory here by forced clear calls, hope the best the
// rubbish gets out.
pdfDoc = null;
clean_up_idiotic_PDFParser_font_cache_which_eats_up_tons_of_megabytes();
return result;
}
/**
* delete files and directories if a directory is not empty, delete also everything inside because
* deletion sometimes fails on windows, there is also a windows exec included
*
* @param path
*/
public static void deletedelete(final File path) {
if (path == null || !path.exists()) {
return;
}
// empty the directory first
if (path.isDirectory()) {
final String[] list = path.list();
if (list != null) {
for (final String s : list) {
deletedelete(new File(path, s));
}
}
}
if (path.exists()) path.delete();
/*
int c = 0;
while ( c++ < 20 ) {
if ( !path.exists() ) {
break;
}
if ( path.delete() ) {
break;
}
// some OS may be slow when giving up file pointer
//System.runFinalization();
//System.gc();
try {
Thread.sleep(200);
} catch (final InterruptedException e ) {
break;
}
}
*/
if (path.exists()) {
path.deleteOnExit();
String p = "";
try {
p = path.getCanonicalPath();
} catch (final IOException e1) {
ConcurrentLog.logException(e1);
}
if (System.getProperties().getProperty("os.name", "").toLowerCase().startsWith("windows")) {
// deleting files on windows sometimes does not work with java
try {
final String command = "cmd /C del /F /Q \"" + p + "\"";
final Process r = Runtime.getRuntime().exec(command);
if (r == null) {
ConcurrentLog.severe("FileUtils", "cannot execute command: " + command);
} else {
final byte[] response = read(r.getInputStream());
ConcurrentLog.info("FileUtils", "deletedelete: " + UTF8.String(response));
}
} catch (final IOException e) {
ConcurrentLog.logException(e);
}
}
if (path.exists()) {
ConcurrentLog.severe("FileUtils", "cannot delete file " + p);
}
}
}
static final void sendRespondHeader(
final HashMap<String, Object> conProp,
final OutputStream respond,
String httpVersion,
final int httpStatusCode,
String httpStatusText,
ResponseHeader responseHeader)
throws IOException {
if (respond == null) throw new NullPointerException("The outputstream must not be null.");
if (conProp == null)
throw new NullPointerException("The connection property structure must not be null.");
if (httpVersion == null)
httpVersion = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
if (httpVersion == null) httpVersion = HeaderFramework.HTTP_VERSION_1_1;
if (responseHeader == null) responseHeader = new ResponseHeader(httpStatusCode);
try {
if ((httpStatusText == null) || (httpStatusText.length() == 0)) {
if (HeaderFramework.http1_1.containsKey(Integer.toString(httpStatusCode)))
// http1_1 includes http1_0 messages
httpStatusText = HeaderFramework.http1_1.get(Integer.toString(httpStatusCode));
else httpStatusText = "Unknown";
}
final StringBuilder header = new StringBuilder(560);
// "HTTP/0.9" does not have a status line or header in the response
if (!httpVersion.toUpperCase().equals(HeaderFramework.HTTP_VERSION_0_9)) {
// write status line
header
.append(httpVersion)
.append(" ")
.append(Integer.toString(httpStatusCode))
.append(" ")
.append(httpStatusText)
.append("\r\n");
// prepare header
if (!responseHeader.containsKey(HeaderFramework.DATE))
responseHeader.put(HeaderFramework.DATE, HeaderFramework.formatRFC1123(new Date()));
if (!responseHeader.containsKey(HeaderFramework.CONTENT_TYPE))
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html; charset=UTF-8"); // fix this
if (!responseHeader.containsKey(RequestHeader.CONNECTION)
&& conProp.containsKey(HeaderFramework.CONNECTION_PROP_PERSISTENT))
responseHeader.put(
RequestHeader.CONNECTION,
(String) conProp.get(HeaderFramework.CONNECTION_PROP_PERSISTENT));
if (!responseHeader.containsKey(RequestHeader.PROXY_CONNECTION)
&& conProp.containsKey(HeaderFramework.CONNECTION_PROP_PERSISTENT))
responseHeader.put(
RequestHeader.PROXY_CONNECTION,
(String) conProp.get(HeaderFramework.CONNECTION_PROP_PERSISTENT));
if (conProp.containsKey(HeaderFramework.CONNECTION_PROP_PERSISTENT)
&& conProp.get(HeaderFramework.CONNECTION_PROP_PERSISTENT).equals("keep-alive")
&& !responseHeader.containsKey(HeaderFramework.TRANSFER_ENCODING)
&& !responseHeader.containsKey(HeaderFramework.CONTENT_LENGTH))
responseHeader.put(HeaderFramework.CONTENT_LENGTH, "0");
// read custom headers
final Iterator<ResponseHeader.Entry> it =
responseHeader.getAdditionalHeaderProperties().iterator();
ResponseHeader.Entry e;
while (it.hasNext()) {
// Append user properties to the main String
// TODO: Should we check for user properites. What if they intersect properties that are
// already in header?
e = it.next();
header.append(e.getKey()).append(": ").append(e.getValue()).append("\r\n");
}
// write header
final Iterator<String> i = responseHeader.keySet().iterator();
String key;
char tag;
int count;
while (i.hasNext()) {
key = i.next();
tag = key.charAt(0);
if ((tag != '*')
&& (tag != '#')) { // '#' in key is reserved for proxy attributes as artificial header
// values
count = responseHeader.keyCount(key);
for (int j = 0; j < count; j++) {
header
.append(key)
.append(": ")
.append(responseHeader.getSingle(key, j))
.append("\r\n");
}
}
}
// end header
header.append("\r\n");
// sending headers to the client
respond.write(UTF8.getBytes(header.toString()));
// flush stream
respond.flush();
}
conProp.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_HEADER, responseHeader);
conProp.put(
HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_STATUS, Integer.toString(httpStatusCode));
} catch (final Exception e) {
// any interruption may be caused be network error or because the user has closed
// the windows during transmission. We simply pass it as IOException
throw new IOException(e.getMessage());
}
}
private static final void sendRespondError(
final HashMap<String, Object> conProp,
final OutputStream respond,
final int errorcase,
final int httpStatusCode,
String httpStatusText,
final String detailedErrorMsgText,
final Object detailedErrorMsgFile,
final serverObjects detailedErrorMsgValues,
final Throwable stackTrace,
ResponseHeader header)
throws IOException {
FileInputStream fis = null;
ByteArrayOutputStream o = null;
try {
// setting the proper http status message
String httpVersion = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
if (httpVersion == null) httpVersion = "HTTP/1.1";
if ((httpStatusText == null) || (httpStatusText.length() == 0)) {
// http1_1 includes http1_0 messages
if (HeaderFramework.http1_1.containsKey(Integer.toString(httpStatusCode)))
httpStatusText = HeaderFramework.http1_1.get(Integer.toString(httpStatusCode));
else httpStatusText = "Unknown";
}
// generating the desired request url
String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH);
if (path == null) path = "/";
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS);
final String method = (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD);
final int port = Domains.stripToPort(host);
host = Domains.stripToHostName(host);
String urlString;
try {
urlString =
(new DigestURL(
(method.equals(HeaderFramework.METHOD_CONNECT) ? "https" : "http"),
host,
port,
(args == null) ? path : path + "?" + args))
.toString();
} catch (final MalformedURLException e) {
urlString = "invalid URL";
}
// set rewrite values
final serverObjects tp = new serverObjects();
String clientIP = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP);
if (clientIP == null) clientIP = Domains.LOCALHOST;
tp.put("peerName", (switchboard.peers == null) ? "" : switchboard.peers.myName());
tp.put("errorMessageType", Integer.toString(errorcase));
tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText);
tp.put("requestMethod", (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD));
tp.put("requestURL", urlString);
switch (errorcase) {
case ERRORCASE_FILE:
tp.put(
"errorMessageType_file",
(detailedErrorMsgFile == null) ? "" : detailedErrorMsgFile.toString());
if ((detailedErrorMsgValues != null) && !detailedErrorMsgValues.isEmpty()) {
// rewriting the value-names and add the proper name prefix:
for (final Entry<String, String> entry : detailedErrorMsgValues.entrySet()) {
tp.put("errorMessageType_" + entry.getKey(), entry.getValue());
}
}
break;
case ERRORCASE_MESSAGE:
default:
tp.put(
"errorMessageType_detailedErrorMsg",
(detailedErrorMsgText == null)
? ""
: detailedErrorMsgText.replaceAll("\n", "<br />"));
break;
}
// building the stacktrace
if (stackTrace != null) {
tp.put("printStackTrace", "1");
final ByteBuffer errorMsg = new ByteBuffer(100);
final PrintStream printStream = new PrintStream(errorMsg);
stackTrace.printStackTrace(printStream);
tp.put("printStackTrace_exception", stackTrace.toString());
tp.put("printStackTrace_stacktrace", UTF8.String(errorMsg.getBytes()));
printStream.close();
} else {
tp.put("printStackTrace", "0");
}
// Generated Tue, 23 Aug 2005 11:19:14 GMT by brain.wg (squid/2.5.STABLE3)
// adding some system information
final String systemDate = HeaderFramework.formatRFC1123(new Date());
tp.put("date", systemDate);
// rewrite the file
final File htRootPath =
new File(
switchboard.getAppPath(),
switchboard.getConfig(
SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT));
TemplateEngine.writeTemplate(
"/proxymsg/error.html",
fis = new FileInputStream(new File(htRootPath, "/proxymsg/error.html")),
o = new ByteArrayOutputStream(512),
tp);
final byte[] result = o.toByteArray();
o.close();
o = null;
if (header == null) header = new ResponseHeader(httpStatusCode);
header.put(
HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_STATUS, Integer.toString(httpStatusCode));
header.put(HeaderFramework.DATE, systemDate);
header.put(HeaderFramework.CONTENT_TYPE, "text/html");
header.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(result.length));
header.put(HeaderFramework.PRAGMA, "no-cache, no-store");
sendRespondHeader(conProp, respond, httpVersion, httpStatusCode, httpStatusText, header);
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
// write the array to the client
FileUtils.copy(result, respond);
}
respond.flush();
} finally {
if (fis != null)
try {
fis.close();
} catch (final Exception e) {
ConcurrentLog.logException(e);
}
if (o != null)
try {
o.close();
} catch (final Exception e) {
ConcurrentLog.logException(e);
}
}
}
@Override
public Request pop(boolean delay, CrawlSwitchboard cs, RobotsTxt robots) throws IOException {
// returns a crawl entry from the stack and ensures minimum delta times
long sleeptime = 0;
Request crawlEntry = null;
CrawlProfile profileEntry = null;
synchronized (this) {
mainloop:
while (true) {
Index depthStack = getLowestStack();
if (depthStack == null) return null;
Row.Entry rowEntry = null;
while (depthStack.size() > 0) {
rowEntry = depthStack.removeOne();
if (rowEntry != null) break;
}
if (rowEntry == null) continue mainloop;
crawlEntry = new Request(rowEntry);
// check blacklist (again) because the user may have created blacklist entries after the
// queue has been filled
if (Switchboard.urlBlacklist.isListed(BlacklistType.CRAWLER, crawlEntry.url())) {
if (log.isFine()) log.fine("URL '" + crawlEntry.url() + "' is in blacklist.");
continue mainloop;
}
// at this point we must check if the crawlEntry has relevance because the crawl profile
// still exists
// if not: return null. A calling method must handle the null value and try again
profileEntry = cs.get(UTF8.getBytes(crawlEntry.profileHandle()));
if (profileEntry == null) {
if (log.isFine()) log.fine("no profile entry for handle " + crawlEntry.profileHandle());
continue mainloop;
}
// depending on the caching policy we need sleep time to avoid DoS-like situations
sleeptime = Latency.getDomainSleepTime(robots, profileEntry, crawlEntry.url());
break;
}
}
if (crawlEntry == null) return null;
ClientIdentification.Agent agent =
profileEntry == null
? ClientIdentification.yacyInternetCrawlerAgent
: profileEntry.getAgent();
long robotsTime = Latency.getRobotsTime(robots, crawlEntry.url(), agent);
Latency.updateAfterSelection(crawlEntry.url(), profileEntry == null ? 0 : robotsTime);
if (delay && sleeptime > 0) {
// force a busy waiting here
// in best case, this should never happen if the balancer works properly
// this is only to protection against the worst case, where the crawler could
// behave in a DoS-manner
if (log.isInfo())
log.info(
"forcing crawl-delay of "
+ sleeptime
+ " milliseconds for "
+ crawlEntry.url().getHost()
+ ": "
+ Latency.waitingRemainingExplain(crawlEntry.url(), robots, agent));
long loops = sleeptime / 1000;
long rest = sleeptime % 1000;
if (loops < 3) {
rest = rest + 1000 * loops;
loops = 0;
}
Thread.currentThread()
.setName(
"Balancer waiting for "
+ crawlEntry.url().getHost()
+ ": "
+ sleeptime
+ " milliseconds");
synchronized (this) {
// must be synchronized here to avoid 'takeover' moves from other threads which then idle
// the same time which would not be enough
if (rest > 0) {
try {
this.wait(rest);
} catch (final InterruptedException e) {
}
}
for (int i = 0; i < loops; i++) {
if (log.isInfo())
log.info(
"waiting for "
+ crawlEntry.url().getHost()
+ ": "
+ (loops - i)
+ " seconds remaining...");
try {
this.wait(1000);
} catch (final InterruptedException e) {
}
}
}
Latency.updateAfterSelection(crawlEntry.url(), robotsTime);
}
return crawlEntry;
}