@Override
public void sessionCompleted(HTTPSession session) {
try {
recordSession(session);
if (_executor != null) {
_executor.execute(new FilterTask(_content));
} else {
_receiver.filter(_content);
}
} finally {
session.close();
// Release the (old URL) order
_visitCounter.release(_content, null);
}
}
public boolean filter(UniMap content) {
HTTPSession session = _client.createSession();
session.setUrl(content.get(URL).toString());
HTTPSession.Method method = content.get(HTTP_METHOD);
if (method != null) {
session.setMethod(method);
}
RequestContent reqContent = content.get(REQUEST_CONTENT);
if (reqContent != null) {
session.setRequestContent(reqContent);
}
CharSequence etag = content.get(ETAG);
if (etag != null) {
session.addRequestHeader(new Header("If-None-Match", etag));
}
// Add If-Modified-Since (from LAST_VISIT) in RFC 822 format
Date lastVisit = content.get(LAST_VISIT);
if (lastVisit != null) {
session.addRequestHeader(Headers.createDateHeader("If-Modified-Since", lastVisit));
}
// FIXME: Use recorded LAST_SUCCESS_VISIT
// (i.e. last actual fetch for lastVisit) or Last-Modified from
// last success stored?
// REQUEST_HEADERS may be added to input content and override same
// named statically set "fixed" headers.
List<Header> headers = content.get(REQUEST_HEADERS);
if (headers != null) {
for (Header f : _fixedRequestHeaders) {
if (Headers.getFirst(headers, f.name().toString()) == null) {
session.addRequestHeader(f);
}
}
session.addRequestHeaders(headers);
} else {
session.addRequestHeaders(_fixedRequestHeaders);
}
session.setMaxContentLength(_maxContentLength);
session.setAcceptedContentTypes(_acceptedContentTypes);
_client.request(session, new Handler(content));
return true;
}
private void recordSession(HTTPSession session) {
_content.set(STATUS, session.statusCode());
_content.set(REQUEST_HEADERS, session.requestHeaders());
_content.set(RESPONSE_HEADERS, session.responseHeaders());
Exception error = session.error();
if (error != null) {
_content.set(REASON, "i.c.f.ContentFetcher: " + error);
_log.warn("Url: {} :: {}", session.url(), error.toString());
_log.debug("Stack Trace: ", error);
} else if ((session.statusCode() < 200) || (session.statusCode() > 307)) {
_log.warn(
"Url: {}; Response: {} {}", session.url(), session.statusCode(), session.statusText());
}
List<Header> headers = _content.get(RESPONSE_HEADERS);
if (headers == null) {
headers = Collections.emptyList();
}
Header etag = Headers.getFirst(headers, "ETag");
if (etag != null) {
_content.set(ETAG, Headers.asCharSequence(etag.value()));
}
ByteBuffer body = session.responseBody();
if ((body != null) && (body.remaining() > 0)) {
ContentSource cs = new ContentSource(body);
// Set default encoding
cs.setDefaultEncoding(_defaultEncoding);
// Set better default if charset in Content-Type
ContentType ctype = Headers.contentType(headers);
if (ctype != null) {
String eName = ctype.charset();
if (eName != null) {
Charset enc = Charsets.lookup(eName);
if (enc != null) cs.setDefaultEncoding(enc, 0.10F);
}
}
_content.set(SOURCE, cs);
}
}
