/**
* Writes an array of bytes on the stream. This method may not be used until this stream has been
* passed to one of the methods in HTTPConnection (i.e. until it has been associated with a
* request).
*
* @param buf an array containing the data to write
* @param off the offset of the data whithin the buffer
* @param len the number bytes (starting at <var>off</var>) to write
* @exception IOException if any exception is thrown by the socket, or if writing <var>len</var>
* bytes would cause more bytes to be written than this stream is willing to accept.
* @exception IllegalAccessError if this stream has not been associated with a request yet
*/
public synchronized void write(byte[] buf, int off, int len)
throws IOException, IllegalAccessError {
if (req == null) throw new IllegalAccessError("Stream not associated with a request");
if (ignore) return;
if (length != -1 && rcvd + len > length) {
IOException ioe =
new IOException("Tried to write too many bytes (" + (rcvd + len) + " > " + length + ")");
req.getConnection().closeDemux(ioe, false);
req.getConnection().outputFinished();
throw ioe;
}
try {
if (bos != null) bos.write(buf, off, len);
else if (length != -1) os.write(buf, off, len);
else os.write(Codecs.chunkedEncode(buf, off, len, null, false));
} catch (IOException ioe) {
req.getConnection().closeDemux(ioe, true);
req.getConnection().outputFinished();
throw ioe;
}
rcvd += len;
}
/* ------------------------------------------------------------ */
protected void checkForwardedHeaders(EndPoint endpoint, Request request) throws IOException {
HttpFields httpFields = request.getConnection().getRequestFields();
// Retrieving headers from the request
String forwardedHost = getLeftMostValue(httpFields.getStringField(getForwardedHostHeader()));
String forwardedServer =
getLeftMostValue(httpFields.getStringField(getForwardedServerHeader()));
String forwardedFor = getLeftMostValue(httpFields.getStringField(getForwardedForHeader()));
if (_hostHeader != null) {
// Update host header
httpFields.put(HttpHeaders.HOST_BUFFER, _hostHeader);
request.setServerName(null);
request.setServerPort(-1);
request.getServerName();
} else if (forwardedHost != null) {
// Update host header
httpFields.put(HttpHeaders.HOST_BUFFER, forwardedHost);
request.setServerName(null);
request.setServerPort(-1);
request.getServerName();
} else if (forwardedServer != null) {
// Use provided server name
request.setServerName(forwardedServer);
}
if (forwardedFor != null) {
request.setRemoteAddr(forwardedFor);
InetAddress inetAddress = null;
if (_useDNS) {
try {
inetAddress = InetAddress.getByName(forwardedFor);
} catch (UnknownHostException e) {
Log.ignore(e);
}
}
request.setRemoteHost(inetAddress == null ? forwardedFor : inetAddress.getHostName());
}
}
/** Invoked by the HTTPClient. */
public int requestHandler(Request req, Response[] resp) {
HTTPConnection con = req.getConnection();
URI new_loc, cur_loc;
// check for retries
HttpOutputStream out = req.getStream();
if (out != null && deferred_redir_list.get(out) != null) {
copyFrom((RedirectionModule) deferred_redir_list.remove(out));
req.copyFrom(saved_req);
if (new_con) return REQ_NEWCON_RST;
else return REQ_RESTART;
}
// handle permanent redirections
try {
cur_loc =
new URI(
new URI(con.getProtocol(), con.getHost(), con.getPort(), null), req.getRequestURI());
} catch (ParseException pe) {
throw new Error("HTTPClient Internal Error: unexpected exception '" + pe + "'", pe);
}
// handle permanent redirections
Hashtable perm_redir_list =
Util.getList(perm_redir_cntxt_list, req.getConnection().getContext());
if ((new_loc = (URI) perm_redir_list.get(cur_loc)) != null) {
/*
* copy query if present in old url but not in new url. This isn't
* strictly conforming, but some scripts fail to properly propagate the
* query string to the Location header. Unfortunately it looks like we're
* f****d either way: some scripts fail if you don't propagate the query
* string, some fail if you do... God, don't you just love it when people
* can't read a spec? Anway, since we can't get it right for all scripts
* we opt to follow the spec. String nres = new_loc.getPathAndQuery(),
* oquery = Util.getQuery(req.getRequestURI()), nquery =
* Util.getQuery(nres); if (nquery == null && oquery != null) nres += "?"
* + oquery;
*/
String nres = new_loc.getPathAndQuery();
req.setRequestURI(nres);
try {
lastURI = new URI(new_loc, nres);
} catch (ParseException pe) {
if (LOG.isTraceEnabled()) {
LOG.trace("An exception occurred: " + pe.getMessage());
}
}
if (LOG.isDebugEnabled())
LOG.debug(
"Matched request in permanent redirection list - redoing request to "
+ lastURI.toExternalForm());
if (!con.isCompatibleWith(new_loc)) {
try {
con = new HTTPConnection(new_loc);
} catch (ProtocolNotSuppException e) {
throw new Error("HTTPClient Internal Error: unexpected " + "exception '" + e + "'", e);
}
con.setContext(req.getConnection().getContext());
req.setConnection(con);
return REQ_NEWCON_RST;
} else {
return REQ_RESTART;
}
}
return REQ_CONTINUE;
}
/** Invoked by the HTTPClient. */
public int responsePhase2Handler(Response resp, Request req) throws IOException {
/* handle various response status codes until satisfied */
int sts = resp.getStatusCode();
switch (sts) {
case 302: // General (temporary) Redirection (handle like 303)
/*
* Note we only do this munging for POST and PUT. For GET it's not
* necessary; for HEAD we probably want to do another HEAD. For all
* others (i.e. methods from WebDAV, IPP, etc) it's somewhat unclear -
* servers supporting those should really return a 307 or 303, but some
* don't (guess who...), so we just don't touch those.
*/
if (req.getMethod().equals("POST") || req.getMethod().equals("PUT")) {
if (LOG.isDebugEnabled())
LOG.debug(
"Received status: " + sts + " " + resp.getReasonLine() + " - treating as 303");
sts = 303;
}
case 301: // Moved Permanently
case 303: // See Other (use GET)
case 307: // Moved Temporarily (we mean it!)
if (LOG.isDebugEnabled()) LOG.debug("Handling status: " + sts + " " + resp.getReasonLine());
// the spec says automatic redirection may only be done if
// the second request is a HEAD or GET.
if (!req.getMethod().equals("GET") && !req.getMethod().equals("HEAD") && sts != 303) {
if (LOG.isDebugEnabled())
LOG.debug("Not redirected because method is neither HEAD nor GET");
if (sts == 301 && resp.getHeader("Location") != null)
update_perm_redir_list(req, resLocHdr(resp.getHeader("Location"), req));
resp.setEffectiveURI(lastURI);
return RSP_CONTINUE;
}
case 305: // Use Proxy
case 306: // Switch Proxy
if (sts == 305 || sts == 306) {
if (LOG.isDebugEnabled())
LOG.debug("Handling status: " + sts + " " + resp.getReasonLine());
}
// Don't accept 305 from a proxy
if (sts == 305 && req.getConnection().getProxyHost() != null) {
if (LOG.isDebugEnabled()) LOG.debug("305 ignored because a proxy is already in use");
resp.setEffectiveURI(lastURI);
return RSP_CONTINUE;
}
/*
* the level is a primitive way of preventing infinite redirections.
* RFC-2068 set the max to 5, but RFC-2616 has loosened this. Since some
* sites (notably M$) need more levels, this is now set to the
* (arbitrary) value of 15 (god only knows why they need to do even 5
* redirections...).
*/
if (level >= 15 || resp.getHeader("Location") == null) {
if (LOG.isDebugEnabled()) {
if (level >= 15) LOG.debug("Not redirected because of too many levels of redirection");
else LOG.debug("Not redirected because no Location header was present");
}
resp.setEffectiveURI(lastURI);
return RSP_CONTINUE;
}
level++;
URI loc = resLocHdr(resp.getHeader("Location"), req);
HTTPConnection mvd;
String nres;
new_con = false;
if (sts == 305) {
mvd =
new HTTPConnection(
req.getConnection().getProtocol(),
req.getConnection().getHost(),
req.getConnection().getPort());
mvd.setCurrentProxy(loc.getHost(), loc.getPort());
mvd.setContext(req.getConnection().getContext());
new_con = true;
nres = req.getRequestURI();
/*
* There was some discussion about this, and especially Foteos
* Macrides (Lynx) said a 305 should also imply a change to GET (for
* security reasons) - see the thread starting at
* http://www.ics.uci.edu/pub/ietf/http/hypermail/1997q4/0351.html
* However, this is not in the latest draft, but since I agree with
* Foteos we do it anyway...
*/
req.setMethod("GET");
req.setData(null);
req.setStream(null);
} else if (sts == 306) {
// We'll have to wait for Josh to create a new spec here.
return RSP_CONTINUE;
} else {
if (req.getConnection().isCompatibleWith(loc)) {
mvd = req.getConnection();
nres = loc.getPathAndQuery();
} else {
try {
mvd = new HTTPConnection(loc);
nres = loc.getPathAndQuery();
} catch (ProtocolNotSuppException e) {
if (req.getConnection().getProxyHost() == null
|| !loc.getScheme().equalsIgnoreCase("ftp")) return RSP_CONTINUE;
// We're using a proxy and the protocol is ftp -
// maybe the proxy will also proxy ftp...
mvd =
new HTTPConnection(
"http",
req.getConnection().getProxyHost(),
req.getConnection().getProxyPort());
mvd.setCurrentProxy(null, 0);
nres = loc.toExternalForm();
}
mvd.setContext(req.getConnection().getContext());
new_con = true;
}
/*
* copy query if present in old url but not in new url. This isn't
* strictly conforming, but some scripts fail to propagate the query
* properly to the Location header. See comment on line 126. String
* oquery = Util.getQuery(req.getRequestURI()), nquery =
* Util.getQuery(nres); if (nquery == null && oquery != null) nres +=
* "?" + oquery;
*/
if (sts == 303) {
// 303 means "use GET"
if (!req.getMethod().equals("HEAD")) req.setMethod("GET");
req.setData(null);
req.setStream(null);
} else {
// If they used an output stream then they'll have
// to do the resend themselves
if (req.getStream() != null) {
if (!HTTPConnection.deferStreamed) {
if (LOG.isDebugEnabled())
LOG.debug("Status " + sts + " not handled - request has an output stream");
return RSP_CONTINUE;
}
saved_req = (Request) req.clone();
deferred_redir_list.put(req.getStream(), this);
req.getStream().reset();
resp.setRetryRequest(true);
}
if (sts == 301) {
// update permanent redirection list
try {
update_perm_redir_list(req, new URI(loc, nres));
} catch (ParseException pe) {
throw new Error(
"HTTPClient Internal Error: " + "unexpected exception '" + pe + "'", pe);
}
}
}
// Adjust Referer, if present
NVPair[] hdrs = req.getHeaders();
for (int idx = 0; idx < hdrs.length; idx++)
if (hdrs[idx].getName().equalsIgnoreCase("Referer")) {
HTTPConnection con = req.getConnection();
hdrs[idx] = new NVPair("Referer", con + req.getRequestURI());
break;
}
}
req.setConnection(mvd);
req.setRequestURI(nres);
try {
resp.getInputStream().close();
} catch (IOException ioe) {
if (LOG.isTraceEnabled()) {
LOG.trace("An exception occurred: " + ioe.getMessage());
}
}
if (sts != 305 && sts != 306) {
try {
lastURI = new URI(loc, nres);
} catch (ParseException pe) {
if (LOG.isTraceEnabled()) {
LOG.trace("An exception occurred: " + pe.getMessage());
}
}
if (LOG.isDebugEnabled())
LOG.debug(
"Request redirected to "
+ lastURI.toExternalForm()
+ " using method "
+ req.getMethod());
} else {
if (LOG.isDebugEnabled())
LOG.debug(
"Resending request using "
+ "proxy "
+ mvd.getProxyHost()
+ ":"
+ mvd.getProxyPort());
}
if (req.getStream() != null) return RSP_CONTINUE;
else if (new_con) return RSP_NEWCON_REQ;
else return RSP_REQUEST;
default:
return RSP_CONTINUE;
}
}
/**
* Closes the stream and causes the data to be sent if it has not already been done so. This
* method <strong>must</strong> be invoked when all data has been written.
*
* @exception IOException if any exception is thrown by the underlying socket, or if too few bytes
* were written.
* @exception IllegalAccessError if this stream has not been associated with a request yet.
*/
public synchronized void close() throws IOException, IllegalAccessError {
if (req == null) throw new IllegalAccessError("Stream not associated with a request");
if (ignore) return;
if (bos != null) {
req.setData(bos.toByteArray());
req.setStream(null);
if (trailers.length > 0) {
NVPair[] hdrs = req.getHeaders();
// remove any Trailer header field
int len = hdrs.length;
for (int idx = 0; idx < len; idx++) {
if (hdrs[idx].getName().equalsIgnoreCase("Trailer")) {
System.arraycopy(hdrs, idx + 1, hdrs, idx, len - idx - 1);
len--;
}
}
// add the trailers to the headers
hdrs = Util.resizeArray(hdrs, len + trailers.length);
System.arraycopy(trailers, 0, hdrs, len, trailers.length);
req.setHeaders(hdrs);
}
if (DebugConn) System.err.println("OutS: Sending request");
try {
resp = req.getConnection().sendRequest(req, con_to);
} catch (ModuleException me) {
throw new IOException(me.toString());
}
notify();
} else {
if (rcvd < length) {
IOException ioe =
new IOException(
"Premature close: only "
+ rcvd
+ " bytes written instead of the "
+ "expected "
+ length);
req.getConnection().closeDemux(ioe, false);
req.getConnection().outputFinished();
throw ioe;
}
try {
if (length == -1) {
if (DebugConn && trailers.length > 0) {
System.err.println("OutS: Sending trailers:");
for (int idx = 0; idx < trailers.length; idx++)
System.err.println(
" " + trailers[idx].getName() + ": " + trailers[idx].getValue());
}
os.write(Codecs.chunkedEncode(null, 0, 0, trailers, true));
}
os.flush();
if (DebugConn) System.err.println("OutS: All data sent");
} catch (IOException ioe) {
req.getConnection().closeDemux(ioe, true);
throw ioe;
} finally {
req.getConnection().outputFinished();
}
}
}
/** Invoked by the HTTPClient. */
public int requestHandler(Request req, Response[] resp) {
// First remove any Cookie headers we might have set for a previous
// request
NVPair[] hdrs = req.getHeaders();
int length = hdrs.length;
for (int idx = 0; idx < hdrs.length; idx++) {
int beg = idx;
while (idx < hdrs.length && hdrs[idx].getName().equalsIgnoreCase("Cookie")) idx++;
if (idx - beg > 0) {
length -= idx - beg;
System.arraycopy(hdrs, idx, hdrs, beg, length - beg);
}
}
if (length < hdrs.length) {
hdrs = Util.resizeArray(hdrs, length);
req.setHeaders(hdrs);
}
// Now set any new cookie headers
Hashtable cookie_list = Util.getList(cookie_cntxt_list, req.getConnection().getContext());
if (cookie_list.size() == 0) return REQ_CONTINUE; // no need to create a lot of objects
Vector names = new Vector();
Vector lens = new Vector();
int version = 0;
synchronized (cookie_list) {
Enumeration list = cookie_list.elements();
Vector remove_list = null;
while (list.hasMoreElements()) {
Cookie cookie = (Cookie) list.nextElement();
if (cookie.hasExpired()) {
if (LOG.isDebugEnabled()) LOG.debug("Cookie has expired and is being removed: " + cookie);
if (remove_list == null) remove_list = new Vector();
remove_list.addElement(cookie);
continue;
}
if (cookie.sendWith(req)
&& (cookie_handler == null || cookie_handler.sendCookie(cookie, req))) {
int len = cookie.getPath().length();
int idx;
// insert in correct position
for (idx = 0; idx < lens.size(); idx++)
if (((Integer) lens.elementAt(idx)).intValue() < len) break;
names.insertElementAt(cookie.toExternalForm(), idx);
lens.insertElementAt(new Integer(len), idx);
if (cookie instanceof Cookie2)
version = Math.max(version, ((Cookie2) cookie).getVersion());
}
}
// remove any marked cookies
// Note: we can't do this during the enumeration!
if (remove_list != null) {
for (int idx = 0; idx < remove_list.size(); idx++)
cookie_list.remove(remove_list.elementAt(idx));
}
}
if (!names.isEmpty()) {
StringBuffer value = new StringBuffer();
if (version > 0) value.append("$Version=\"" + version + "\"; ");
value.append((String) names.elementAt(0));
for (int idx = 1; idx < names.size(); idx++) {
value.append("; ");
value.append((String) names.elementAt(idx));
}
hdrs = Util.resizeArray(hdrs, hdrs.length + 1);
hdrs[hdrs.length - 1] = new NVPair("Cookie", value.toString());
// add Cookie2 header if necessary
if (version != 1) // we currently know about version 1 only
{
int idx;
for (idx = 0; idx < hdrs.length; idx++)
if (hdrs[idx].getName().equalsIgnoreCase("Cookie2")) break;
if (idx == hdrs.length) {
hdrs = Util.resizeArray(hdrs, hdrs.length + 1);
hdrs[hdrs.length - 1] = new NVPair("Cookie2", "$Version=\"1\"");
}
}
req.setHeaders(hdrs);
if (LOG.isDebugEnabled()) LOG.debug("Sending cookies '" + value + "'");
}
return REQ_CONTINUE;
}
/** Invoked by the HTTPClient. */
public void responsePhase1Handler(Response resp, RoRequest roreq)
throws IOException, ModuleException {
try {
resp.getStatusCode();
} catch (RetryException re) {
Log.write(Log.MODS, "RtryM: Caught RetryException");
boolean got_lock = false;
try {
synchronized (re.first) {
got_lock = true;
// initialize idempotent sequence checking
IdempotentSequence seq = new IdempotentSequence();
for (RetryException e = re.first; e != null; e = e.next) seq.add(e.request);
for (RetryException e = re.first; e != null; e = e.next) {
Log.write(Log.MODS, "RtryM: handling exception ", e);
Request req = e.request;
HTTPConnection con = req.getConnection();
/* Don't retry if either the sequence is not idempotent
* (Sec 8.1.4 and 9.1.2), or we've already retried enough
* times, or the headers have been read and parsed
* already
*/
if (!seq.isIdempotent(req)
|| (con.ServProtVersKnown
&& con.ServerProtocolVersion >= HTTP_1_1
&& req.num_retries > 0)
|| ((!con.ServProtVersKnown || con.ServerProtocolVersion <= HTTP_1_0)
&& req.num_retries > 4)
|| e.response.got_headers) {
e.first = null;
continue;
}
/**
* if an output stream was used (i.e. we don't have the data to resend) then delegate
* the responsibility for resending to the application.
*/
if (req.getStream() != null) {
if (HTTPConnection.deferStreamed) {
req.getStream().reset();
e.response.setRetryRequest(true);
}
e.first = null;
continue;
}
/* If we have an entity then setup either the entity-delay
* or the Expect header
*/
if (req.getData() != null && e.conn_reset) {
if (con.ServProtVersKnown && con.ServerProtocolVersion >= HTTP_1_1)
addToken(req, "Expect", "100-continue");
else req.delay_entity = 5000L << req.num_retries;
}
/* If the next request in line has an entity and we're
* talking to an HTTP/1.0 server then close the socket
* after this request. This is so that the available()
* call (to watch for an error response from the server)
* will work correctly.
*/
if (e.next != null
&& e.next.request.getData() != null
&& (!con.ServProtVersKnown || con.ServerProtocolVersion < HTTP_1_1)
&& e.conn_reset) {
addToken(req, "Connection", "close");
}
/* If this an HTTP/1.1 server then don't pipeline retries.
* The problem is that if the server for some reason
* decides not to use persistent connections and it does
* not do a correct shutdown of the connection, then the
* response will be ReSeT. If we did pipeline then we
* would keep falling into this trap indefinitely.
*
* Note that for HTTP/1.0 servers, if they don't support
* keep-alives then the normal code will already handle
* this accordingly and won't pipe over the same
* connection.
*/
if (con.ServProtVersKnown && con.ServerProtocolVersion >= HTTP_1_1 && e.conn_reset) {
req.dont_pipeline = true;
}
// The above is too risky - for moment let's be safe
// and never pipeline retried request at all.
req.dont_pipeline = true;
// now resend the request
Log.write(
Log.MODS,
"RtryM: Retrying request '" + req.getMethod() + " " + req.getRequestURI() + "'");
if (e.conn_reset) req.num_retries++;
e.response.http_resp.set(req, con.sendRequest(req, e.response.timeout));
e.exception = null;
e.first = null;
}
}
} catch (NullPointerException npe) {
if (got_lock) throw npe;
} catch (ParseException pe) {
throw new IOException(pe.getMessage());
}
if (re.exception != null) throw re.exception;
re.restart = true;
throw re;
}
}
