public Connection data(Map<String, String> data) {
Validate.notNull(data, "Data map must not be null");
for (Map.Entry<String, String> entry : data.entrySet()) {
req.data(KeyVal.create(entry.getKey(), entry.getValue()));
}
return this;
}
public Connection url(String url) {
Validate.notEmpty(url, "Must supply a valid URL");
try {
req.url(new URL(url));
} catch (MalformedURLException e) {
throw new IllegalArgumentException("Malformed URL: " + url, e);
}
return this;
}
public Connection data(String... keyvals) {
Validate.notNull(keyvals, "Data key value pairs must not be null");
Validate.isTrue(keyvals.length % 2 == 0, "Must supply an even number of key value pairs");
for (int i = 0; i < keyvals.length; i += 2) {
String key = keyvals[i];
String value = keyvals[i + 1];
Validate.notEmpty(key, "Data key must not be empty");
Validate.notNull(value, "Data value must not be null");
req.data(KeyVal.create(key, value));
}
return this;
}
private static String getRequestCookieString(Connection.Request req) {
StringBuilder sb = new StringBuilder();
boolean first = true;
for (Map.Entry<String, String> cookie : req.cookies().entrySet()) {
if (!first) sb.append("; ");
else first = false;
sb.append(cookie.getKey()).append('=').append(cookie.getValue());
// todo: spec says only ascii, no escaping / encoding defined. validate on set? or escape
// somehow here?
}
return sb.toString();
}
// set up connection defaults, and details from request
private static HttpURLConnection createConnection(Connection.Request req) throws IOException {
HttpURLConnection conn = (HttpURLConnection) req.url().openConnection();
conn.setRequestMethod(req.method().name());
conn.setInstanceFollowRedirects(false); // don't rely on native redirection support
conn.setConnectTimeout(req.timeout());
conn.setReadTimeout(req.timeout());
if (req.method() == Method.POST) conn.setDoOutput(true);
if (req.cookies().size() > 0) conn.addRequestProperty("Cookie", getRequestCookieString(req));
for (Map.Entry<String, String> header : req.headers().entrySet()) {
conn.addRequestProperty(header.getKey(), header.getValue());
}
return conn;
}
// for get url reqs, serialise the data map into the url
private static void serialiseRequestUrl(Connection.Request req) throws IOException {
URL in = req.url();
StringBuilder url = new StringBuilder();
boolean first = true;
// reconstitute the query, ready for appends
url.append(in.getProtocol())
.append("://")
.append(in.getAuthority()) // includes host, port
.append(in.getPath())
.append("?");
if (in.getQuery() != null) {
url.append(in.getQuery());
first = false;
}
for (Connection.KeyVal keyVal : req.data()) {
if (!first) url.append('&');
else first = false;
url.append(URLEncoder.encode(keyVal.key(), DataUtil.defaultCharset))
.append('=')
.append(URLEncoder.encode(keyVal.value(), DataUtil.defaultCharset));
}
req.url(new URL(url.toString()));
req.data().clear(); // moved into url as get params
}
public Document parse() throws IOException {
Validate.isTrue(
executed,
"Request must be executed (with .execute(), .get(), or .post() before parsing response");
if (!req.ignoreContentType()
&& (contentType == null
|| !(contentType.startsWith("text/")
|| contentType.startsWith("application/xml")
|| contentType.startsWith("application/xhtml+xml"))))
throw new IOException(
String.format(
"Unhandled content type \"%s\" on URL %s. Must be text/*, application/xml, or application/xhtml+xml",
contentType, url.toString()));
Document doc = DataUtil.parseByteData(byteData, charset, url.toExternalForm());
byteData.rewind();
charset = doc.outputSettings().charset().name(); // update charset from meta-equiv, possibly
return doc;
}
public Connection ignoreContentType(boolean ignoreContentType) {
req.ignoreContentType(ignoreContentType);
return this;
}
public Connection ignoreHttpErrors(boolean ignoreHttpErrors) {
req.ignoreHttpErrors(ignoreHttpErrors);
return this;
}
public Connection method(Method method) {
req.method(method);
return this;
}
public Connection referrer(String referrer) {
Validate.notNull(referrer, "Referrer must not be null");
req.header("Referer", referrer);
return this;
}
public Connection followRedirects(boolean followRedirects) {
req.followRedirects(followRedirects);
return this;
}
public Connection timeout(int millis) {
req.timeout(millis);
return this;
}
static Response execute(Connection.Request req, Response previousResponse) throws IOException {
Validate.notNull(req, "Request must not be null");
String protocol = req.url().getProtocol();
Validate.isTrue(
protocol.equals("http") || protocol.equals("https"),
"Only http & https protocols supported");
// set up the request for execution
if (req.method() == Connection.Method.GET && req.data().size() > 0)
serialiseRequestUrl(req); // appends query string
HttpURLConnection conn = createConnection(req);
conn.connect();
if (req.method() == Connection.Method.POST) writePost(req.data(), conn.getOutputStream());
int status = conn.getResponseCode();
boolean needsRedirect = false;
if (status != HttpURLConnection.HTTP_OK) {
if (status == HttpURLConnection.HTTP_MOVED_TEMP
|| status == HttpURLConnection.HTTP_MOVED_PERM
|| status == HttpURLConnection.HTTP_SEE_OTHER) needsRedirect = true;
else if (!req.ignoreHttpErrors())
throw new IOException(status + " error loading URL " + req.url().toString());
}
Response res = new Response(previousResponse);
res.setupFromConnection(conn, previousResponse);
if (needsRedirect && req.followRedirects()) {
req.method(
Method
.GET); // always redirect with a get. any data param from original req are dropped.
req.data().clear();
req.url(new URL(req.url(), res.header("Location")));
for (Map.Entry<String, String> cookie :
res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts)
req.cookie(cookie.getKey(), cookie.getValue());
}
return execute(req, res);
}
res.req = req;
InputStream bodyStream = null;
InputStream dataStream = null;
try {
dataStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream();
bodyStream =
res.hasHeader("Content-Encoding")
&& res.header("Content-Encoding").equalsIgnoreCase("gzip")
? new BufferedInputStream(new GZIPInputStream(dataStream))
: new BufferedInputStream(dataStream);
res.byteData = DataUtil.readToByteBuffer(bodyStream);
res.charset =
DataUtil.getCharsetFromContentType(
res.contentType); // may be null, readInputStream deals with it
} finally {
if (bodyStream != null) bodyStream.close();
if (dataStream != null) dataStream.close();
}
res.executed = true;
return res;
}
public Connection header(String name, String value) {
req.header(name, value);
return this;
}
public Connection cookie(String name, String value) {
req.cookie(name, value);
return this;
}
public Connection url(URL url) {
req.url(url);
return this;
}
public Connection data(String key, String value) {
req.data(KeyVal.create(key, value));
return this;
}
public Connection userAgent(String userAgent) {
Validate.notNull(userAgent, "User agent must not be null");
req.header("User-Agent", userAgent);
return this;
}
public Document post() throws IOException {
req.method(Method.POST);
execute();
return res.parse();
}