public Connection data(Map<String, String> data) { Validate.notNull(data, "Data map must not be null"); for (Map.Entry<String, String> entry : data.entrySet()) { req.data(KeyVal.create(entry.getKey(), entry.getValue())); } return this; }
public Connection url(String url) { Validate.notEmpty(url, "Must supply a valid URL"); try { req.url(new URL(url)); } catch (MalformedURLException e) { throw new IllegalArgumentException("Malformed URL: " + url, e); } return this; }
public Connection data(String... keyvals) { Validate.notNull(keyvals, "Data key value pairs must not be null"); Validate.isTrue(keyvals.length % 2 == 0, "Must supply an even number of key value pairs"); for (int i = 0; i < keyvals.length; i += 2) { String key = keyvals[i]; String value = keyvals[i + 1]; Validate.notEmpty(key, "Data key must not be empty"); Validate.notNull(value, "Data value must not be null"); req.data(KeyVal.create(key, value)); } return this; }
private static String getRequestCookieString(Connection.Request req) { StringBuilder sb = new StringBuilder(); boolean first = true; for (Map.Entry<String, String> cookie : req.cookies().entrySet()) { if (!first) sb.append("; "); else first = false; sb.append(cookie.getKey()).append('=').append(cookie.getValue()); // todo: spec says only ascii, no escaping / encoding defined. validate on set? or escape // somehow here? } return sb.toString(); }
// set up connection defaults, and details from request private static HttpURLConnection createConnection(Connection.Request req) throws IOException { HttpURLConnection conn = (HttpURLConnection) req.url().openConnection(); conn.setRequestMethod(req.method().name()); conn.setInstanceFollowRedirects(false); // don't rely on native redirection support conn.setConnectTimeout(req.timeout()); conn.setReadTimeout(req.timeout()); if (req.method() == Method.POST) conn.setDoOutput(true); if (req.cookies().size() > 0) conn.addRequestProperty("Cookie", getRequestCookieString(req)); for (Map.Entry<String, String> header : req.headers().entrySet()) { conn.addRequestProperty(header.getKey(), header.getValue()); } return conn; }
// for get url reqs, serialise the data map into the url private static void serialiseRequestUrl(Connection.Request req) throws IOException { URL in = req.url(); StringBuilder url = new StringBuilder(); boolean first = true; // reconstitute the query, ready for appends url.append(in.getProtocol()) .append("://") .append(in.getAuthority()) // includes host, port .append(in.getPath()) .append("?"); if (in.getQuery() != null) { url.append(in.getQuery()); first = false; } for (Connection.KeyVal keyVal : req.data()) { if (!first) url.append('&'); else first = false; url.append(URLEncoder.encode(keyVal.key(), DataUtil.defaultCharset)) .append('=') .append(URLEncoder.encode(keyVal.value(), DataUtil.defaultCharset)); } req.url(new URL(url.toString())); req.data().clear(); // moved into url as get params }
public Document parse() throws IOException { Validate.isTrue( executed, "Request must be executed (with .execute(), .get(), or .post() before parsing response"); if (!req.ignoreContentType() && (contentType == null || !(contentType.startsWith("text/") || contentType.startsWith("application/xml") || contentType.startsWith("application/xhtml+xml")))) throw new IOException( String.format( "Unhandled content type \"%s\" on URL %s. Must be text/*, application/xml, or application/xhtml+xml", contentType, url.toString())); Document doc = DataUtil.parseByteData(byteData, charset, url.toExternalForm()); byteData.rewind(); charset = doc.outputSettings().charset().name(); // update charset from meta-equiv, possibly return doc; }
public Connection ignoreContentType(boolean ignoreContentType) { req.ignoreContentType(ignoreContentType); return this; }
public Connection ignoreHttpErrors(boolean ignoreHttpErrors) { req.ignoreHttpErrors(ignoreHttpErrors); return this; }
public Connection method(Method method) { req.method(method); return this; }
public Connection referrer(String referrer) { Validate.notNull(referrer, "Referrer must not be null"); req.header("Referer", referrer); return this; }
public Connection followRedirects(boolean followRedirects) { req.followRedirects(followRedirects); return this; }
public Connection timeout(int millis) { req.timeout(millis); return this; }
static Response execute(Connection.Request req, Response previousResponse) throws IOException { Validate.notNull(req, "Request must not be null"); String protocol = req.url().getProtocol(); Validate.isTrue( protocol.equals("http") || protocol.equals("https"), "Only http & https protocols supported"); // set up the request for execution if (req.method() == Connection.Method.GET && req.data().size() > 0) serialiseRequestUrl(req); // appends query string HttpURLConnection conn = createConnection(req); conn.connect(); if (req.method() == Connection.Method.POST) writePost(req.data(), conn.getOutputStream()); int status = conn.getResponseCode(); boolean needsRedirect = false; if (status != HttpURLConnection.HTTP_OK) { if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER) needsRedirect = true; else if (!req.ignoreHttpErrors()) throw new IOException(status + " error loading URL " + req.url().toString()); } Response res = new Response(previousResponse); res.setupFromConnection(conn, previousResponse); if (needsRedirect && req.followRedirects()) { req.method( Method .GET); // always redirect with a get. any data param from original req are dropped. req.data().clear(); req.url(new URL(req.url(), res.header("Location"))); for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts) req.cookie(cookie.getKey(), cookie.getValue()); } return execute(req, res); } res.req = req; InputStream bodyStream = null; InputStream dataStream = null; try { dataStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream(); bodyStream = res.hasHeader("Content-Encoding") && res.header("Content-Encoding").equalsIgnoreCase("gzip") ? new BufferedInputStream(new GZIPInputStream(dataStream)) : new BufferedInputStream(dataStream); res.byteData = DataUtil.readToByteBuffer(bodyStream); res.charset = DataUtil.getCharsetFromContentType( res.contentType); // may be null, readInputStream deals with it } finally { if (bodyStream != null) bodyStream.close(); if (dataStream != null) dataStream.close(); } res.executed = true; return res; }
public Connection header(String name, String value) { req.header(name, value); return this; }
public Connection cookie(String name, String value) { req.cookie(name, value); return this; }
public Connection url(URL url) { req.url(url); return this; }
public Connection data(String key, String value) { req.data(KeyVal.create(key, value)); return this; }
public Connection userAgent(String userAgent) { Validate.notNull(userAgent, "User agent must not be null"); req.header("User-Agent", userAgent); return this; }
public Document post() throws IOException { req.method(Method.POST); execute(); return res.parse(); }