private URLResponse connect(String urlString, int zaznamId) { ByteArrayOutputStream bos = new ByteArrayOutputStream(); HttpURLConnection httpUrlConn = null; URL url = null; try { url = new URL(urlString); httpUrlConn = (HttpURLConnection) (url).openConnection(); httpUrlConn.setReadTimeout(2500); httpUrlConn.setConnectTimeout(2500); // jak na to ? httpUrlConn.setInstanceFollowRedirects(true); int respCode = httpUrlConn.getResponseCode(); String respMessage = httpUrlConn.getResponseMessage(); InputStream is = httpUrlConn.getInputStream(); IOUtils.copyStreams(is, bos); return new URLResponse( respCode, respMessage, url, httpUrlConn.getURL(), zaznamId, bos.toByteArray()); } catch (Exception ex) { log.error(ex, ex.getMessage()); return new URLResponse( -1, ex.getMessage(), url, httpUrlConn.getURL(), zaznamId, bos.toByteArray()); } finally { if (httpUrlConn != null) { httpUrlConn.disconnect(); } } }
/** * 下载指定pdf * * @param url pdf地址 * @return 下载后保存的全路径 * @throws Exception */ public String downloadPdf(String url) throws Exception { URL u = new URL(url); HttpURLConnection conn = (HttpURLConnection) u.openConnection(); conn.setRequestMethod("GET"); conn.connect(); InputStream is = conn.getInputStream(); String fileName = getTimestampFromUrl(url); Path tmp = Paths.get(SAVED_DIR).resolve(TMP_FILE); Path out = Paths.get(SAVED_DIR).resolve(fileName + ".pdf"); OutputStream os = Files.newOutputStream(tmp); log.info(String.format("Downloading %s (%s)", u.toString(), conn.getURL().toString())); // 开始下载 byte[] buffer = new byte[1024 * 1024]; int len; long size = 0; while ((len = is.read(buffer)) > -1) { os.write(buffer, 0, len); size += len; log.info(size + " bytes has been downloaded..."); } Files.move(tmp, out, StandardCopyOption.REPLACE_EXISTING); log.info("Downloaded " + out.toString() + " finished"); conn.disconnect(); return out.toString(); }
static String getResponseHeaderByKey(HttpURLConnection http, String key) { if (null == key) { return null; } Map<String, List<String>> headers = http.getHeaderFields(); if (null == headers) { return null; } String header = null; for (Entry<String, List<String>> entry : headers.entrySet()) { if (key.equalsIgnoreCase(entry.getKey())) { if ("set-cookie".equalsIgnoreCase(key)) { header = combinCookies(entry.getValue(), http.getURL().getHost()); } else { header = listToString(entry.getValue(), ","); } break; } } return header; }
@Test public void post() throws Exception { given(connection.getURL()).willReturn(new URL(URL)); given(connection.getResponseCode()).willReturn(200); given(connection.getResponseMessage()).willReturn("OK"); Map<String, List<String>> responseHeaderFields = new LinkedHashMap<String, List<String>>(); responseHeaderFields.put("Set-Cookie", Arrays.asList("aaa")); given(connection.getHeaderFields()).willReturn(responseHeaderFields); Request request = new Request( "POST", URL, Arrays.asList(new Header("Hoge", "Piyo")), new FormUrlEncodedTypedOutput().addField("foo", "bar")); Response response = underTest.execute(request); verify(connection).setRequestMethod("POST"); verify(connection).addRequestProperty("Hoge", "Piyo"); verify(connection).getOutputStream(); assertThat(response.getUrl()).isEqualTo(URL); assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getReason()).isEqualTo("OK"); assertThat(response.getHeaders()).isEqualTo(Arrays.asList(new Header("Set-Cookie", "aaa"))); assertThat(output.toString("UTF-8")).isEqualTo("foo=bar"); }
// TODO: http://www.baeldung.com/unshorten-url-httpclient private String expandShortURL(String address) throws IOException { URL url = new URL(address); HttpURLConnection connection = (HttpURLConnection) url.openConnection(Proxy.NO_PROXY); // using proxy may increase latency connection.setConnectTimeout(5000); connection.setReadTimeout(10000); connection.setInstanceFollowRedirects(false); connection.addRequestProperty("User-Agent", "Mozilla"); connection.connect(); String expandedURL; expandedURL = connection.getHeaderField("location"); if (expandedURL == null || expandedURL.length() == 0) { URL tmpURL = connection.getURL(); expandedURL = tmpURL.toString(); } InputStream myStream = connection.getInputStream(); myStream.close(); if (expandedURL == null || expandedURL.length() == 0) { log.error("ERROR: Expanded URL is empty!!!"); logConn.error("ERROR: Expanded URL is empty!!!"); } return expandedURL; }
/** * This method will test the return type to make sure that it is between 200 (inclusive) and 300 * (exclusive), i.e. that it is "successful". If it is not then it will throw an {@link * RequestFailureException} with the response code and message from the error stream. * * @param connection * @throws RequestFailureException * @throws IOException * @see HttpURLConnection#getResponseCode() * @see HttpURLConnection#getErrorStream() */ private void testResponseCode(HttpURLConnection connection) throws RequestFailureException, IOException { int responseCode = connection.getResponseCode(); if (responseCode < 200 || responseCode >= 300) { // Not one of the OK response codes so get the message off the error stream and throw an // exception InputStream errorStream = connection.getErrorStream(); String errorStreamString = null; String message = null; if (errorStream != null) { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); int read = -1; while ((read = errorStream.read()) != -1) { outputStream.write(read); } errorStreamString = outputStream.toString(getCharset(connection.getContentType())); // The content of the error stream coming from Massive seems to vary in structure (HTML, // JSON) see if it is is JSON and get a good message or use it as is message = parseErrorObject(errorStreamString); } throw new RequestFailureException( responseCode, message, connection.getURL(), errorStreamString); } }
// 读取响应的数据 private String doHttpRequest(HttpURLConnection connection) { if (DEBUG) Log.d(TAG, "BetterHttp httpRequest [1] url = " + connection.getURL()); try { // 读取响应码 int responseCode = connection.getResponseCode(); if (DEBUG) Log.d(TAG, "BetterHttp httpRequest [2] StatusLine : " + responseCode); // 200返回码表示成功,其余的表示失败 if (responseCode == HttpURLConnection.HTTP_OK) { // 在Gingerbread这个版本中, 添加了自动压缩数据的功能gzip // Content-Length返回的是压缩后的数据长度,所以使用 getContentLength()函数得到的Buffer数据大小是不正确的; // 要使用从响应中一直读取字节流直到InputStream.read()函数返回-1为止; InputStream inputStream = getUngzippedContent(connection); // 解析响应数据 StringBuilder sb = new StringBuilder(); BufferedReader reader = new BufferedReader(new UnicodeReader(inputStream, "UTF-8")); for (String s = reader.readLine(); s != null; s = reader.readLine()) { sb.append(s); } reader.close(); String responseData = sb.toString(); if (DEBUG) Log.d(TAG, "BetterHttp httpRequest [3] Response = " + responseData); return responseData; } } catch (IOException ex) { ex.printStackTrace(); } return null; }
/** * Adds an authentication header for basic authentication * * @param con the connection * @throws SdsTransferException thrown if something went wrong. Check for nested exceptions */ protected void addBasicAuthorizationHeader(HttpURLConnection con) throws SdsTransferException { CredentialsAgentResponse response; String token; try { response = credAgent.getCredentials( RequestorType.SERVER, con.getURL().getHost(), false /* don't know yet whether the credentials will succeed */); } catch (CredentialsAgentException e) { throw new SdsTransferException(e); } if (response == null) { token = ":"; } else if (response.isCanceled()) { cancel = true; return; } else { String username = response.getUsername() == null ? "" : response.getUsername(); String password = response.getPassword() == null ? "" : String.valueOf(response.getPassword()); token = username + ":" + password; con.addRequestProperty( "Authorization", "Basic " + Base64.getEncoder().encodeToString(token.getBytes(StandardCharsets.UTF_8))); } }
private CookieOrigin getCookieOrigin(HttpURLConnection connection) { URL url = connection.getURL(); String path = url.getPath(); if (null == path || path.trim().length() <= 0) { path = "/"; } int port = url.getPort() > 0 ? url.getPort() : 0; return new CookieOrigin(url.getHost(), port, path, false); }
@Override public String toString() { return uc.getRequestMethod() + " " + uc.getURL() + " returned a response status of " + this.getStatus() + " " + this.getClientResponseStatus(); }
private void write(HttpURLConnection connection, String json) { try { OutputStream outputStream = connection.getOutputStream(); OutputStreamWriter out = new OutputStreamWriter(outputStream); out.write(json); out.close(); } catch (IOException e) { throw new HttpException(String.format("Cannot write to %s", connection.getURL()), e); } }
public HttpResponse(HttpURLConnection urlConnection, byte[] body) { try { this.status = urlConnection.getResponseCode(); this.url = urlConnection.getURL().toString(); } catch (IOException e) { e.printStackTrace(); } this.headers = urlConnection.getHeaderFields(); this.body = body; }
@Override public String getRequestUri() { String result = queryUrl; if (connection != null) { URL url = connection.getURL(); if (url != null) { result = url.toString(); } } return result; }
/** * Throw an exception if the connection failed * * @param connection */ void processError(HttpURLConnection connection) { try { int code = connection.getResponseCode(); if (code == 200) return; URL url = connection.getURL(); String error = connection.getResponseMessage(); if (code == 403) { throw new TwitterException.E403( error + " " + url + " (" + (name == null ? "anonymous" : name) + ")"); } if (code == 404) { throw new TwitterException.E404(error + " " + url); } if (code >= 500 && code < 600) { throw new TwitterException.E50X(error + " " + url); } boolean rateLimitExceeded = (error != null && error.contains("Rate limit exceeded")); if (rateLimitExceeded) { throw new TwitterException.RateLimit(error); } // Rate limiter can sometimes cause a 400 Bad Request if (code == 400) { String json = getPage("http://twitter.com/account/rate_limit_status.json", null, true); try { JSONObject obj = new JSONObject(json); int hits = obj.getInt("remaining_hits"); if (hits < 1) throw new TwitterException.RateLimit(error); } catch (JSONException e) { // oh well } } String message = error != null ? error : ""; // just report it as a vanilla exception throw new TwitterException(code + " " + message + " " + url); } catch (SocketTimeoutException e) { URL url = connection.getURL(); throw new TwitterException.Timeout(timeOutMilliSecs + "milli-secs for " + url); } catch (IOException e) { throw new TwitterException(e); } }
private void handleError(HttpURLConnection connection) throws IOException { if (connection.getResponseCode() != 200) { String error = null; InputStream err = connection.getErrorStream(); if (err != null) { error = Utils.toString(err); } if (error == null) { error = connection.getResponseMessage(); } throw new RuntimeException("Unable to send request to " + connection.getURL() + ": " + error); } }
public void a(HttpURLConnection httpurlconnection, Object obj) { a("=== HTTP Request ==="); a( (new StringBuilder()) .append(httpurlconnection.getRequestMethod()) .append(" ") .append(httpurlconnection.getURL().toString()) .toString()); if (obj instanceof String) { a((new StringBuilder()).append("Content: ").append((String) obj).toString()); } a(httpurlconnection.getRequestProperties()); }
@Test public void errorResponseThrowsHttpError() throws Exception { given(connection.getURL()).willReturn(new URL(URL)); given(connection.getResponseCode()).willReturn(400); Map<String, List<String>> responseHeaderFields = new LinkedHashMap<String, List<String>>(); given(connection.getHeaderFields()) .willReturn(responseHeaderFields, Collections.<String, List<String>>emptyMap()); Request request = new Request("GET", URL, Collections.<Header>emptyList(), null); Response response = underTest.execute(request); assertThat(response.getStatus()).isEqualTo(400); }
private HttpRequest createRequest(String source) { HttpRequest request = HttpRequest.get(source); // Add credentials if a secure connection to github.com HttpURLConnection connection = request.getConnection(); if (connection instanceof HttpsURLConnection && HOST_DEFAULT.equals(connection.getURL().getHost())) { Account account = AccountUtils.getAccount(context); if (account != null) { String password = AccountManager.get(context).getPassword(account); if (!TextUtils.isEmpty(password)) request.basic(account.name, password); } } return request; }
@Test public void testSignRequest() throws MalformedURLException { final TwitterAuthConfig config = new TwitterAuthConfig("consumerKey", "consumerSecret"); final TwitterAuthToken accessToken = new TwitterAuthToken("token", "tokenSecret"); final HttpURLConnection connection = mock(HttpURLConnection.class); when(connection.getRequestMethod()).thenReturn("GET"); when(connection.getURL()) .thenReturn(new URL("https://api.twitter.com/1.1/statuses/home_timeline.json")); OAuth1aService.signRequest(config, accessToken, connection, null); verify(connection).setRequestProperty(eq(HttpRequest.HEADER_AUTHORIZATION), any(String.class)); // TODO: Make it so that nonce and timestamp can be specified for testing puproses? }
/** * @param payload payload to write * @param lengthDesc what to use in error log when an IOException occurs * @param connection connection to write to */ void writePayloadToConnection(Payload payload, Object lengthDesc, HttpURLConnection connection) throws IOException { connection.setDoOutput(true); CountingOutputStream out = new CountingOutputStream(connection.getOutputStream()); try { payload.writeTo(out); } catch (IOException e) { logger.error( e, "error after writing %d/%s bytes to %s", out.getCount(), lengthDesc, connection.getURL()); throw e; } }
@Test public void postAndGet() throws Exception { given(connection.getURL()).willReturn(new URL(URL), new URL(URL + "/secret")); given(connection.getResponseCode()).willReturn(302, 200); Map<String, List<String>> responseHeaderFields = new LinkedHashMap<String, List<String>>(); responseHeaderFields.put("Location", Arrays.asList("http://example.com/secret")); given(connection.getHeaderFields()) .willReturn(responseHeaderFields, Collections.<String, List<String>>emptyMap()); Request request = new Request("POST", URL, Collections.<Header>emptyList(), null); Response response = underTest.execute(request); assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getUrl()).isEqualTo(URL + "/secret"); }
public static byte[] downloadTileAndUpdateStore( int x, int y, int zoom, HttpMapSource mapSource, boolean useTileStore) throws UnrecoverableDownloadException, IOException, InterruptedException { if (zoom < 0) throw new UnrecoverableDownloadException("Negative zoom!"); HttpURLConnection conn = mapSource.getTileUrlConnection(zoom, x, y); if (conn == null) throw new UnrecoverableDownloadException( "Tile x=" + x + " y=" + y + " zoom=" + zoom + " is not a valid tile in map source " + mapSource); log.trace("Downloading " + conn.getURL()); prepareConnection(conn); conn.connect(); int code = conn.getResponseCode(); byte[] data = loadBodyDataInBuffer(conn); if (code != HttpURLConnection.HTTP_OK) throw new DownloadFailedException(conn, code); checkContentType(conn, data); checkContentLength(conn, data); String eTag = conn.getHeaderField("ETag"); long timeLastModified = conn.getLastModified(); long timeExpires = conn.getExpiration(); Utilities.checkForInterruption(); TileImageType imageType = Utilities.getImageType(data); if (imageType == null) throw new UnrecoverableDownloadException("The returned image is of unknown format"); if (useTileStore) { TileStore.getInstance() .putTileData(data, x, y, zoom, mapSource, timeLastModified, timeExpires, eTag); } Utilities.checkForInterruption(); return data; }
// set up url, method, header, cookies private void setupFromConnection(HttpURLConnection conn, Connection.Response previousResponse) throws IOException { method = Connection.Method.valueOf(conn.getRequestMethod()); url = conn.getURL(); statusCode = conn.getResponseCode(); statusMessage = conn.getResponseMessage(); contentType = conn.getContentType(); Map<String, List<String>> resHeaders = conn.getHeaderFields(); processResponseHeaders(resHeaders); // if from a redirect, map previous response cookies into this response if (previousResponse != null) { for (Map.Entry<String, String> prevCookie : previousResponse.cookies().entrySet()) { if (!hasCookie(prevCookie.getKey())) cookie(prevCookie.getKey(), prevCookie.getValue()); } } }
/** * Convenience method to follow http to https redirects. Will follow a total of 5 redirects, then * fail out due to foolishness on the url's part. * * @param c the {@link URLConnection} to open * @return an {@link InputStream} that is not necessarily at the same url, possibly at a 403 * redirect. * @throws IOException * @see {@link #getURLStream(URL)} */ protected static InputStream openConnectionCheckRedirects(URLConnection c) throws IOException { boolean redir; int redirects = 0; InputStream in = null; do { if (c instanceof HttpURLConnection) { ((HttpURLConnection) c).setInstanceFollowRedirects(false); } // We want to open the input stream before getting headers // because getHeaderField() et al swallow IOExceptions. in = c.getInputStream(); redir = false; if (c instanceof HttpURLConnection) { HttpURLConnection http = (HttpURLConnection) c; int stat = http.getResponseCode(); if (stat == 300 || stat == 301 || stat == 302 || stat == 303 || stat == 305 || stat == 307) { URL base = http.getURL(); String loc = http.getHeaderField("Location"); URL target = null; if (loc != null) { target = new URL(base, loc); } http.disconnect(); // Redirection should be allowed only for HTTP and HTTPS // and should be limited to 5 redirections at most. if (target == null || !(target.getProtocol().equals("http") || target.getProtocol().equals("https")) || redirects >= 5) { throw new SecurityException("illegal URL redirect"); } redir = true; c = target.openConnection(); redirects++; } } } while (redir); return in; }
/** * Constructs a canonicalized string for signing a request. * * @param conn the HttpURLConnection to canonicalize * @param accountName the account name associated with the request * @param contentLength the length of the content written to the outputstream in bytes, -1 if * unknown * @return a canonicalized string. * @throws StorageException */ @Override protected String canonicalize( final HttpURLConnection conn, final String accountName, final Long contentLength) throws StorageException { if (contentLength < -1) { throw new InvalidParameterException(SR.INVALID_CONTENT_LENGTH); } final String dateString = Utility.getStandardHeaderValue(conn, Constants.HeaderConstants.DATE); if (Utility.isNullOrEmpty(dateString)) { throw new IllegalArgumentException(SR.MISSING_MANDATORY_DATE_HEADER); } final StringBuilder canonicalizedString = new StringBuilder(ExpectedTableLiteCanonicalizedStringLength); canonicalizedString.append(dateString); appendCanonicalizedElement( canonicalizedString, getCanonicalizedResourceLite(conn.getURL(), accountName)); return canonicalizedString.toString(); }
/** * Constructs a canonicalized string for signing a request. * * @param conn the HttpURLConnection to canonicalize * @param accountName the account name associated with the request * @param contentLength the length of the content written to the outputstream in bytes, -1 if * unknown * @param opContext the OperationContext for the given request * @return a canonicalized string. * @throws StorageException */ @Override protected String canonicalize( final HttpURLConnection conn, final String accountName, final Long contentLength, final OperationContext opContext) throws StorageException { if (contentLength < -1) { throw new InvalidParameterException(SR.INVALID_CONTENT_LENGTH); } return canonicalizeHttpRequest( conn.getURL(), accountName, conn.getRequestMethod(), Utility.getStandardHeaderValue(conn, Constants.HeaderConstants.CONTENT_TYPE), contentLength, null, conn, opContext); }
private void testDataNodeRedirect(Path path) throws IOException { // Create the file if (hdfs.exists(path)) { hdfs.delete(path, true); } FSDataOutputStream out = hdfs.create(path, (short) 1); out.writeBytes("0123456789"); out.close(); // Get the path's block location so we can determine // if we were redirected to the right DN. FileStatus status = hdfs.getFileStatus(path); BlockLocation[] locations = hdfs.getFileBlockLocations(status, 0, 10); String locationName = locations[0].getNames()[0]; // Connect to the NN to get redirected URL u = hftpFs.getNamenodeURL( "/data" + ServletUtil.encodePath(path.toUri().getPath()), "ugi=userx,groupy"); HttpURLConnection conn = (HttpURLConnection) u.openConnection(); HttpURLConnection.setFollowRedirects(true); conn.connect(); conn.getInputStream(); boolean checked = false; // Find the datanode that has the block according to locations // and check that the URL was redirected to this DN's info port for (DataNode node : cluster.getDataNodes()) { DatanodeRegistration dnR = node.dnRegistration; if (dnR.getName().equals(locationName)) { checked = true; assertEquals(dnR.getInfoPort(), conn.getURL().getPort()); } } assertTrue( "The test never checked that location of " + "the block and hftp desitnation are the same", checked); }
private InputStream getInputStream() throws IOException { if (status != StreamStatus.NORMAL) { if (in != null) { in.close(); in = null; } // Use the original url if no resolved url exists, if it // is the first time a request is made. final URLOpener opener = (resolvedURL.getURL() == null) ? originalURL : resolvedURL; final HttpURLConnection connection = opener.openConnection(); connection.setRequestMethod("GET"); if (startPos != 0) { connection.setRequestProperty("Range", "bytes=" + startPos + "-"); } connection.connect(); in = connection.getInputStream(); int respCode = connection.getResponseCode(); if (startPos != 0 && respCode != HttpURLConnection.HTTP_PARTIAL) { // We asked for a byte range but did not receive a partial content // response... throw new IOException("HTTP_PARTIAL expected, received " + respCode); } else if (startPos == 0 && respCode != HttpURLConnection.HTTP_OK) { // We asked for all bytes from the beginning but didn't receive a 200 // response (none of the other 2xx codes are valid here) throw new IOException("HTTP_OK expected, received " + respCode); } resolvedURL.setURL(connection.getURL()); status = StreamStatus.NORMAL; } return in; }
Response readResponse(HttpURLConnection connection) throws IOException { int status = connection.getResponseCode(); String reason = connection.getResponseMessage(); if (reason == null) reason = ""; // HttpURLConnection treats empty reason as null. List<Header> headers = new ArrayList<Header>(); for (Map.Entry<String, List<String>> field : connection.getHeaderFields().entrySet()) { String name = field.getKey(); for (String value : field.getValue()) { headers.add(new Header(name, value)); } } String mimeType = connection.getContentType(); int length = connection.getContentLength(); InputStream stream; if (status >= 400) { stream = connection.getErrorStream(); } else { stream = connection.getInputStream(); } TypedInput responseBody = new TypedInputStream(mimeType, length, stream); return new Response(connection.getURL().toString(), status, reason, headers, responseBody); }
@Override public void run() { Log.d("Movie", "run: Working thread Started"); while (isContinued) { synchronized (this) { try { wait(); } catch (InterruptedException e) { e.printStackTrace(); } HttpURLConnection conn = null; if (conn == null) { try { if (req.getURL() == null) { req = null; break; } URL url = new URL(req.getURL()); conn = (HttpURLConnection) url.openConnection(); conn.setDoOutput(true); conn.setDoInput(true); ((HttpURLConnection) conn).setRequestMethod("GET"); conn.setUseCaches(false); conn.setAllowUserInteraction(true); HttpURLConnection.setFollowRedirects(true); conn.setInstanceFollowRedirects(true); conn.setRequestProperty( "User-agent", "Mozilla/5.0 (Windows; U; Windows NT 6.0; zh-TW; rv:1.9.1.2) " + "Gecko/20090729 Firefox/3.5.2 GTB5 (.NET CLR 3.5.30729)"); conn.setRequestProperty( "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); conn.setRequestProperty("Accept-Language", "zh-tw,en-us;q=0.7,en;q=0.3"); conn.setRequestProperty("Accept-Charse", "Big5,utf-8;q=0.7,*;q=0.7"); conn.connect(); } catch (IOException e) { e.printStackTrace(); } } try { conn.getOutputStream().flush(); Log.d("Movie", "run: " + conn.getURL()); BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8")); String line; StringBuffer buf = new StringBuffer(); while ((line = in.readLine()) != null) { System.out.println(line); buf.append(line); } req.execute(buf.toString()); } catch (IOException e) { e.printStackTrace(); } conn.disconnect(); conn = null; req = null; } } }