@Test public void testUnsatisfiableRangeRequest() throws IOException, URISyntaxException { final URL testResourceUrl = new URL(VAULT_BASE_URI.toURL(), "unsatisfiableRangeRequestTestFile.txt"); final HttpClient client = new HttpClient(); // prepare file content: final byte[] fileContent = "This is some test file content.".getBytes(); // put request: final EntityEnclosingMethod putMethod = new PutMethod(testResourceUrl.toString()); putMethod.setRequestEntity(new ByteArrayRequestEntity(fileContent)); final int putResponse = client.executeMethod(putMethod); putMethod.releaseConnection(); Assert.assertEquals(201, putResponse); // get request: final HttpMethod getMethod = new GetMethod(testResourceUrl.toString()); getMethod.addRequestHeader("Range", "chunks=1-2"); final int getResponse = client.executeMethod(getMethod); final byte[] response = new byte[fileContent.length]; IOUtils.read(getMethod.getResponseBodyAsStream(), response); getMethod.releaseConnection(); Assert.assertEquals(416, getResponse); Assert.assertArrayEquals(fileContent, response); }
private HttpMethod createProxyRequest(String targetUrl, HttpServletRequest request) throws IOException { URI targetUri; try { targetUri = new URI(uriEncode(targetUrl)); } catch (URISyntaxException e) { throw new RuntimeException(e); } HttpMethod commonsHttpMethod = httpMethodProvider.getMethod(request.getMethod(), targetUri.toString()); commonsHttpMethod.setFollowRedirects(false); Enumeration<String> headerNames = request.getHeaderNames(); while (headerNames.hasMoreElements()) { String headerName = headerNames.nextElement(); Enumeration<String> headerVals = request.getHeaders(headerName); while (headerVals.hasMoreElements()) { String headerValue = headerVals.nextElement(); headerValue = headerFilter.processRequestHeader(headerName, headerValue); if (headerValue != null) { commonsHttpMethod.addRequestHeader(new Header(headerName, headerValue)); } } } return commonsHttpMethod; }
protected <T> void populateRequestHeaders( HttpMethod httpMethod, HttpInvocation<T> httpInvocation) { List<Header> requestHeaders = httpInvocation.getRequestHeaders(); if (!CollectionUtils.isEmpty(requestHeaders)) { for (Header header : requestHeaders) { httpMethod.addRequestHeader(header); } } }
private Result excuteMethod(HttpMethod method, String cookie, Map<String, String> headers) { Result result = new Result(); result.setSuccess(true); int responseCode = -1; try { // String cookie = // "240=240; CurLoginUserGUID=a44e6f49751b4d8b9659636ae20540d6; // strCertificate=410df02a24354be8bb194aeec0829589;"; if (cookie != null) { method.addRequestHeader("Cookie", cookie); } if (headers != null) { for (Entry<String, String> entry : headers.entrySet()) { method.addRequestHeader(entry.getKey(), entry.getValue()); } } method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, httpRequestRetryHandler); org.apache.commons.httpclient.HttpClient client = this.createHttpClient(); client.executeMethod(method); responseCode = method.getStatusCode(); if (responseCode != HttpStatus.SC_OK) { result.setSuccess(false); result.setErrorCode(ErrorCodeEnum.NOT_FOUND_ERROR); result.setMessage("responseCode:" + responseCode); result.setResponseCode(responseCode); } InputStream is = method.getResponseBodyAsStream(); try { result.setResultBytes(IOUtils.toByteArray(is)); } finally { IOUtils.closeQuietly(is); } } catch (Exception e) { result.setSuccess(false); result.setErrorCode(ErrorCodeEnum.SYSTEM_ERROR); result.setMessage(e.getMessage()); result.setCause(e); } finally { result.setMethodStatusCode(method.getStatusCode()); method.releaseConnection(); } return result; }
private void configureHttpMethod( boolean skipContentCache, CacheData cacheData, long onceTimeOut, HttpMethod httpMethod) { if (skipContentCache && null != cacheData) { if (null != cacheData.getLastModifiedHeader() && Constants.NULL != cacheData.getLastModifiedHeader()) { httpMethod.addRequestHeader(Constants.IF_MODIFIED_SINCE, cacheData.getLastModifiedHeader()); } if (null != cacheData.getMd5() && Constants.NULL != cacheData.getMd5()) { httpMethod.addRequestHeader(Constants.CONTENT_MD5, cacheData.getMd5()); } } httpMethod.addRequestHeader(Constants.ACCEPT_ENCODING, "gzip,deflate"); HttpMethodParams params = new HttpMethodParams(); params.setSoTimeout((int) onceTimeOut); httpMethod.setParams(params); httpClient .getHostConfiguration() .setHost( diamondConfigure.getDomainNameList().get(this.domainNamePos.get()), diamondConfigure.getPort()); }
public void resetMethod() { // httpClient.getState().clearCookies(); httpClient = clientMap.get(account.getUsername()); // for(String s1:account.getCookie().split("; ")){ // Cookie cookie = new Cookie(domain, name, value) // httpClient.getState().addCookie(cookie); // } httpMethod = new GetMethod(); // for (Header h : httpMethod.getRequestHeaders()) { // httpMethod.removeRequestHeader(h); // } // httpClient.getState().clearCookies(); httpMethod.addRequestHeader( "User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36"); httpMethod.addRequestHeader("Host", "weibo.com"); httpMethod.addRequestHeader("Connection", "keep-alive"); httpMethod.addRequestHeader("Accept-Language", "zh-CN,zh;q=0.8"); httpMethod.addRequestHeader("Cache-Control", "max-age=0"); httpMethod.addRequestHeader( "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); httpMethod.addRequestHeader("Accept-Encoding", "deflate,sdch"); httpMethod.removeRequestHeader("Cookie"); // ck = // "__utma=15428400.1695645915.1351319378.1351319378.1351319378.1; // SINAGLOBAL=1743311276659.3694.1359040897731; ssoln=18260402168%40139.com; myuid=2350795254; // [email protected]; wvr=5; UOR=www.juyouqu.com,widget.weibo.com,login.sina.com.cn; // UV5PAGE=usr513_160; // SUE=es%3D6edfb5cebc465646c2b99373cb2a29fe%26ev%3Dv1%26es2%3D2d0e68c7608a20158aa2045b5dfb59ee%26rs0%3DcEHTC0Ck28iBwSm7ReH2rIUG1n%252FJjjyJ3TkFC8oU2gEFF7qEa83xCOp%252FdFSR05fBeMEZuKAm5ZMBr721u67n7Rfq7KrNI7cHkJFqnvrl1ut%252Bid0pcoUAmyM9JP316GEN%252FB9E1AqHJns7jVNcflxekc2QPaMOf4BzOUX7zShB0LM%253D%26rv%3D0; SUP=cv%3D1%26bt%3D1383912034%26et%3D1383998434%26d%3Dc909%26i%3D15d3%26us%3D1%26vf%3D0%26vt%3D0%26ac%3D2%26st%3D0%26uid%3D3807150373%26name%3Djobpassion%2540gmail.com%26nick%3D%25E6%25B5%25B7%25E6%25B7%2598%25E8%25B5%2584%25E8%25AE%25AF2013%26fmp%3D%26lcp%3D2013-10-08%252021%253A20%253A24; SUS=SID-3807150373-1383912034-JA-jqf3k-628deb26b5773de29473c3d67d64b10b; ALF=1386504033; SSOLoginState=1383912034; UUG=usrmdins41458; _s_tentry=weibo.com; Apache=8601501327939.331.1383912035126; ULV=1383912035149:97:5:3:8601501327939.331.1383912035126:1383837545412; UV5=usrmdins311164; SinaRot_wb_r_topic=75" // httpMethod.addRequestHeader("Cookie", account.getCookie()); // postMethod.recycle(); postMethod = new PostMethod(); postMethod.addRequestHeader( "User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36"); postMethod.addRequestHeader("Host", "weibo.com"); postMethod.addRequestHeader("Connection", "keep-alive"); postMethod.addRequestHeader("Accept-Language", "zh-CN,zh;q=0.8"); // postMethod.addRequestHeader("Cache-Control", "max-age=0"); postMethod.addRequestHeader("Accept", "*/*"); postMethod.addRequestHeader("Accept-Encoding", "deflate,sdch"); postMethod.addRequestHeader("Origin", "http://weibo.com"); // postMethod.addRequestHeader("Content-Length", "101"); postMethod.addRequestHeader("Content-Type", "application/x-www-form-urlencoded"); postMethod.addRequestHeader("X-Requested-With", "XMLHttpRequest"); postMethod.addRequestHeader("Referer", "http://weibo.com/xiena?refer=interest"); // postMethod.addRequestHeader("Cookie", account.getCookie()); }
@Override public void run() throws Exception { super.run(); logger.info("Starting download in TASK " + fileURL); login(); final GetMethod method = getGetMethod(RD_GENERATE); if (makeRedirectedRequest(method)) { // we make the main request checkProblems(); // check problems final HttpMethod postMethod = getMethodBuilder() .setAction(RD_GENERATE) .setParameter("link", fileURL) .setParameter("password", "") .setParameter("remote", "0") .setParameter("time", "" + System.currentTimeMillis()) .setBaseURL(RD_BASE_URL) .setReferer(RD_BASE_URL) .toGetMethod(); postMethod.addRequestHeader("X-Requested-With", "XMLHttpRequest"); if (!makeRequest(postMethod)) { throw new ServiceConnectionProblemException("Error posting login info"); } final String content = getContentAsString(); checkRequest(content); httpFile.setFileName(PlugUtils.getStringBetween(content, "file_name\":\"", "\",")); final String dlLink = PlugUtils.getStringBetween(content, "main_link\":\"", "\","); final String genLinks = PlugUtils.getStringBetween(content, "generated_links\":[[", "]],"); final Matcher match = PlugUtils.matcher("\"(.+?)\",\"(.+?)\",\"(.+?)\"", genLinks); while (match.find()) { if (match.group(3).equals(dlLink)) { httpFile.setFileName(match.group(1)); break; } } final HttpMethod dlMethod = getGetMethod(dlLink.replace("\\", "")); if (!tryDownloadAndSaveFile(dlMethod)) { checkProblems(); // if downloading failed throw new ServiceConnectionProblemException( "Error starting download"); // some unknown problem } } else { checkProblems(); throw new ServiceConnectionProblemException(); } }
/** * 执行Http请求 * * @param request 请求数据 * @param strParaFileName 文件类型的参数名 * @param strFilePath 文件路径 * @return * @throws HttpException, IOException */ public HttpResponse execute(HttpRequest request, String strParaFileName, String strFilePath) throws HttpException, IOException { HttpClient httpclient = new HttpClient(connectionManager); // 设置连接超时 int connectionTimeout = defaultConnectionTimeout; if (request.getConnectionTimeout() > 0) { connectionTimeout = request.getConnectionTimeout(); } httpclient.getHttpConnectionManager().getParams().setConnectionTimeout(connectionTimeout); // 设置回应超时 int soTimeout = defaultSoTimeout; if (request.getTimeout() > 0) { soTimeout = request.getTimeout(); } httpclient.getHttpConnectionManager().getParams().setSoTimeout(soTimeout); // 设置等待ConnectionManager释放connection的时间 httpclient.getParams().setConnectionManagerTimeout(defaultHttpConnectionManagerTimeout); String charset = request.getCharset(); charset = charset == null ? DEFAULT_CHARSET : charset; HttpMethod method = null; // get模式且不带上传文件 if (request.getMethod().equals(HttpRequest.METHOD_GET)) { method = new GetMethod(request.getUrl()); method.getParams().setCredentialCharset(charset); // parseNotifyConfig会保证使用GET方法时,request一定使用QueryString method.setQueryString(request.getQueryString()); } else if (strParaFileName.equals("") && strFilePath.equals("")) { // post模式且不带上传文件 method = new PostMethod(request.getUrl()); ((PostMethod) method).addParameters(request.getParameters()); method.addRequestHeader( "Content-Type", "application/x-www-form-urlencoded; text/html; charset=" + charset); } else { // post模式且带上传文件 method = new PostMethod(request.getUrl()); List<Part> parts = new ArrayList<Part>(); for (int i = 0; i < request.getParameters().length; i++) { parts.add( new StringPart( request.getParameters()[i].getName(), request.getParameters()[i].getValue(), charset)); } // 增加文件参数,strParaFileName是参数名,使用本地文件 parts.add(new FilePart(strParaFileName, new FilePartSource(new File(strFilePath)))); // 设置请求体 ((PostMethod) method) .setRequestEntity( new MultipartRequestEntity(parts.toArray(new Part[0]), new HttpMethodParams())); } // 设置Http Header中的User-Agent属性 method.addRequestHeader("User-Agent", "Mozilla/4.0"); HttpResponse response = new HttpResponse(); try { httpclient.executeMethod(method); if (request.getResultType().equals(HttpResultType.STRING)) { response.setStringResult(method.getResponseBodyAsString()); } else if (request.getResultType().equals(HttpResultType.BYTES)) { response.setByteResult(method.getResponseBody()); } response.setResponseHeaders(method.getResponseHeaders()); } catch (UnknownHostException ex) { return null; } catch (IOException ex) { return null; } catch (Exception ex) { return null; } finally { method.releaseConnection(); } return response; }
public NamedList<Object> request(final SolrRequest request, ResponseParser processor) throws SolrServerException, IOException { HttpMethod method = null; InputStream is = null; SolrParams params = request.getParams(); Collection<ContentStream> streams = requestWriter.getContentStreams(request); String path = requestWriter.getPath(request); if (path == null || !path.startsWith("/")) { path = "/select"; } ResponseParser parser = request.getResponseParser(); if (parser == null) { parser = _parser; } // The parser 'wt=' and 'version=' params are used instead of the original params ModifiableSolrParams wparams = new ModifiableSolrParams(); wparams.set(CommonParams.WT, parser.getWriterType()); wparams.set(CommonParams.VERSION, parser.getVersion()); if (params == null) { params = wparams; } else { params = new DefaultSolrParams(wparams, params); } if (_invariantParams != null) { params = new DefaultSolrParams(_invariantParams, params); } int tries = _maxRetries + 1; try { while (tries-- > 0) { // Note: since we aren't do intermittent time keeping // ourselves, the potential non-timeout latency could be as // much as tries-times (plus scheduling effects) the given // timeAllowed. try { if (SolrRequest.METHOD.GET == request.getMethod()) { if (streams != null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!"); } method = new GetMethod(_baseURL + path + ClientUtils.toQueryString(params, false)); } else if (SolrRequest.METHOD.POST == request.getMethod()) { String url = _baseURL + path; boolean isMultipart = (streams != null && streams.size() > 1); if (streams == null || isMultipart) { PostMethod post = new PostMethod(url); post.getParams().setContentCharset("UTF-8"); if (!this.useMultiPartPost && !isMultipart) { post.addRequestHeader( "Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); } List<Part> parts = new LinkedList<Part>(); Iterator<String> iter = params.getParameterNamesIterator(); while (iter.hasNext()) { String p = iter.next(); String[] vals = params.getParams(p); if (vals != null) { for (String v : vals) { if (this.useMultiPartPost || isMultipart) { parts.add(new StringPart(p, v, "UTF-8")); } else { post.addParameter(p, v); } } } } if (isMultipart) { int i = 0; for (ContentStream content : streams) { final ContentStream c = content; String charSet = null; String transferEncoding = null; parts.add( new PartBase(c.getName(), c.getContentType(), charSet, transferEncoding) { @Override protected long lengthOfData() throws IOException { return c.getSize(); } @Override protected void sendData(OutputStream out) throws IOException { InputStream in = c.getStream(); try { IOUtils.copy(in, out); } finally { in.close(); } } }); } } if (parts.size() > 0) { post.setRequestEntity( new MultipartRequestEntity( parts.toArray(new Part[parts.size()]), post.getParams())); } method = post; } // It is has one stream, it is the post body, put the params in the URL else { String pstr = ClientUtils.toQueryString(params, false); PostMethod post = new PostMethod(url + pstr); // Single stream as body // Using a loop just to get the first one final ContentStream[] contentStream = new ContentStream[1]; for (ContentStream content : streams) { contentStream[0] = content; break; } if (contentStream[0] instanceof RequestWriter.LazyContentStream) { post.setRequestEntity( new RequestEntity() { public long getContentLength() { return -1; } public String getContentType() { return contentStream[0].getContentType(); } public boolean isRepeatable() { return false; } public void writeRequest(OutputStream outputStream) throws IOException { ((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream); } }); } else { is = contentStream[0].getStream(); post.setRequestEntity( new InputStreamRequestEntity(is, contentStream[0].getContentType())); } method = post; } } else { throw new SolrServerException("Unsupported method: " + request.getMethod()); } } catch (NoHttpResponseException r) { // This is generally safe to retry on method.releaseConnection(); method = null; if (is != null) { is.close(); } // If out of tries then just rethrow (as normal error). if ((tries < 1)) { throw r; } // log.warn( "Caught: " + r + ". Retrying..." ); } } } catch (IOException ex) { throw new SolrServerException("error reading streams", ex); } method.setFollowRedirects(_followRedirects); method.addRequestHeader("User-Agent", AGENT); if (_allowCompression) { method.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate")); } try { // Execute the method. // System.out.println( "EXECUTE:"+method.getURI() ); int statusCode = _httpClient.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { StringBuilder msg = new StringBuilder(); msg.append(method.getStatusLine().getReasonPhrase()); msg.append("\n\n"); msg.append(method.getStatusText()); msg.append("\n\n"); msg.append("request: " + method.getURI()); throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8")); } // Read the contents String charset = "UTF-8"; if (method instanceof HttpMethodBase) { charset = ((HttpMethodBase) method).getResponseCharSet(); } InputStream respBody = method.getResponseBodyAsStream(); // Jakarta Commons HTTPClient doesn't handle any // compression natively. Handle gzip or deflate // here if applicable. if (_allowCompression) { Header contentEncodingHeader = method.getResponseHeader("Content-Encoding"); if (contentEncodingHeader != null) { String contentEncoding = contentEncodingHeader.getValue(); if (contentEncoding.contains("gzip")) { // log.debug( "wrapping response in GZIPInputStream" ); respBody = new GZIPInputStream(respBody); } else if (contentEncoding.contains("deflate")) { // log.debug( "wrapping response in InflaterInputStream" ); respBody = new InflaterInputStream(respBody); } } else { Header contentTypeHeader = method.getResponseHeader("Content-Type"); if (contentTypeHeader != null) { String contentType = contentTypeHeader.getValue(); if (contentType != null) { if (contentType.startsWith("application/x-gzip-compressed")) { // log.debug( "wrapping response in GZIPInputStream" ); respBody = new GZIPInputStream(respBody); } else if (contentType.startsWith("application/x-deflate")) { // log.debug( "wrapping response in InflaterInputStream" ); respBody = new InflaterInputStream(respBody); } } } } } return processor.processResponse(respBody, charset); } catch (HttpException e) { throw new SolrServerException(e); } catch (IOException e) { throw new SolrServerException(e); } finally { method.releaseConnection(); if (is != null) { is.close(); } } }
@Test public void testAsyncRangeRequests() throws IOException, URISyntaxException, InterruptedException { final URL testResourceUrl = new URL(VAULT_BASE_URI.toURL(), "asyncRangeRequestTestFile.txt"); final MultiThreadedHttpConnectionManager cm = new MultiThreadedHttpConnectionManager(); cm.getParams().setDefaultMaxConnectionsPerHost(50); final HttpClient client = new HttpClient(cm); // prepare 8MiB test data: final byte[] plaintextData = new byte[2097152 * Integer.BYTES]; final ByteBuffer bbIn = ByteBuffer.wrap(plaintextData); for (int i = 0; i < 2097152; i++) { bbIn.putInt(i); } // put request: final EntityEnclosingMethod putMethod = new PutMethod(testResourceUrl.toString()); putMethod.setRequestEntity(new ByteArrayRequestEntity(plaintextData)); final int putResponse = client.executeMethod(putMethod); putMethod.releaseConnection(); Assert.assertEquals(201, putResponse); // multiple async range requests: final List<ForkJoinTask<?>> tasks = new ArrayList<>(); final Random generator = new Random(System.currentTimeMillis()); final AtomicBoolean success = new AtomicBoolean(true); // 10 full interrupted requests: for (int i = 0; i < 10; i++) { final ForkJoinTask<?> task = ForkJoinTask.adapt( () -> { try { final HttpMethod getMethod = new GetMethod(testResourceUrl.toString()); final int statusCode = client.executeMethod(getMethod); if (statusCode != 200) { LOG.error("Invalid status code for interrupted full request"); success.set(false); } getMethod.getResponseBodyAsStream().read(); getMethod.getResponseBodyAsStream().close(); getMethod.releaseConnection(); } catch (IOException e) { throw new RuntimeException(e); } }); tasks.add(task); } // 50 crappy interrupted range requests: for (int i = 0; i < 50; i++) { final int lower = generator.nextInt(plaintextData.length); final ForkJoinTask<?> task = ForkJoinTask.adapt( () -> { try { final HttpMethod getMethod = new GetMethod(testResourceUrl.toString()); getMethod.addRequestHeader("Range", "bytes=" + lower + "-"); final int statusCode = client.executeMethod(getMethod); if (statusCode != 206) { LOG.error("Invalid status code for interrupted range request"); success.set(false); } getMethod.getResponseBodyAsStream().read(); getMethod.getResponseBodyAsStream().close(); getMethod.releaseConnection(); } catch (IOException e) { throw new RuntimeException(e); } }); tasks.add(task); } // 50 normal open range requests: for (int i = 0; i < 50; i++) { final int lower = generator.nextInt(plaintextData.length - 512); final int upper = plaintextData.length - 1; final ForkJoinTask<?> task = ForkJoinTask.adapt( () -> { try { final HttpMethod getMethod = new GetMethod(testResourceUrl.toString()); getMethod.addRequestHeader("Range", "bytes=" + lower + "-"); final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1); final int statusCode = client.executeMethod(getMethod); final byte[] responseBody = new byte[upper - lower + 10]; final int bytesRead = IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody); getMethod.releaseConnection(); if (statusCode != 206) { LOG.error("Invalid status code for open range request"); success.set(false); } else if (upper - lower + 1 != bytesRead) { LOG.error("Invalid response length for open range request"); success.set(false); } else if (!Arrays.equals( expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) { LOG.error("Invalid response body for open range request"); success.set(false); } } catch (IOException e) { throw new RuntimeException(e); } }); tasks.add(task); } // 200 normal closed range requests: for (int i = 0; i < 200; i++) { final int pos1 = generator.nextInt(plaintextData.length - 512); final int pos2 = pos1 + 512; final ForkJoinTask<?> task = ForkJoinTask.adapt( () -> { try { final int lower = Math.min(pos1, pos2); final int upper = Math.max(pos1, pos2); final HttpMethod getMethod = new GetMethod(testResourceUrl.toString()); getMethod.addRequestHeader("Range", "bytes=" + lower + "-" + upper); final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1); final int statusCode = client.executeMethod(getMethod); final byte[] responseBody = new byte[upper - lower + 1]; final int bytesRead = IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody); getMethod.releaseConnection(); if (statusCode != 206) { LOG.error("Invalid status code for closed range request"); success.set(false); } else if (upper - lower + 1 != bytesRead) { LOG.error("Invalid response length for closed range request"); success.set(false); } else if (!Arrays.equals( expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) { LOG.error("Invalid response body for closed range request"); success.set(false); } } catch (IOException e) { throw new RuntimeException(e); } }); tasks.add(task); } Collections.shuffle(tasks, generator); final ForkJoinPool pool = new ForkJoinPool(4); for (ForkJoinTask<?> task : tasks) { pool.execute(task); } for (ForkJoinTask<?> task : tasks) { task.join(); } pool.shutdown(); cm.shutdown(); Assert.assertTrue(success.get()); }
public void authorize(String username, String password) { authentication = Base64Converter.encode(username + ":" + password); method.addRequestHeader("Authorization", "basic " + authentication); }
public void process(Exchange exchange) throws Exception { // if we bridge endpoint then we need to skip matching headers with the HTTP_QUERY to avoid // sending // duplicated headers to the receiver, so use this skipRequestHeaders as the list of headers to // skip Map<String, Object> skipRequestHeaders = null; if (getEndpoint().isBridgeEndpoint()) { exchange.setProperty(Exchange.SKIP_GZIP_ENCODING, Boolean.TRUE); String queryString = exchange.getIn().getHeader(Exchange.HTTP_QUERY, String.class); if (queryString != null) { skipRequestHeaders = URISupport.parseQuery(queryString); } // Need to remove the Host key as it should be not used exchange.getIn().getHeaders().remove("host"); } HttpMethod method = createMethod(exchange); Message in = exchange.getIn(); String httpProtocolVersion = in.getHeader(Exchange.HTTP_PROTOCOL_VERSION, String.class); if (httpProtocolVersion != null) { // set the HTTP protocol version HttpMethodParams params = method.getParams(); params.setVersion(HttpVersion.parse(httpProtocolVersion)); } HeaderFilterStrategy strategy = getEndpoint().getHeaderFilterStrategy(); // propagate headers as HTTP headers for (Map.Entry<String, Object> entry : in.getHeaders().entrySet()) { String key = entry.getKey(); Object headerValue = in.getHeader(key); if (headerValue != null) { // use an iterator as there can be multiple values. (must not use a delimiter, and allow // empty values) final Iterator<?> it = ObjectHelper.createIterator(headerValue, null, true); // the value to add as request header final List<String> values = new ArrayList<String>(); // if its a multi value then check each value if we can add it and for multi values they // should be combined into a single value while (it.hasNext()) { String value = exchange.getContext().getTypeConverter().convertTo(String.class, it.next()); // we should not add headers for the parameters in the uri if we bridge the endpoint // as then we would duplicate headers on both the endpoint uri, and in HTTP headers as // well if (skipRequestHeaders != null && skipRequestHeaders.containsKey(key)) { continue; } if (value != null && strategy != null && !strategy.applyFilterToCamelHeaders(key, value, exchange)) { values.add(value); } } // add the value(s) as a http request header if (values.size() > 0) { // use the default toString of a ArrayList to create in the form [xxx, yyy] // if multi valued, for a single value, then just output the value as is String s = values.size() > 1 ? values.toString() : values.get(0); method.addRequestHeader(key, s); } } } // lets store the result in the output message. try { if (LOG.isDebugEnabled()) { LOG.debug("Executing http {} method: {}", method.getName(), method.getURI().toString()); } int responseCode = executeMethod(method); LOG.debug("Http responseCode: {}", responseCode); if (!throwException) { // if we do not use failed exception then populate response for all response codes populateResponse(exchange, method, in, strategy, responseCode); } else { if (responseCode >= 100 && responseCode < 300) { // only populate response for OK response populateResponse(exchange, method, in, strategy, responseCode); } else { // operation failed so populate exception to throw throw populateHttpOperationFailedException(exchange, method, responseCode); } } } finally { method.releaseConnection(); } }
protected byte[] URLtoByteArray( String location, Http.Method method, Map<String, String> headers, Cookie[] cookies, Http.Auth auth, Http.Body body, List<Http.FilePart> fileParts, Map<String, String> parts, Http.Response response, boolean followRedirects) throws IOException { byte[] bytes = null; HttpMethod httpMethod = null; HttpState httpState = null; try { _cookies.set(null); if (location == null) { return null; } else if (!location.startsWith(Http.HTTP_WITH_SLASH) && !location.startsWith(Http.HTTPS_WITH_SLASH)) { location = Http.HTTP_WITH_SLASH + location; } HostConfiguration hostConfiguration = getHostConfiguration(location); HttpClient httpClient = getClient(hostConfiguration); if (method.equals(Http.Method.POST) || method.equals(Http.Method.PUT)) { if (method.equals(Http.Method.POST)) { httpMethod = new PostMethod(location); } else { httpMethod = new PutMethod(location); } if (body != null) { RequestEntity requestEntity = new StringRequestEntity(body.getContent(), body.getContentType(), body.getCharset()); EntityEnclosingMethod entityEnclosingMethod = (EntityEnclosingMethod) httpMethod; entityEnclosingMethod.setRequestEntity(requestEntity); } else if (method.equals(Http.Method.POST)) { PostMethod postMethod = (PostMethod) httpMethod; processPostMethod(postMethod, fileParts, parts); } } else if (method.equals(Http.Method.DELETE)) { httpMethod = new DeleteMethod(location); } else if (method.equals(Http.Method.HEAD)) { httpMethod = new HeadMethod(location); } else { httpMethod = new GetMethod(location); } if (headers != null) { for (Map.Entry<String, String> header : headers.entrySet()) { httpMethod.addRequestHeader(header.getKey(), header.getValue()); } } if ((method.equals(Http.Method.POST) || method.equals(Http.Method.PUT)) && ((body != null) || ((fileParts != null) && !fileParts.isEmpty()) | ((parts != null) && !parts.isEmpty()))) { } else if (!hasRequestHeader(httpMethod, HttpHeaders.CONTENT_TYPE)) { httpMethod.addRequestHeader( HttpHeaders.CONTENT_TYPE, ContentTypes.APPLICATION_X_WWW_FORM_URLENCODED); } if (!hasRequestHeader(httpMethod, HttpHeaders.USER_AGENT)) { httpMethod.addRequestHeader(HttpHeaders.USER_AGENT, _DEFAULT_USER_AGENT); } httpState = new HttpState(); if ((cookies != null) && (cookies.length > 0)) { org.apache.commons.httpclient.Cookie[] commonsCookies = toCommonsCookies(cookies); httpState.addCookies(commonsCookies); HttpMethodParams httpMethodParams = httpMethod.getParams(); httpMethodParams.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); } if (auth != null) { httpMethod.setDoAuthentication(true); httpState.setCredentials( new AuthScope(auth.getHost(), auth.getPort(), auth.getRealm()), new UsernamePasswordCredentials(auth.getUsername(), auth.getPassword())); } proxifyState(httpState, hostConfiguration); httpClient.executeMethod(hostConfiguration, httpMethod, httpState); Header locationHeader = httpMethod.getResponseHeader("location"); if ((locationHeader != null) && !locationHeader.equals(location)) { String redirect = locationHeader.getValue(); if (followRedirects) { return URLtoByteArray( redirect, Http.Method.GET, headers, cookies, auth, body, fileParts, parts, response, followRedirects); } else { response.setRedirect(redirect); } } InputStream inputStream = httpMethod.getResponseBodyAsStream(); if (inputStream != null) { Header contentLength = httpMethod.getResponseHeader(HttpHeaders.CONTENT_LENGTH); if (contentLength != null) { response.setContentLength(GetterUtil.getInteger(contentLength.getValue())); } Header contentType = httpMethod.getResponseHeader(HttpHeaders.CONTENT_TYPE); if (contentType != null) { response.setContentType(contentType.getValue()); } bytes = FileUtil.getBytes(inputStream); } for (Header header : httpMethod.getResponseHeaders()) { response.addHeader(header.getName(), header.getValue()); } return bytes; } finally { try { if (httpState != null) { _cookies.set(toServletCookies(httpState.getCookies())); } } catch (Exception e) { _log.error(e, e); } try { if (httpMethod != null) { httpMethod.releaseConnection(); } } catch (Exception e) { _log.error(e, e); } } }
/** * 执行Http请求 * * @param request * @return */ public HttpResponse execute(HttpRequest request) { HttpClient httpclient = new HttpClient(connectionManager); // 设置连接超时 int connectionTimeout = defaultConnectionTimeout; if (request.getConnectionTimeout() > 0) { connectionTimeout = request.getConnectionTimeout(); } httpclient.getHttpConnectionManager().getParams().setConnectionTimeout(connectionTimeout); // 设置回应超时 int soTimeout = defaultSoTimeout; if (request.getTimeout() > 0) { soTimeout = request.getTimeout(); } httpclient.getHttpConnectionManager().getParams().setSoTimeout(soTimeout); // 设置等待ConnectionManager释放connection的时间 httpclient.getParams().setConnectionManagerTimeout(defaultHttpConnectionManagerTimeout); String charset = request.getCharset(); charset = charset == null ? DEFAULT_CHARSET : charset; HttpMethod method = null; if (request.getMethod().equals(HttpRequest.METHOD_GET)) { method = new GetMethod(request.getUrl()); method.getParams().setCredentialCharset(charset); // parseNotifyConfig会保证使用GET方法时,request一定使用QueryString method.setQueryString(request.getQueryString()); } else { method = new PostMethod(request.getUrl()); ((PostMethod) method).addParameters(request.getParameters()); method.addRequestHeader( "Content-Type", "application/x-www-form-urlencoded; text/html; charset=" + charset); } // 设置Http Header中的User-Agent属性 method.addRequestHeader("User-Agent", "Mozilla/4.0"); HttpResponse response = new HttpResponse(); try { httpclient.executeMethod(method); if (request.getResultType().equals(HttpResultType.STRING)) { response.setStringResult(method.getResponseBodyAsString()); } else if (request.getResultType().equals(HttpResultType.BYTES)) { response.setByteResult(method.getResponseBody()); } response.setResponseHeaders(method.getResponseHeaders()); } catch (UnknownHostException ex) { return null; } catch (IOException ex) { return null; } catch (Exception ex) { return null; } finally { method.releaseConnection(); } return response; }
protected String executeMethodWithErrorHandling(HttpMethod method) { method.addRequestHeader(HttpMethodParams.USER_AGENT, "trico.eve.dev"); return read(method, exec(method)); }