@Test
public void testUnsatisfiableRangeRequest() throws IOException, URISyntaxException {
final URL testResourceUrl =
new URL(VAULT_BASE_URI.toURL(), "unsatisfiableRangeRequestTestFile.txt");
final HttpClient client = new HttpClient();
// prepare file content:
final byte[] fileContent = "This is some test file content.".getBytes();
// put request:
final EntityEnclosingMethod putMethod = new PutMethod(testResourceUrl.toString());
putMethod.setRequestEntity(new ByteArrayRequestEntity(fileContent));
final int putResponse = client.executeMethod(putMethod);
putMethod.releaseConnection();
Assert.assertEquals(201, putResponse);
// get request:
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
getMethod.addRequestHeader("Range", "chunks=1-2");
final int getResponse = client.executeMethod(getMethod);
final byte[] response = new byte[fileContent.length];
IOUtils.read(getMethod.getResponseBodyAsStream(), response);
getMethod.releaseConnection();
Assert.assertEquals(416, getResponse);
Assert.assertArrayEquals(fileContent, response);
}
private HttpMethod createProxyRequest(String targetUrl, HttpServletRequest request)
throws IOException {
URI targetUri;
try {
targetUri = new URI(uriEncode(targetUrl));
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
HttpMethod commonsHttpMethod =
httpMethodProvider.getMethod(request.getMethod(), targetUri.toString());
commonsHttpMethod.setFollowRedirects(false);
Enumeration<String> headerNames = request.getHeaderNames();
while (headerNames.hasMoreElements()) {
String headerName = headerNames.nextElement();
Enumeration<String> headerVals = request.getHeaders(headerName);
while (headerVals.hasMoreElements()) {
String headerValue = headerVals.nextElement();
headerValue = headerFilter.processRequestHeader(headerName, headerValue);
if (headerValue != null) {
commonsHttpMethod.addRequestHeader(new Header(headerName, headerValue));
}
}
}
return commonsHttpMethod;
}
protected <T> void populateRequestHeaders(
HttpMethod httpMethod, HttpInvocation<T> httpInvocation) {
List<Header> requestHeaders = httpInvocation.getRequestHeaders();
if (!CollectionUtils.isEmpty(requestHeaders)) {
for (Header header : requestHeaders) {
httpMethod.addRequestHeader(header);
}
}
}
private Result excuteMethod(HttpMethod method, String cookie, Map<String, String> headers) {
Result result = new Result();
result.setSuccess(true);
int responseCode = -1;
try {
// String cookie =
// "240=240; CurLoginUserGUID=a44e6f49751b4d8b9659636ae20540d6;
// strCertificate=410df02a24354be8bb194aeec0829589;";
if (cookie != null) {
method.addRequestHeader("Cookie", cookie);
}
if (headers != null) {
for (Entry<String, String> entry : headers.entrySet()) {
method.addRequestHeader(entry.getKey(), entry.getValue());
}
}
method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, httpRequestRetryHandler);
org.apache.commons.httpclient.HttpClient client = this.createHttpClient();
client.executeMethod(method);
responseCode = method.getStatusCode();
if (responseCode != HttpStatus.SC_OK) {
result.setSuccess(false);
result.setErrorCode(ErrorCodeEnum.NOT_FOUND_ERROR);
result.setMessage("responseCode:" + responseCode);
result.setResponseCode(responseCode);
}
InputStream is = method.getResponseBodyAsStream();
try {
result.setResultBytes(IOUtils.toByteArray(is));
} finally {
IOUtils.closeQuietly(is);
}
} catch (Exception e) {
result.setSuccess(false);
result.setErrorCode(ErrorCodeEnum.SYSTEM_ERROR);
result.setMessage(e.getMessage());
result.setCause(e);
} finally {
result.setMethodStatusCode(method.getStatusCode());
method.releaseConnection();
}
return result;
}
private void configureHttpMethod(
boolean skipContentCache, CacheData cacheData, long onceTimeOut, HttpMethod httpMethod) {
if (skipContentCache && null != cacheData) {
if (null != cacheData.getLastModifiedHeader()
&& Constants.NULL != cacheData.getLastModifiedHeader()) {
httpMethod.addRequestHeader(Constants.IF_MODIFIED_SINCE, cacheData.getLastModifiedHeader());
}
if (null != cacheData.getMd5() && Constants.NULL != cacheData.getMd5()) {
httpMethod.addRequestHeader(Constants.CONTENT_MD5, cacheData.getMd5());
}
}
httpMethod.addRequestHeader(Constants.ACCEPT_ENCODING, "gzip,deflate");
HttpMethodParams params = new HttpMethodParams();
params.setSoTimeout((int) onceTimeOut);
httpMethod.setParams(params);
httpClient
.getHostConfiguration()
.setHost(
diamondConfigure.getDomainNameList().get(this.domainNamePos.get()),
diamondConfigure.getPort());
}
public void resetMethod() {
// httpClient.getState().clearCookies();
httpClient = clientMap.get(account.getUsername());
// for(String s1:account.getCookie().split("; ")){
// Cookie cookie = new Cookie(domain, name, value)
// httpClient.getState().addCookie(cookie);
// }
httpMethod = new GetMethod();
// for (Header h : httpMethod.getRequestHeaders()) {
// httpMethod.removeRequestHeader(h);
// }
// httpClient.getState().clearCookies();
httpMethod.addRequestHeader(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36");
httpMethod.addRequestHeader("Host", "weibo.com");
httpMethod.addRequestHeader("Connection", "keep-alive");
httpMethod.addRequestHeader("Accept-Language", "zh-CN,zh;q=0.8");
httpMethod.addRequestHeader("Cache-Control", "max-age=0");
httpMethod.addRequestHeader(
"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
httpMethod.addRequestHeader("Accept-Encoding", "deflate,sdch");
httpMethod.removeRequestHeader("Cookie");
// ck =
// "__utma=15428400.1695645915.1351319378.1351319378.1351319378.1;
// SINAGLOBAL=1743311276659.3694.1359040897731; ssoln=18260402168%40139.com; myuid=2350795254;
// [email protected]; wvr=5; UOR=www.juyouqu.com,widget.weibo.com,login.sina.com.cn;
// UV5PAGE=usr513_160;
// SUE=es%3D6edfb5cebc465646c2b99373cb2a29fe%26ev%3Dv1%26es2%3D2d0e68c7608a20158aa2045b5dfb59ee%26rs0%3DcEHTC0Ck28iBwSm7ReH2rIUG1n%252FJjjyJ3TkFC8oU2gEFF7qEa83xCOp%252FdFSR05fBeMEZuKAm5ZMBr721u67n7Rfq7KrNI7cHkJFqnvrl1ut%252Bid0pcoUAmyM9JP316GEN%252FB9E1AqHJns7jVNcflxekc2QPaMOf4BzOUX7zShB0LM%253D%26rv%3D0; SUP=cv%3D1%26bt%3D1383912034%26et%3D1383998434%26d%3Dc909%26i%3D15d3%26us%3D1%26vf%3D0%26vt%3D0%26ac%3D2%26st%3D0%26uid%3D3807150373%26name%3Djobpassion%2540gmail.com%26nick%3D%25E6%25B5%25B7%25E6%25B7%2598%25E8%25B5%2584%25E8%25AE%25AF2013%26fmp%3D%26lcp%3D2013-10-08%252021%253A20%253A24; SUS=SID-3807150373-1383912034-JA-jqf3k-628deb26b5773de29473c3d67d64b10b; ALF=1386504033; SSOLoginState=1383912034; UUG=usrmdins41458; _s_tentry=weibo.com; Apache=8601501327939.331.1383912035126; ULV=1383912035149:97:5:3:8601501327939.331.1383912035126:1383837545412; UV5=usrmdins311164; SinaRot_wb_r_topic=75"
// httpMethod.addRequestHeader("Cookie", account.getCookie());
// postMethod.recycle();
postMethod = new PostMethod();
postMethod.addRequestHeader(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36");
postMethod.addRequestHeader("Host", "weibo.com");
postMethod.addRequestHeader("Connection", "keep-alive");
postMethod.addRequestHeader("Accept-Language", "zh-CN,zh;q=0.8");
// postMethod.addRequestHeader("Cache-Control", "max-age=0");
postMethod.addRequestHeader("Accept", "*/*");
postMethod.addRequestHeader("Accept-Encoding", "deflate,sdch");
postMethod.addRequestHeader("Origin", "http://weibo.com");
// postMethod.addRequestHeader("Content-Length", "101");
postMethod.addRequestHeader("Content-Type", "application/x-www-form-urlencoded");
postMethod.addRequestHeader("X-Requested-With", "XMLHttpRequest");
postMethod.addRequestHeader("Referer", "http://weibo.com/xiena?refer=interest");
// postMethod.addRequestHeader("Cookie", account.getCookie());
}
@Override
public void run() throws Exception {
super.run();
logger.info("Starting download in TASK " + fileURL);
login();
final GetMethod method = getGetMethod(RD_GENERATE);
if (makeRedirectedRequest(method)) { // we make the main request
checkProblems(); // check problems
final HttpMethod postMethod =
getMethodBuilder()
.setAction(RD_GENERATE)
.setParameter("link", fileURL)
.setParameter("password", "")
.setParameter("remote", "0")
.setParameter("time", "" + System.currentTimeMillis())
.setBaseURL(RD_BASE_URL)
.setReferer(RD_BASE_URL)
.toGetMethod();
postMethod.addRequestHeader("X-Requested-With", "XMLHttpRequest");
if (!makeRequest(postMethod)) {
throw new ServiceConnectionProblemException("Error posting login info");
}
final String content = getContentAsString();
checkRequest(content);
httpFile.setFileName(PlugUtils.getStringBetween(content, "file_name\":\"", "\","));
final String dlLink = PlugUtils.getStringBetween(content, "main_link\":\"", "\",");
final String genLinks = PlugUtils.getStringBetween(content, "generated_links\":[[", "]],");
final Matcher match = PlugUtils.matcher("\"(.+?)\",\"(.+?)\",\"(.+?)\"", genLinks);
while (match.find()) {
if (match.group(3).equals(dlLink)) {
httpFile.setFileName(match.group(1));
break;
}
}
final HttpMethod dlMethod = getGetMethod(dlLink.replace("\\", ""));
if (!tryDownloadAndSaveFile(dlMethod)) {
checkProblems(); // if downloading failed
throw new ServiceConnectionProblemException(
"Error starting download"); // some unknown problem
}
} else {
checkProblems();
throw new ServiceConnectionProblemException();
}
}
/**
* 执行Http请求
*
* @param request 请求数据
* @param strParaFileName 文件类型的参数名
* @param strFilePath 文件路径
* @return
* @throws HttpException, IOException
*/
public HttpResponse execute(HttpRequest request, String strParaFileName, String strFilePath)
throws HttpException, IOException {
HttpClient httpclient = new HttpClient(connectionManager);
// 设置连接超时
int connectionTimeout = defaultConnectionTimeout;
if (request.getConnectionTimeout() > 0) {
connectionTimeout = request.getConnectionTimeout();
}
httpclient.getHttpConnectionManager().getParams().setConnectionTimeout(connectionTimeout);
// 设置回应超时
int soTimeout = defaultSoTimeout;
if (request.getTimeout() > 0) {
soTimeout = request.getTimeout();
}
httpclient.getHttpConnectionManager().getParams().setSoTimeout(soTimeout);
// 设置等待ConnectionManager释放connection的时间
httpclient.getParams().setConnectionManagerTimeout(defaultHttpConnectionManagerTimeout);
String charset = request.getCharset();
charset = charset == null ? DEFAULT_CHARSET : charset;
HttpMethod method = null;
// get模式且不带上传文件
if (request.getMethod().equals(HttpRequest.METHOD_GET)) {
method = new GetMethod(request.getUrl());
method.getParams().setCredentialCharset(charset);
// parseNotifyConfig会保证使用GET方法时，request一定使用QueryString
method.setQueryString(request.getQueryString());
} else if (strParaFileName.equals("") && strFilePath.equals("")) {
// post模式且不带上传文件
method = new PostMethod(request.getUrl());
((PostMethod) method).addParameters(request.getParameters());
method.addRequestHeader(
"Content-Type", "application/x-www-form-urlencoded; text/html; charset=" + charset);
} else {
// post模式且带上传文件
method = new PostMethod(request.getUrl());
List<Part> parts = new ArrayList<Part>();
for (int i = 0; i < request.getParameters().length; i++) {
parts.add(
new StringPart(
request.getParameters()[i].getName(),
request.getParameters()[i].getValue(),
charset));
}
// 增加文件参数，strParaFileName是参数名，使用本地文件
parts.add(new FilePart(strParaFileName, new FilePartSource(new File(strFilePath))));
// 设置请求体
((PostMethod) method)
.setRequestEntity(
new MultipartRequestEntity(parts.toArray(new Part[0]), new HttpMethodParams()));
}
// 设置Http Header中的User-Agent属性
method.addRequestHeader("User-Agent", "Mozilla/4.0");
HttpResponse response = new HttpResponse();
try {
httpclient.executeMethod(method);
if (request.getResultType().equals(HttpResultType.STRING)) {
response.setStringResult(method.getResponseBodyAsString());
} else if (request.getResultType().equals(HttpResultType.BYTES)) {
response.setByteResult(method.getResponseBody());
}
response.setResponseHeaders(method.getResponseHeaders());
} catch (UnknownHostException ex) {
return null;
} catch (IOException ex) {
return null;
} catch (Exception ex) {
return null;
} finally {
method.releaseConnection();
}
return response;
}
public NamedList<Object> request(final SolrRequest request, ResponseParser processor)
throws SolrServerException, IOException {
HttpMethod method = null;
InputStream is = null;
SolrParams params = request.getParams();
Collection<ContentStream> streams = requestWriter.getContentStreams(request);
String path = requestWriter.getPath(request);
if (path == null || !path.startsWith("/")) {
path = "/select";
}
ResponseParser parser = request.getResponseParser();
if (parser == null) {
parser = _parser;
}
// The parser 'wt=' and 'version=' params are used instead of the original params
ModifiableSolrParams wparams = new ModifiableSolrParams();
wparams.set(CommonParams.WT, parser.getWriterType());
wparams.set(CommonParams.VERSION, parser.getVersion());
if (params == null) {
params = wparams;
} else {
params = new DefaultSolrParams(wparams, params);
}
if (_invariantParams != null) {
params = new DefaultSolrParams(_invariantParams, params);
}
int tries = _maxRetries + 1;
try {
while (tries-- > 0) {
// Note: since we aren't do intermittent time keeping
// ourselves, the potential non-timeout latency could be as
// much as tries-times (plus scheduling effects) the given
// timeAllowed.
try {
if (SolrRequest.METHOD.GET == request.getMethod()) {
if (streams != null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
}
method = new GetMethod(_baseURL + path + ClientUtils.toQueryString(params, false));
} else if (SolrRequest.METHOD.POST == request.getMethod()) {
String url = _baseURL + path;
boolean isMultipart = (streams != null && streams.size() > 1);
if (streams == null || isMultipart) {
PostMethod post = new PostMethod(url);
post.getParams().setContentCharset("UTF-8");
if (!this.useMultiPartPost && !isMultipart) {
post.addRequestHeader(
"Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
}
List<Part> parts = new LinkedList<Part>();
Iterator<String> iter = params.getParameterNamesIterator();
while (iter.hasNext()) {
String p = iter.next();
String[] vals = params.getParams(p);
if (vals != null) {
for (String v : vals) {
if (this.useMultiPartPost || isMultipart) {
parts.add(new StringPart(p, v, "UTF-8"));
} else {
post.addParameter(p, v);
}
}
}
}
if (isMultipart) {
int i = 0;
for (ContentStream content : streams) {
final ContentStream c = content;
String charSet = null;
String transferEncoding = null;
parts.add(
new PartBase(c.getName(), c.getContentType(), charSet, transferEncoding) {
@Override
protected long lengthOfData() throws IOException {
return c.getSize();
}
@Override
protected void sendData(OutputStream out) throws IOException {
InputStream in = c.getStream();
try {
IOUtils.copy(in, out);
} finally {
in.close();
}
}
});
}
}
if (parts.size() > 0) {
post.setRequestEntity(
new MultipartRequestEntity(
parts.toArray(new Part[parts.size()]), post.getParams()));
}
method = post;
}
// It is has one stream, it is the post body, put the params in the URL
else {
String pstr = ClientUtils.toQueryString(params, false);
PostMethod post = new PostMethod(url + pstr);
// Single stream as body
// Using a loop just to get the first one
final ContentStream[] contentStream = new ContentStream[1];
for (ContentStream content : streams) {
contentStream[0] = content;
break;
}
if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
post.setRequestEntity(
new RequestEntity() {
public long getContentLength() {
return -1;
}
public String getContentType() {
return contentStream[0].getContentType();
}
public boolean isRepeatable() {
return false;
}
public void writeRequest(OutputStream outputStream) throws IOException {
((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream);
}
});
} else {
is = contentStream[0].getStream();
post.setRequestEntity(
new InputStreamRequestEntity(is, contentStream[0].getContentType()));
}
method = post;
}
} else {
throw new SolrServerException("Unsupported method: " + request.getMethod());
}
} catch (NoHttpResponseException r) {
// This is generally safe to retry on
method.releaseConnection();
method = null;
if (is != null) {
is.close();
}
// If out of tries then just rethrow (as normal error).
if ((tries < 1)) {
throw r;
}
// log.warn( "Caught: " + r + ". Retrying..." );
}
}
} catch (IOException ex) {
throw new SolrServerException("error reading streams", ex);
}
method.setFollowRedirects(_followRedirects);
method.addRequestHeader("User-Agent", AGENT);
if (_allowCompression) {
method.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate"));
}
try {
// Execute the method.
// System.out.println( "EXECUTE:"+method.getURI() );
int statusCode = _httpClient.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
StringBuilder msg = new StringBuilder();
msg.append(method.getStatusLine().getReasonPhrase());
msg.append("\n\n");
msg.append(method.getStatusText());
msg.append("\n\n");
msg.append("request: " + method.getURI());
throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8"));
}
// Read the contents
String charset = "UTF-8";
if (method instanceof HttpMethodBase) {
charset = ((HttpMethodBase) method).getResponseCharSet();
}
InputStream respBody = method.getResponseBodyAsStream();
// Jakarta Commons HTTPClient doesn't handle any
// compression natively. Handle gzip or deflate
// here if applicable.
if (_allowCompression) {
Header contentEncodingHeader = method.getResponseHeader("Content-Encoding");
if (contentEncodingHeader != null) {
String contentEncoding = contentEncodingHeader.getValue();
if (contentEncoding.contains("gzip")) {
// log.debug( "wrapping response in GZIPInputStream" );
respBody = new GZIPInputStream(respBody);
} else if (contentEncoding.contains("deflate")) {
// log.debug( "wrapping response in InflaterInputStream" );
respBody = new InflaterInputStream(respBody);
}
} else {
Header contentTypeHeader = method.getResponseHeader("Content-Type");
if (contentTypeHeader != null) {
String contentType = contentTypeHeader.getValue();
if (contentType != null) {
if (contentType.startsWith("application/x-gzip-compressed")) {
// log.debug( "wrapping response in GZIPInputStream" );
respBody = new GZIPInputStream(respBody);
} else if (contentType.startsWith("application/x-deflate")) {
// log.debug( "wrapping response in InflaterInputStream" );
respBody = new InflaterInputStream(respBody);
}
}
}
}
}
return processor.processResponse(respBody, charset);
} catch (HttpException e) {
throw new SolrServerException(e);
} catch (IOException e) {
throw new SolrServerException(e);
} finally {
method.releaseConnection();
if (is != null) {
is.close();
}
}
}
@Test
public void testAsyncRangeRequests()
throws IOException, URISyntaxException, InterruptedException {
final URL testResourceUrl = new URL(VAULT_BASE_URI.toURL(), "asyncRangeRequestTestFile.txt");
final MultiThreadedHttpConnectionManager cm = new MultiThreadedHttpConnectionManager();
cm.getParams().setDefaultMaxConnectionsPerHost(50);
final HttpClient client = new HttpClient(cm);
// prepare 8MiB test data:
final byte[] plaintextData = new byte[2097152 * Integer.BYTES];
final ByteBuffer bbIn = ByteBuffer.wrap(plaintextData);
for (int i = 0; i < 2097152; i++) {
bbIn.putInt(i);
}
// put request:
final EntityEnclosingMethod putMethod = new PutMethod(testResourceUrl.toString());
putMethod.setRequestEntity(new ByteArrayRequestEntity(plaintextData));
final int putResponse = client.executeMethod(putMethod);
putMethod.releaseConnection();
Assert.assertEquals(201, putResponse);
// multiple async range requests:
final List<ForkJoinTask<?>> tasks = new ArrayList<>();
final Random generator = new Random(System.currentTimeMillis());
final AtomicBoolean success = new AtomicBoolean(true);
// 10 full interrupted requests:
for (int i = 0; i < 10; i++) {
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
final int statusCode = client.executeMethod(getMethod);
if (statusCode != 200) {
LOG.error("Invalid status code for interrupted full request");
success.set(false);
}
getMethod.getResponseBodyAsStream().read();
getMethod.getResponseBodyAsStream().close();
getMethod.releaseConnection();
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 50 crappy interrupted range requests:
for (int i = 0; i < 50; i++) {
final int lower = generator.nextInt(plaintextData.length);
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
getMethod.addRequestHeader("Range", "bytes=" + lower + "-");
final int statusCode = client.executeMethod(getMethod);
if (statusCode != 206) {
LOG.error("Invalid status code for interrupted range request");
success.set(false);
}
getMethod.getResponseBodyAsStream().read();
getMethod.getResponseBodyAsStream().close();
getMethod.releaseConnection();
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 50 normal open range requests:
for (int i = 0; i < 50; i++) {
final int lower = generator.nextInt(plaintextData.length - 512);
final int upper = plaintextData.length - 1;
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
getMethod.addRequestHeader("Range", "bytes=" + lower + "-");
final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1);
final int statusCode = client.executeMethod(getMethod);
final byte[] responseBody = new byte[upper - lower + 10];
final int bytesRead =
IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody);
getMethod.releaseConnection();
if (statusCode != 206) {
LOG.error("Invalid status code for open range request");
success.set(false);
} else if (upper - lower + 1 != bytesRead) {
LOG.error("Invalid response length for open range request");
success.set(false);
} else if (!Arrays.equals(
expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) {
LOG.error("Invalid response body for open range request");
success.set(false);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 200 normal closed range requests:
for (int i = 0; i < 200; i++) {
final int pos1 = generator.nextInt(plaintextData.length - 512);
final int pos2 = pos1 + 512;
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final int lower = Math.min(pos1, pos2);
final int upper = Math.max(pos1, pos2);
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
getMethod.addRequestHeader("Range", "bytes=" + lower + "-" + upper);
final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1);
final int statusCode = client.executeMethod(getMethod);
final byte[] responseBody = new byte[upper - lower + 1];
final int bytesRead =
IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody);
getMethod.releaseConnection();
if (statusCode != 206) {
LOG.error("Invalid status code for closed range request");
success.set(false);
} else if (upper - lower + 1 != bytesRead) {
LOG.error("Invalid response length for closed range request");
success.set(false);
} else if (!Arrays.equals(
expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) {
LOG.error("Invalid response body for closed range request");
success.set(false);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
Collections.shuffle(tasks, generator);
final ForkJoinPool pool = new ForkJoinPool(4);
for (ForkJoinTask<?> task : tasks) {
pool.execute(task);
}
for (ForkJoinTask<?> task : tasks) {
task.join();
}
pool.shutdown();
cm.shutdown();
Assert.assertTrue(success.get());
}
public void authorize(String username, String password) {
authentication = Base64Converter.encode(username + ":" + password);
method.addRequestHeader("Authorization", "basic " + authentication);
}
public void process(Exchange exchange) throws Exception {
// if we bridge endpoint then we need to skip matching headers with the HTTP_QUERY to avoid
// sending
// duplicated headers to the receiver, so use this skipRequestHeaders as the list of headers to
// skip
Map<String, Object> skipRequestHeaders = null;
if (getEndpoint().isBridgeEndpoint()) {
exchange.setProperty(Exchange.SKIP_GZIP_ENCODING, Boolean.TRUE);
String queryString = exchange.getIn().getHeader(Exchange.HTTP_QUERY, String.class);
if (queryString != null) {
skipRequestHeaders = URISupport.parseQuery(queryString);
}
// Need to remove the Host key as it should be not used
exchange.getIn().getHeaders().remove("host");
}
HttpMethod method = createMethod(exchange);
Message in = exchange.getIn();
String httpProtocolVersion = in.getHeader(Exchange.HTTP_PROTOCOL_VERSION, String.class);
if (httpProtocolVersion != null) {
// set the HTTP protocol version
HttpMethodParams params = method.getParams();
params.setVersion(HttpVersion.parse(httpProtocolVersion));
}
HeaderFilterStrategy strategy = getEndpoint().getHeaderFilterStrategy();
// propagate headers as HTTP headers
for (Map.Entry<String, Object> entry : in.getHeaders().entrySet()) {
String key = entry.getKey();
Object headerValue = in.getHeader(key);
if (headerValue != null) {
// use an iterator as there can be multiple values. (must not use a delimiter, and allow
// empty values)
final Iterator<?> it = ObjectHelper.createIterator(headerValue, null, true);
// the value to add as request header
final List<String> values = new ArrayList<String>();
// if its a multi value then check each value if we can add it and for multi values they
// should be combined into a single value
while (it.hasNext()) {
String value =
exchange.getContext().getTypeConverter().convertTo(String.class, it.next());
// we should not add headers for the parameters in the uri if we bridge the endpoint
// as then we would duplicate headers on both the endpoint uri, and in HTTP headers as
// well
if (skipRequestHeaders != null && skipRequestHeaders.containsKey(key)) {
continue;
}
if (value != null
&& strategy != null
&& !strategy.applyFilterToCamelHeaders(key, value, exchange)) {
values.add(value);
}
}
// add the value(s) as a http request header
if (values.size() > 0) {
// use the default toString of a ArrayList to create in the form [xxx, yyy]
// if multi valued, for a single value, then just output the value as is
String s = values.size() > 1 ? values.toString() : values.get(0);
method.addRequestHeader(key, s);
}
}
}
// lets store the result in the output message.
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Executing http {} method: {}", method.getName(), method.getURI().toString());
}
int responseCode = executeMethod(method);
LOG.debug("Http responseCode: {}", responseCode);
if (!throwException) {
// if we do not use failed exception then populate response for all response codes
populateResponse(exchange, method, in, strategy, responseCode);
} else {
if (responseCode >= 100 && responseCode < 300) {
// only populate response for OK response
populateResponse(exchange, method, in, strategy, responseCode);
} else {
// operation failed so populate exception to throw
throw populateHttpOperationFailedException(exchange, method, responseCode);
}
}
} finally {
method.releaseConnection();
}
}
protected byte[] URLtoByteArray(
String location,
Http.Method method,
Map<String, String> headers,
Cookie[] cookies,
Http.Auth auth,
Http.Body body,
List<Http.FilePart> fileParts,
Map<String, String> parts,
Http.Response response,
boolean followRedirects)
throws IOException {
byte[] bytes = null;
HttpMethod httpMethod = null;
HttpState httpState = null;
try {
_cookies.set(null);
if (location == null) {
return null;
} else if (!location.startsWith(Http.HTTP_WITH_SLASH)
&& !location.startsWith(Http.HTTPS_WITH_SLASH)) {
location = Http.HTTP_WITH_SLASH + location;
}
HostConfiguration hostConfiguration = getHostConfiguration(location);
HttpClient httpClient = getClient(hostConfiguration);
if (method.equals(Http.Method.POST) || method.equals(Http.Method.PUT)) {
if (method.equals(Http.Method.POST)) {
httpMethod = new PostMethod(location);
} else {
httpMethod = new PutMethod(location);
}
if (body != null) {
RequestEntity requestEntity =
new StringRequestEntity(body.getContent(), body.getContentType(), body.getCharset());
EntityEnclosingMethod entityEnclosingMethod = (EntityEnclosingMethod) httpMethod;
entityEnclosingMethod.setRequestEntity(requestEntity);
} else if (method.equals(Http.Method.POST)) {
PostMethod postMethod = (PostMethod) httpMethod;
processPostMethod(postMethod, fileParts, parts);
}
} else if (method.equals(Http.Method.DELETE)) {
httpMethod = new DeleteMethod(location);
} else if (method.equals(Http.Method.HEAD)) {
httpMethod = new HeadMethod(location);
} else {
httpMethod = new GetMethod(location);
}
if (headers != null) {
for (Map.Entry<String, String> header : headers.entrySet()) {
httpMethod.addRequestHeader(header.getKey(), header.getValue());
}
}
if ((method.equals(Http.Method.POST) || method.equals(Http.Method.PUT))
&& ((body != null)
|| ((fileParts != null) && !fileParts.isEmpty())
| ((parts != null) && !parts.isEmpty()))) {
} else if (!hasRequestHeader(httpMethod, HttpHeaders.CONTENT_TYPE)) {
httpMethod.addRequestHeader(
HttpHeaders.CONTENT_TYPE, ContentTypes.APPLICATION_X_WWW_FORM_URLENCODED);
}
if (!hasRequestHeader(httpMethod, HttpHeaders.USER_AGENT)) {
httpMethod.addRequestHeader(HttpHeaders.USER_AGENT, _DEFAULT_USER_AGENT);
}
httpState = new HttpState();
if ((cookies != null) && (cookies.length > 0)) {
org.apache.commons.httpclient.Cookie[] commonsCookies = toCommonsCookies(cookies);
httpState.addCookies(commonsCookies);
HttpMethodParams httpMethodParams = httpMethod.getParams();
httpMethodParams.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
}
if (auth != null) {
httpMethod.setDoAuthentication(true);
httpState.setCredentials(
new AuthScope(auth.getHost(), auth.getPort(), auth.getRealm()),
new UsernamePasswordCredentials(auth.getUsername(), auth.getPassword()));
}
proxifyState(httpState, hostConfiguration);
httpClient.executeMethod(hostConfiguration, httpMethod, httpState);
Header locationHeader = httpMethod.getResponseHeader("location");
if ((locationHeader != null) && !locationHeader.equals(location)) {
String redirect = locationHeader.getValue();
if (followRedirects) {
return URLtoByteArray(
redirect,
Http.Method.GET,
headers,
cookies,
auth,
body,
fileParts,
parts,
response,
followRedirects);
} else {
response.setRedirect(redirect);
}
}
InputStream inputStream = httpMethod.getResponseBodyAsStream();
if (inputStream != null) {
Header contentLength = httpMethod.getResponseHeader(HttpHeaders.CONTENT_LENGTH);
if (contentLength != null) {
response.setContentLength(GetterUtil.getInteger(contentLength.getValue()));
}
Header contentType = httpMethod.getResponseHeader(HttpHeaders.CONTENT_TYPE);
if (contentType != null) {
response.setContentType(contentType.getValue());
}
bytes = FileUtil.getBytes(inputStream);
}
for (Header header : httpMethod.getResponseHeaders()) {
response.addHeader(header.getName(), header.getValue());
}
return bytes;
} finally {
try {
if (httpState != null) {
_cookies.set(toServletCookies(httpState.getCookies()));
}
} catch (Exception e) {
_log.error(e, e);
}
try {
if (httpMethod != null) {
httpMethod.releaseConnection();
}
} catch (Exception e) {
_log.error(e, e);
}
}
}
/**
* 执行Http请求
*
* @param request
* @return
*/
public HttpResponse execute(HttpRequest request) {
HttpClient httpclient = new HttpClient(connectionManager);
// 设置连接超时
int connectionTimeout = defaultConnectionTimeout;
if (request.getConnectionTimeout() > 0) {
connectionTimeout = request.getConnectionTimeout();
}
httpclient.getHttpConnectionManager().getParams().setConnectionTimeout(connectionTimeout);
// 设置回应超时
int soTimeout = defaultSoTimeout;
if (request.getTimeout() > 0) {
soTimeout = request.getTimeout();
}
httpclient.getHttpConnectionManager().getParams().setSoTimeout(soTimeout);
// 设置等待ConnectionManager释放connection的时间
httpclient.getParams().setConnectionManagerTimeout(defaultHttpConnectionManagerTimeout);
String charset = request.getCharset();
charset = charset == null ? DEFAULT_CHARSET : charset;
HttpMethod method = null;
if (request.getMethod().equals(HttpRequest.METHOD_GET)) {
method = new GetMethod(request.getUrl());
method.getParams().setCredentialCharset(charset);
// parseNotifyConfig会保证使用GET方法时，request一定使用QueryString
method.setQueryString(request.getQueryString());
} else {
method = new PostMethod(request.getUrl());
((PostMethod) method).addParameters(request.getParameters());
method.addRequestHeader(
"Content-Type", "application/x-www-form-urlencoded; text/html; charset=" + charset);
}
// 设置Http Header中的User-Agent属性
method.addRequestHeader("User-Agent", "Mozilla/4.0");
HttpResponse response = new HttpResponse();
try {
httpclient.executeMethod(method);
if (request.getResultType().equals(HttpResultType.STRING)) {
response.setStringResult(method.getResponseBodyAsString());
} else if (request.getResultType().equals(HttpResultType.BYTES)) {
response.setByteResult(method.getResponseBody());
}
response.setResponseHeaders(method.getResponseHeaders());
} catch (UnknownHostException ex) {
return null;
} catch (IOException ex) {
return null;
} catch (Exception ex) {
return null;
} finally {
method.releaseConnection();
}
return response;
}
protected String executeMethodWithErrorHandling(HttpMethod method) {
method.addRequestHeader(HttpMethodParams.USER_AGENT, "trico.eve.dev");
return read(method, exec(method));
}