Exemplo n.º 1
0
  @Test
  public void testUnsatisfiableRangeRequest() throws IOException, URISyntaxException {
    final URL testResourceUrl =
        new URL(VAULT_BASE_URI.toURL(), "unsatisfiableRangeRequestTestFile.txt");
    final HttpClient client = new HttpClient();

    // prepare file content:
    final byte[] fileContent = "This is some test file content.".getBytes();

    // put request:
    final EntityEnclosingMethod putMethod = new PutMethod(testResourceUrl.toString());
    putMethod.setRequestEntity(new ByteArrayRequestEntity(fileContent));
    final int putResponse = client.executeMethod(putMethod);
    putMethod.releaseConnection();
    Assert.assertEquals(201, putResponse);

    // get request:
    final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
    getMethod.addRequestHeader("Range", "chunks=1-2");
    final int getResponse = client.executeMethod(getMethod);
    final byte[] response = new byte[fileContent.length];
    IOUtils.read(getMethod.getResponseBodyAsStream(), response);
    getMethod.releaseConnection();
    Assert.assertEquals(416, getResponse);
    Assert.assertArrayEquals(fileContent, response);
  }
Exemplo n.º 2
0
  private HttpMethod createProxyRequest(String targetUrl, HttpServletRequest request)
      throws IOException {
    URI targetUri;
    try {
      targetUri = new URI(uriEncode(targetUrl));
    } catch (URISyntaxException e) {
      throw new RuntimeException(e);
    }

    HttpMethod commonsHttpMethod =
        httpMethodProvider.getMethod(request.getMethod(), targetUri.toString());

    commonsHttpMethod.setFollowRedirects(false);

    Enumeration<String> headerNames = request.getHeaderNames();
    while (headerNames.hasMoreElements()) {
      String headerName = headerNames.nextElement();
      Enumeration<String> headerVals = request.getHeaders(headerName);
      while (headerVals.hasMoreElements()) {
        String headerValue = headerVals.nextElement();
        headerValue = headerFilter.processRequestHeader(headerName, headerValue);
        if (headerValue != null) {
          commonsHttpMethod.addRequestHeader(new Header(headerName, headerValue));
        }
      }
    }

    return commonsHttpMethod;
  }
Exemplo n.º 3
0
 protected <T> void populateRequestHeaders(
     HttpMethod httpMethod, HttpInvocation<T> httpInvocation) {
   List<Header> requestHeaders = httpInvocation.getRequestHeaders();
   if (!CollectionUtils.isEmpty(requestHeaders)) {
     for (Header header : requestHeaders) {
       httpMethod.addRequestHeader(header);
     }
   }
 }
Exemplo n.º 4
0
  private Result excuteMethod(HttpMethod method, String cookie, Map<String, String> headers) {
    Result result = new Result();
    result.setSuccess(true);
    int responseCode = -1;
    try {
      // String cookie =
      // "240=240; CurLoginUserGUID=a44e6f49751b4d8b9659636ae20540d6;
      // strCertificate=410df02a24354be8bb194aeec0829589;";
      if (cookie != null) {
        method.addRequestHeader("Cookie", cookie);
      }
      if (headers != null) {
        for (Entry<String, String> entry : headers.entrySet()) {
          method.addRequestHeader(entry.getKey(), entry.getValue());
        }
      }
      method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, httpRequestRetryHandler);
      org.apache.commons.httpclient.HttpClient client = this.createHttpClient();
      client.executeMethod(method);
      responseCode = method.getStatusCode();
      if (responseCode != HttpStatus.SC_OK) {
        result.setSuccess(false);
        result.setErrorCode(ErrorCodeEnum.NOT_FOUND_ERROR);
        result.setMessage("responseCode:" + responseCode);
        result.setResponseCode(responseCode);
      }
      InputStream is = method.getResponseBodyAsStream();
      try {
        result.setResultBytes(IOUtils.toByteArray(is));
      } finally {
        IOUtils.closeQuietly(is);
      }

    } catch (Exception e) {
      result.setSuccess(false);
      result.setErrorCode(ErrorCodeEnum.SYSTEM_ERROR);
      result.setMessage(e.getMessage());
      result.setCause(e);
    } finally {
      result.setMethodStatusCode(method.getStatusCode());
      method.releaseConnection();
    }
    return result;
  }
  private void configureHttpMethod(
      boolean skipContentCache, CacheData cacheData, long onceTimeOut, HttpMethod httpMethod) {
    if (skipContentCache && null != cacheData) {
      if (null != cacheData.getLastModifiedHeader()
          && Constants.NULL != cacheData.getLastModifiedHeader()) {
        httpMethod.addRequestHeader(Constants.IF_MODIFIED_SINCE, cacheData.getLastModifiedHeader());
      }
      if (null != cacheData.getMd5() && Constants.NULL != cacheData.getMd5()) {
        httpMethod.addRequestHeader(Constants.CONTENT_MD5, cacheData.getMd5());
      }
    }

    httpMethod.addRequestHeader(Constants.ACCEPT_ENCODING, "gzip,deflate");

    HttpMethodParams params = new HttpMethodParams();
    params.setSoTimeout((int) onceTimeOut);
    httpMethod.setParams(params);
    httpClient
        .getHostConfiguration()
        .setHost(
            diamondConfigure.getDomainNameList().get(this.domainNamePos.get()),
            diamondConfigure.getPort());
  }
Exemplo n.º 6
0
  public void resetMethod() {
    //		httpClient.getState().clearCookies();
    httpClient = clientMap.get(account.getUsername());
    //		for(String s1:account.getCookie().split("; ")){
    //			Cookie cookie = new Cookie(domain, name, value)
    //			httpClient.getState().addCookie(cookie);
    //		}
    httpMethod = new GetMethod();
    //		for (Header h : httpMethod.getRequestHeaders()) {
    //			httpMethod.removeRequestHeader(h);
    //		}
    //		httpClient.getState().clearCookies();
    httpMethod.addRequestHeader(
        "User-Agent",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36");
    httpMethod.addRequestHeader("Host", "weibo.com");
    httpMethod.addRequestHeader("Connection", "keep-alive");
    httpMethod.addRequestHeader("Accept-Language", "zh-CN,zh;q=0.8");
    httpMethod.addRequestHeader("Cache-Control", "max-age=0");
    httpMethod.addRequestHeader(
        "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
    httpMethod.addRequestHeader("Accept-Encoding", "deflate,sdch");
    httpMethod.removeRequestHeader("Cookie");
    // ck =
    // "__utma=15428400.1695645915.1351319378.1351319378.1351319378.1;
    // SINAGLOBAL=1743311276659.3694.1359040897731; ssoln=18260402168%40139.com; myuid=2350795254;
    // [email protected]; wvr=5; UOR=www.juyouqu.com,widget.weibo.com,login.sina.com.cn;
    // UV5PAGE=usr513_160;
    // SUE=es%3D6edfb5cebc465646c2b99373cb2a29fe%26ev%3Dv1%26es2%3D2d0e68c7608a20158aa2045b5dfb59ee%26rs0%3DcEHTC0Ck28iBwSm7ReH2rIUG1n%252FJjjyJ3TkFC8oU2gEFF7qEa83xCOp%252FdFSR05fBeMEZuKAm5ZMBr721u67n7Rfq7KrNI7cHkJFqnvrl1ut%252Bid0pcoUAmyM9JP316GEN%252FB9E1AqHJns7jVNcflxekc2QPaMOf4BzOUX7zShB0LM%253D%26rv%3D0; SUP=cv%3D1%26bt%3D1383912034%26et%3D1383998434%26d%3Dc909%26i%3D15d3%26us%3D1%26vf%3D0%26vt%3D0%26ac%3D2%26st%3D0%26uid%3D3807150373%26name%3Djobpassion%2540gmail.com%26nick%3D%25E6%25B5%25B7%25E6%25B7%2598%25E8%25B5%2584%25E8%25AE%25AF2013%26fmp%3D%26lcp%3D2013-10-08%252021%253A20%253A24; SUS=SID-3807150373-1383912034-JA-jqf3k-628deb26b5773de29473c3d67d64b10b; ALF=1386504033; SSOLoginState=1383912034; UUG=usrmdins41458; _s_tentry=weibo.com; Apache=8601501327939.331.1383912035126; ULV=1383912035149:97:5:3:8601501327939.331.1383912035126:1383837545412; UV5=usrmdins311164; SinaRot_wb_r_topic=75"
    //		httpMethod.addRequestHeader("Cookie", account.getCookie());

    //		postMethod.recycle();
    postMethod = new PostMethod();
    postMethod.addRequestHeader(
        "User-Agent",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36");
    postMethod.addRequestHeader("Host", "weibo.com");
    postMethod.addRequestHeader("Connection", "keep-alive");
    postMethod.addRequestHeader("Accept-Language", "zh-CN,zh;q=0.8");
    // postMethod.addRequestHeader("Cache-Control", "max-age=0");
    postMethod.addRequestHeader("Accept", "*/*");
    postMethod.addRequestHeader("Accept-Encoding", "deflate,sdch");
    postMethod.addRequestHeader("Origin", "http://weibo.com");
    // postMethod.addRequestHeader("Content-Length", "101");
    postMethod.addRequestHeader("Content-Type", "application/x-www-form-urlencoded");
    postMethod.addRequestHeader("X-Requested-With", "XMLHttpRequest");
    postMethod.addRequestHeader("Referer", "http://weibo.com/xiena?refer=interest");
    //		postMethod.addRequestHeader("Cookie", account.getCookie());

  }
 @Override
 public void run() throws Exception {
   super.run();
   logger.info("Starting download in TASK " + fileURL);
   login();
   final GetMethod method = getGetMethod(RD_GENERATE);
   if (makeRedirectedRequest(method)) { // we make the main request
     checkProblems(); // check problems
     final HttpMethod postMethod =
         getMethodBuilder()
             .setAction(RD_GENERATE)
             .setParameter("link", fileURL)
             .setParameter("password", "")
             .setParameter("remote", "0")
             .setParameter("time", "" + System.currentTimeMillis())
             .setBaseURL(RD_BASE_URL)
             .setReferer(RD_BASE_URL)
             .toGetMethod();
     postMethod.addRequestHeader("X-Requested-With", "XMLHttpRequest");
     if (!makeRequest(postMethod)) {
       throw new ServiceConnectionProblemException("Error posting login info");
     }
     final String content = getContentAsString();
     checkRequest(content);
     httpFile.setFileName(PlugUtils.getStringBetween(content, "file_name\":\"", "\","));
     final String dlLink = PlugUtils.getStringBetween(content, "main_link\":\"", "\",");
     final String genLinks = PlugUtils.getStringBetween(content, "generated_links\":[[", "]],");
     final Matcher match = PlugUtils.matcher("\"(.+?)\",\"(.+?)\",\"(.+?)\"", genLinks);
     while (match.find()) {
       if (match.group(3).equals(dlLink)) {
         httpFile.setFileName(match.group(1));
         break;
       }
     }
     final HttpMethod dlMethod = getGetMethod(dlLink.replace("\\", ""));
     if (!tryDownloadAndSaveFile(dlMethod)) {
       checkProblems(); // if downloading failed
       throw new ServiceConnectionProblemException(
           "Error starting download"); // some unknown problem
     }
   } else {
     checkProblems();
     throw new ServiceConnectionProblemException();
   }
 }
  /**
   * 执行Http请求
   *
   * @param request 请求数据
   * @param strParaFileName 文件类型的参数名
   * @param strFilePath 文件路径
   * @return
   * @throws HttpException, IOException
   */
  public HttpResponse execute(HttpRequest request, String strParaFileName, String strFilePath)
      throws HttpException, IOException {
    HttpClient httpclient = new HttpClient(connectionManager);

    // 设置连接超时
    int connectionTimeout = defaultConnectionTimeout;
    if (request.getConnectionTimeout() > 0) {
      connectionTimeout = request.getConnectionTimeout();
    }
    httpclient.getHttpConnectionManager().getParams().setConnectionTimeout(connectionTimeout);

    // 设置回应超时
    int soTimeout = defaultSoTimeout;
    if (request.getTimeout() > 0) {
      soTimeout = request.getTimeout();
    }
    httpclient.getHttpConnectionManager().getParams().setSoTimeout(soTimeout);

    // 设置等待ConnectionManager释放connection的时间
    httpclient.getParams().setConnectionManagerTimeout(defaultHttpConnectionManagerTimeout);

    String charset = request.getCharset();
    charset = charset == null ? DEFAULT_CHARSET : charset;
    HttpMethod method = null;

    // get模式且不带上传文件
    if (request.getMethod().equals(HttpRequest.METHOD_GET)) {
      method = new GetMethod(request.getUrl());
      method.getParams().setCredentialCharset(charset);

      // parseNotifyConfig会保证使用GET方法时,request一定使用QueryString
      method.setQueryString(request.getQueryString());
    } else if (strParaFileName.equals("") && strFilePath.equals("")) {
      // post模式且不带上传文件
      method = new PostMethod(request.getUrl());
      ((PostMethod) method).addParameters(request.getParameters());
      method.addRequestHeader(
          "Content-Type", "application/x-www-form-urlencoded; text/html; charset=" + charset);
    } else {
      // post模式且带上传文件
      method = new PostMethod(request.getUrl());
      List<Part> parts = new ArrayList<Part>();
      for (int i = 0; i < request.getParameters().length; i++) {
        parts.add(
            new StringPart(
                request.getParameters()[i].getName(),
                request.getParameters()[i].getValue(),
                charset));
      }
      // 增加文件参数,strParaFileName是参数名,使用本地文件
      parts.add(new FilePart(strParaFileName, new FilePartSource(new File(strFilePath))));

      // 设置请求体
      ((PostMethod) method)
          .setRequestEntity(
              new MultipartRequestEntity(parts.toArray(new Part[0]), new HttpMethodParams()));
    }

    // 设置Http Header中的User-Agent属性
    method.addRequestHeader("User-Agent", "Mozilla/4.0");
    HttpResponse response = new HttpResponse();

    try {
      httpclient.executeMethod(method);
      if (request.getResultType().equals(HttpResultType.STRING)) {
        response.setStringResult(method.getResponseBodyAsString());
      } else if (request.getResultType().equals(HttpResultType.BYTES)) {
        response.setByteResult(method.getResponseBody());
      }
      response.setResponseHeaders(method.getResponseHeaders());
    } catch (UnknownHostException ex) {

      return null;
    } catch (IOException ex) {

      return null;
    } catch (Exception ex) {

      return null;
    } finally {
      method.releaseConnection();
    }
    return response;
  }
  public NamedList<Object> request(final SolrRequest request, ResponseParser processor)
      throws SolrServerException, IOException {
    HttpMethod method = null;
    InputStream is = null;
    SolrParams params = request.getParams();
    Collection<ContentStream> streams = requestWriter.getContentStreams(request);
    String path = requestWriter.getPath(request);
    if (path == null || !path.startsWith("/")) {
      path = "/select";
    }

    ResponseParser parser = request.getResponseParser();
    if (parser == null) {
      parser = _parser;
    }

    // The parser 'wt=' and 'version=' params are used instead of the original params
    ModifiableSolrParams wparams = new ModifiableSolrParams();
    wparams.set(CommonParams.WT, parser.getWriterType());
    wparams.set(CommonParams.VERSION, parser.getVersion());
    if (params == null) {
      params = wparams;
    } else {
      params = new DefaultSolrParams(wparams, params);
    }

    if (_invariantParams != null) {
      params = new DefaultSolrParams(_invariantParams, params);
    }

    int tries = _maxRetries + 1;
    try {
      while (tries-- > 0) {
        // Note: since we aren't do intermittent time keeping
        // ourselves, the potential non-timeout latency could be as
        // much as tries-times (plus scheduling effects) the given
        // timeAllowed.
        try {
          if (SolrRequest.METHOD.GET == request.getMethod()) {
            if (streams != null) {
              throw new SolrException(
                  SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
            }
            method = new GetMethod(_baseURL + path + ClientUtils.toQueryString(params, false));
          } else if (SolrRequest.METHOD.POST == request.getMethod()) {

            String url = _baseURL + path;
            boolean isMultipart = (streams != null && streams.size() > 1);

            if (streams == null || isMultipart) {
              PostMethod post = new PostMethod(url);
              post.getParams().setContentCharset("UTF-8");
              if (!this.useMultiPartPost && !isMultipart) {
                post.addRequestHeader(
                    "Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
              }

              List<Part> parts = new LinkedList<Part>();
              Iterator<String> iter = params.getParameterNamesIterator();
              while (iter.hasNext()) {
                String p = iter.next();
                String[] vals = params.getParams(p);
                if (vals != null) {
                  for (String v : vals) {
                    if (this.useMultiPartPost || isMultipart) {
                      parts.add(new StringPart(p, v, "UTF-8"));
                    } else {
                      post.addParameter(p, v);
                    }
                  }
                }
              }

              if (isMultipart) {
                int i = 0;
                for (ContentStream content : streams) {
                  final ContentStream c = content;

                  String charSet = null;
                  String transferEncoding = null;
                  parts.add(
                      new PartBase(c.getName(), c.getContentType(), charSet, transferEncoding) {
                        @Override
                        protected long lengthOfData() throws IOException {
                          return c.getSize();
                        }

                        @Override
                        protected void sendData(OutputStream out) throws IOException {
                          InputStream in = c.getStream();
                          try {
                            IOUtils.copy(in, out);
                          } finally {
                            in.close();
                          }
                        }
                      });
                }
              }
              if (parts.size() > 0) {
                post.setRequestEntity(
                    new MultipartRequestEntity(
                        parts.toArray(new Part[parts.size()]), post.getParams()));
              }

              method = post;
            }
            // It is has one stream, it is the post body, put the params in the URL
            else {
              String pstr = ClientUtils.toQueryString(params, false);
              PostMethod post = new PostMethod(url + pstr);

              // Single stream as body
              // Using a loop just to get the first one
              final ContentStream[] contentStream = new ContentStream[1];
              for (ContentStream content : streams) {
                contentStream[0] = content;
                break;
              }
              if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
                post.setRequestEntity(
                    new RequestEntity() {
                      public long getContentLength() {
                        return -1;
                      }

                      public String getContentType() {
                        return contentStream[0].getContentType();
                      }

                      public boolean isRepeatable() {
                        return false;
                      }

                      public void writeRequest(OutputStream outputStream) throws IOException {
                        ((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream);
                      }
                    });

              } else {
                is = contentStream[0].getStream();
                post.setRequestEntity(
                    new InputStreamRequestEntity(is, contentStream[0].getContentType()));
              }
              method = post;
            }
          } else {
            throw new SolrServerException("Unsupported method: " + request.getMethod());
          }
        } catch (NoHttpResponseException r) {
          // This is generally safe to retry on
          method.releaseConnection();
          method = null;
          if (is != null) {
            is.close();
          }
          // If out of tries then just rethrow (as normal error).
          if ((tries < 1)) {
            throw r;
          }
          // log.warn( "Caught: " + r + ". Retrying..." );
        }
      }
    } catch (IOException ex) {
      throw new SolrServerException("error reading streams", ex);
    }

    method.setFollowRedirects(_followRedirects);
    method.addRequestHeader("User-Agent", AGENT);
    if (_allowCompression) {
      method.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate"));
    }

    try {
      // Execute the method.
      // System.out.println( "EXECUTE:"+method.getURI() );

      int statusCode = _httpClient.executeMethod(method);
      if (statusCode != HttpStatus.SC_OK) {
        StringBuilder msg = new StringBuilder();
        msg.append(method.getStatusLine().getReasonPhrase());
        msg.append("\n\n");
        msg.append(method.getStatusText());
        msg.append("\n\n");
        msg.append("request: " + method.getURI());
        throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8"));
      }

      // Read the contents
      String charset = "UTF-8";
      if (method instanceof HttpMethodBase) {
        charset = ((HttpMethodBase) method).getResponseCharSet();
      }
      InputStream respBody = method.getResponseBodyAsStream();
      // Jakarta Commons HTTPClient doesn't handle any
      // compression natively.  Handle gzip or deflate
      // here if applicable.
      if (_allowCompression) {
        Header contentEncodingHeader = method.getResponseHeader("Content-Encoding");
        if (contentEncodingHeader != null) {
          String contentEncoding = contentEncodingHeader.getValue();
          if (contentEncoding.contains("gzip")) {
            // log.debug( "wrapping response in GZIPInputStream" );
            respBody = new GZIPInputStream(respBody);
          } else if (contentEncoding.contains("deflate")) {
            // log.debug( "wrapping response in InflaterInputStream" );
            respBody = new InflaterInputStream(respBody);
          }
        } else {
          Header contentTypeHeader = method.getResponseHeader("Content-Type");
          if (contentTypeHeader != null) {
            String contentType = contentTypeHeader.getValue();
            if (contentType != null) {
              if (contentType.startsWith("application/x-gzip-compressed")) {
                // log.debug( "wrapping response in GZIPInputStream" );
                respBody = new GZIPInputStream(respBody);
              } else if (contentType.startsWith("application/x-deflate")) {
                // log.debug( "wrapping response in InflaterInputStream" );
                respBody = new InflaterInputStream(respBody);
              }
            }
          }
        }
      }
      return processor.processResponse(respBody, charset);
    } catch (HttpException e) {
      throw new SolrServerException(e);
    } catch (IOException e) {
      throw new SolrServerException(e);
    } finally {
      method.releaseConnection();
      if (is != null) {
        is.close();
      }
    }
  }
Exemplo n.º 10
0
  @Test
  public void testAsyncRangeRequests()
      throws IOException, URISyntaxException, InterruptedException {
    final URL testResourceUrl = new URL(VAULT_BASE_URI.toURL(), "asyncRangeRequestTestFile.txt");

    final MultiThreadedHttpConnectionManager cm = new MultiThreadedHttpConnectionManager();
    cm.getParams().setDefaultMaxConnectionsPerHost(50);
    final HttpClient client = new HttpClient(cm);

    // prepare 8MiB test data:
    final byte[] plaintextData = new byte[2097152 * Integer.BYTES];
    final ByteBuffer bbIn = ByteBuffer.wrap(plaintextData);
    for (int i = 0; i < 2097152; i++) {
      bbIn.putInt(i);
    }

    // put request:
    final EntityEnclosingMethod putMethod = new PutMethod(testResourceUrl.toString());
    putMethod.setRequestEntity(new ByteArrayRequestEntity(plaintextData));
    final int putResponse = client.executeMethod(putMethod);
    putMethod.releaseConnection();
    Assert.assertEquals(201, putResponse);

    // multiple async range requests:
    final List<ForkJoinTask<?>> tasks = new ArrayList<>();
    final Random generator = new Random(System.currentTimeMillis());

    final AtomicBoolean success = new AtomicBoolean(true);

    // 10 full interrupted requests:
    for (int i = 0; i < 10; i++) {
      final ForkJoinTask<?> task =
          ForkJoinTask.adapt(
              () -> {
                try {
                  final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
                  final int statusCode = client.executeMethod(getMethod);
                  if (statusCode != 200) {
                    LOG.error("Invalid status code for interrupted full request");
                    success.set(false);
                  }
                  getMethod.getResponseBodyAsStream().read();
                  getMethod.getResponseBodyAsStream().close();
                  getMethod.releaseConnection();
                } catch (IOException e) {
                  throw new RuntimeException(e);
                }
              });
      tasks.add(task);
    }

    // 50 crappy interrupted range requests:
    for (int i = 0; i < 50; i++) {
      final int lower = generator.nextInt(plaintextData.length);
      final ForkJoinTask<?> task =
          ForkJoinTask.adapt(
              () -> {
                try {
                  final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
                  getMethod.addRequestHeader("Range", "bytes=" + lower + "-");
                  final int statusCode = client.executeMethod(getMethod);
                  if (statusCode != 206) {
                    LOG.error("Invalid status code for interrupted range request");
                    success.set(false);
                  }
                  getMethod.getResponseBodyAsStream().read();
                  getMethod.getResponseBodyAsStream().close();
                  getMethod.releaseConnection();
                } catch (IOException e) {
                  throw new RuntimeException(e);
                }
              });
      tasks.add(task);
    }

    // 50 normal open range requests:
    for (int i = 0; i < 50; i++) {
      final int lower = generator.nextInt(plaintextData.length - 512);
      final int upper = plaintextData.length - 1;
      final ForkJoinTask<?> task =
          ForkJoinTask.adapt(
              () -> {
                try {
                  final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
                  getMethod.addRequestHeader("Range", "bytes=" + lower + "-");
                  final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1);
                  final int statusCode = client.executeMethod(getMethod);
                  final byte[] responseBody = new byte[upper - lower + 10];
                  final int bytesRead =
                      IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody);
                  getMethod.releaseConnection();
                  if (statusCode != 206) {
                    LOG.error("Invalid status code for open range request");
                    success.set(false);
                  } else if (upper - lower + 1 != bytesRead) {
                    LOG.error("Invalid response length for open range request");
                    success.set(false);
                  } else if (!Arrays.equals(
                      expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) {
                    LOG.error("Invalid response body for open range request");
                    success.set(false);
                  }
                } catch (IOException e) {
                  throw new RuntimeException(e);
                }
              });
      tasks.add(task);
    }

    // 200 normal closed range requests:
    for (int i = 0; i < 200; i++) {
      final int pos1 = generator.nextInt(plaintextData.length - 512);
      final int pos2 = pos1 + 512;
      final ForkJoinTask<?> task =
          ForkJoinTask.adapt(
              () -> {
                try {
                  final int lower = Math.min(pos1, pos2);
                  final int upper = Math.max(pos1, pos2);
                  final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
                  getMethod.addRequestHeader("Range", "bytes=" + lower + "-" + upper);
                  final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1);
                  final int statusCode = client.executeMethod(getMethod);
                  final byte[] responseBody = new byte[upper - lower + 1];
                  final int bytesRead =
                      IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody);
                  getMethod.releaseConnection();
                  if (statusCode != 206) {
                    LOG.error("Invalid status code for closed range request");
                    success.set(false);
                  } else if (upper - lower + 1 != bytesRead) {
                    LOG.error("Invalid response length for closed range request");
                    success.set(false);
                  } else if (!Arrays.equals(
                      expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) {
                    LOG.error("Invalid response body for closed range request");
                    success.set(false);
                  }
                } catch (IOException e) {
                  throw new RuntimeException(e);
                }
              });
      tasks.add(task);
    }

    Collections.shuffle(tasks, generator);

    final ForkJoinPool pool = new ForkJoinPool(4);
    for (ForkJoinTask<?> task : tasks) {
      pool.execute(task);
    }
    for (ForkJoinTask<?> task : tasks) {
      task.join();
    }
    pool.shutdown();
    cm.shutdown();

    Assert.assertTrue(success.get());
  }
Exemplo n.º 11
0
  public void authorize(String username, String password) {

    authentication = Base64Converter.encode(username + ":" + password);
    method.addRequestHeader("Authorization", "basic " + authentication);
  }
Exemplo n.º 12
0
  public void process(Exchange exchange) throws Exception {
    // if we bridge endpoint then we need to skip matching headers with the HTTP_QUERY to avoid
    // sending
    // duplicated headers to the receiver, so use this skipRequestHeaders as the list of headers to
    // skip
    Map<String, Object> skipRequestHeaders = null;

    if (getEndpoint().isBridgeEndpoint()) {
      exchange.setProperty(Exchange.SKIP_GZIP_ENCODING, Boolean.TRUE);
      String queryString = exchange.getIn().getHeader(Exchange.HTTP_QUERY, String.class);
      if (queryString != null) {
        skipRequestHeaders = URISupport.parseQuery(queryString);
      }
      // Need to remove the Host key as it should be not used
      exchange.getIn().getHeaders().remove("host");
    }
    HttpMethod method = createMethod(exchange);
    Message in = exchange.getIn();
    String httpProtocolVersion = in.getHeader(Exchange.HTTP_PROTOCOL_VERSION, String.class);
    if (httpProtocolVersion != null) {
      // set the HTTP protocol version
      HttpMethodParams params = method.getParams();
      params.setVersion(HttpVersion.parse(httpProtocolVersion));
    }

    HeaderFilterStrategy strategy = getEndpoint().getHeaderFilterStrategy();

    // propagate headers as HTTP headers
    for (Map.Entry<String, Object> entry : in.getHeaders().entrySet()) {
      String key = entry.getKey();
      Object headerValue = in.getHeader(key);

      if (headerValue != null) {
        // use an iterator as there can be multiple values. (must not use a delimiter, and allow
        // empty values)
        final Iterator<?> it = ObjectHelper.createIterator(headerValue, null, true);

        // the value to add as request header
        final List<String> values = new ArrayList<String>();

        // if its a multi value then check each value if we can add it and for multi values they
        // should be combined into a single value
        while (it.hasNext()) {
          String value =
              exchange.getContext().getTypeConverter().convertTo(String.class, it.next());

          // we should not add headers for the parameters in the uri if we bridge the endpoint
          // as then we would duplicate headers on both the endpoint uri, and in HTTP headers as
          // well
          if (skipRequestHeaders != null && skipRequestHeaders.containsKey(key)) {
            continue;
          }
          if (value != null
              && strategy != null
              && !strategy.applyFilterToCamelHeaders(key, value, exchange)) {
            values.add(value);
          }
        }

        // add the value(s) as a http request header
        if (values.size() > 0) {
          // use the default toString of a ArrayList to create in the form [xxx, yyy]
          // if multi valued, for a single value, then just output the value as is
          String s = values.size() > 1 ? values.toString() : values.get(0);
          method.addRequestHeader(key, s);
        }
      }
    }

    // lets store the result in the output message.
    try {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Executing http {} method: {}", method.getName(), method.getURI().toString());
      }
      int responseCode = executeMethod(method);
      LOG.debug("Http responseCode: {}", responseCode);

      if (!throwException) {
        // if we do not use failed exception then populate response for all response codes
        populateResponse(exchange, method, in, strategy, responseCode);
      } else {
        if (responseCode >= 100 && responseCode < 300) {
          // only populate response for OK response
          populateResponse(exchange, method, in, strategy, responseCode);
        } else {
          // operation failed so populate exception to throw
          throw populateHttpOperationFailedException(exchange, method, responseCode);
        }
      }
    } finally {
      method.releaseConnection();
    }
  }
Exemplo n.º 13
0
  protected byte[] URLtoByteArray(
      String location,
      Http.Method method,
      Map<String, String> headers,
      Cookie[] cookies,
      Http.Auth auth,
      Http.Body body,
      List<Http.FilePart> fileParts,
      Map<String, String> parts,
      Http.Response response,
      boolean followRedirects)
      throws IOException {

    byte[] bytes = null;

    HttpMethod httpMethod = null;
    HttpState httpState = null;

    try {
      _cookies.set(null);

      if (location == null) {
        return null;
      } else if (!location.startsWith(Http.HTTP_WITH_SLASH)
          && !location.startsWith(Http.HTTPS_WITH_SLASH)) {

        location = Http.HTTP_WITH_SLASH + location;
      }

      HostConfiguration hostConfiguration = getHostConfiguration(location);

      HttpClient httpClient = getClient(hostConfiguration);

      if (method.equals(Http.Method.POST) || method.equals(Http.Method.PUT)) {

        if (method.equals(Http.Method.POST)) {
          httpMethod = new PostMethod(location);
        } else {
          httpMethod = new PutMethod(location);
        }

        if (body != null) {
          RequestEntity requestEntity =
              new StringRequestEntity(body.getContent(), body.getContentType(), body.getCharset());

          EntityEnclosingMethod entityEnclosingMethod = (EntityEnclosingMethod) httpMethod;

          entityEnclosingMethod.setRequestEntity(requestEntity);
        } else if (method.equals(Http.Method.POST)) {
          PostMethod postMethod = (PostMethod) httpMethod;

          processPostMethod(postMethod, fileParts, parts);
        }
      } else if (method.equals(Http.Method.DELETE)) {
        httpMethod = new DeleteMethod(location);
      } else if (method.equals(Http.Method.HEAD)) {
        httpMethod = new HeadMethod(location);
      } else {
        httpMethod = new GetMethod(location);
      }

      if (headers != null) {
        for (Map.Entry<String, String> header : headers.entrySet()) {
          httpMethod.addRequestHeader(header.getKey(), header.getValue());
        }
      }

      if ((method.equals(Http.Method.POST) || method.equals(Http.Method.PUT))
          && ((body != null)
              || ((fileParts != null) && !fileParts.isEmpty())
                  | ((parts != null) && !parts.isEmpty()))) {
      } else if (!hasRequestHeader(httpMethod, HttpHeaders.CONTENT_TYPE)) {
        httpMethod.addRequestHeader(
            HttpHeaders.CONTENT_TYPE, ContentTypes.APPLICATION_X_WWW_FORM_URLENCODED);
      }

      if (!hasRequestHeader(httpMethod, HttpHeaders.USER_AGENT)) {
        httpMethod.addRequestHeader(HttpHeaders.USER_AGENT, _DEFAULT_USER_AGENT);
      }

      httpState = new HttpState();

      if ((cookies != null) && (cookies.length > 0)) {
        org.apache.commons.httpclient.Cookie[] commonsCookies = toCommonsCookies(cookies);

        httpState.addCookies(commonsCookies);

        HttpMethodParams httpMethodParams = httpMethod.getParams();

        httpMethodParams.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
      }

      if (auth != null) {
        httpMethod.setDoAuthentication(true);

        httpState.setCredentials(
            new AuthScope(auth.getHost(), auth.getPort(), auth.getRealm()),
            new UsernamePasswordCredentials(auth.getUsername(), auth.getPassword()));
      }

      proxifyState(httpState, hostConfiguration);

      httpClient.executeMethod(hostConfiguration, httpMethod, httpState);

      Header locationHeader = httpMethod.getResponseHeader("location");

      if ((locationHeader != null) && !locationHeader.equals(location)) {
        String redirect = locationHeader.getValue();

        if (followRedirects) {
          return URLtoByteArray(
              redirect,
              Http.Method.GET,
              headers,
              cookies,
              auth,
              body,
              fileParts,
              parts,
              response,
              followRedirects);
        } else {
          response.setRedirect(redirect);
        }
      }

      InputStream inputStream = httpMethod.getResponseBodyAsStream();

      if (inputStream != null) {
        Header contentLength = httpMethod.getResponseHeader(HttpHeaders.CONTENT_LENGTH);

        if (contentLength != null) {
          response.setContentLength(GetterUtil.getInteger(contentLength.getValue()));
        }

        Header contentType = httpMethod.getResponseHeader(HttpHeaders.CONTENT_TYPE);

        if (contentType != null) {
          response.setContentType(contentType.getValue());
        }

        bytes = FileUtil.getBytes(inputStream);
      }

      for (Header header : httpMethod.getResponseHeaders()) {
        response.addHeader(header.getName(), header.getValue());
      }

      return bytes;
    } finally {
      try {
        if (httpState != null) {
          _cookies.set(toServletCookies(httpState.getCookies()));
        }
      } catch (Exception e) {
        _log.error(e, e);
      }

      try {
        if (httpMethod != null) {
          httpMethod.releaseConnection();
        }
      } catch (Exception e) {
        _log.error(e, e);
      }
    }
  }
  /**
   * 执行Http请求
   *
   * @param request
   * @return
   */
  public HttpResponse execute(HttpRequest request) {
    HttpClient httpclient = new HttpClient(connectionManager);

    // 设置连接超时
    int connectionTimeout = defaultConnectionTimeout;
    if (request.getConnectionTimeout() > 0) {
      connectionTimeout = request.getConnectionTimeout();
    }
    httpclient.getHttpConnectionManager().getParams().setConnectionTimeout(connectionTimeout);

    // 设置回应超时
    int soTimeout = defaultSoTimeout;
    if (request.getTimeout() > 0) {
      soTimeout = request.getTimeout();
    }
    httpclient.getHttpConnectionManager().getParams().setSoTimeout(soTimeout);

    // 设置等待ConnectionManager释放connection的时间
    httpclient.getParams().setConnectionManagerTimeout(defaultHttpConnectionManagerTimeout);

    String charset = request.getCharset();
    charset = charset == null ? DEFAULT_CHARSET : charset;
    HttpMethod method = null;

    if (request.getMethod().equals(HttpRequest.METHOD_GET)) {
      method = new GetMethod(request.getUrl());
      method.getParams().setCredentialCharset(charset);

      // parseNotifyConfig会保证使用GET方法时,request一定使用QueryString
      method.setQueryString(request.getQueryString());
    } else {
      method = new PostMethod(request.getUrl());
      ((PostMethod) method).addParameters(request.getParameters());
      method.addRequestHeader(
          "Content-Type", "application/x-www-form-urlencoded; text/html; charset=" + charset);
    }

    // 设置Http Header中的User-Agent属性
    method.addRequestHeader("User-Agent", "Mozilla/4.0");
    HttpResponse response = new HttpResponse();

    try {
      httpclient.executeMethod(method);
      if (request.getResultType().equals(HttpResultType.STRING)) {
        response.setStringResult(method.getResponseBodyAsString());
      } else if (request.getResultType().equals(HttpResultType.BYTES)) {
        response.setByteResult(method.getResponseBody());
      }
      response.setResponseHeaders(method.getResponseHeaders());
    } catch (UnknownHostException ex) {

      return null;
    } catch (IOException ex) {

      return null;
    } catch (Exception ex) {

      return null;
    } finally {
      method.releaseConnection();
    }
    return response;
  }
Exemplo n.º 15
0
 protected String executeMethodWithErrorHandling(HttpMethod method) {
   method.addRequestHeader(HttpMethodParams.USER_AGENT, "trico.eve.dev");
   return read(method, exec(method));
 }