Example #1
0
  protected HttpDownloadResult doDownloadUrl(
      final URL url,
      final Integer timeout,
      final String username,
      final String password,
      final HttpHeader httpHeader)
      throws IOException {
    logger.trace("downloadUrl started");
    final Duration duration = durationUtil.getDuration();
    InputStream inputStream = null;
    ByteArrayOutputStream outputStream = null;
    try {
      final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
      if (username != null && password != null || url.getUserInfo() != null) {
        final String stringUserIdPassword =
            url.getUserInfo() != null ? url.getUserInfo() : username + ":" + password;
        final String base64UserIdPassword =
            base64Util.encode(stringUserIdPassword.getBytes("ASCII"));
        connection.setRequestProperty("Authorization", "Basic " + base64UserIdPassword);
      }
      // connection.setConnectTimeout(timeout);
      // connection.setReadTimeout(timeout);
      connection.setRequestProperty("User-Agent", USERAGENT);
      if (httpHeader != null) {
        for (final String httpHeaderField : httpHeader.getKeys()) {
          connection.setRequestProperty(httpHeaderField, httpHeader.getValue(httpHeaderField));
        }
      }
      connection.connect();

      final int responseCode;
      try {
        responseCode = connection.getResponseCode();
      } catch (RuntimeException e) {
        final HttpDownloadResult httpDownloadResult =
            new HttpDownloadResultImpl(url, duration.getTime(), null, null, null, null, -1);
        logger.trace("downloadUrl " + url + " failed");
        return httpDownloadResult;
      }
      try {
        final String contentType = connection.getContentType();
        final Encoding contentEncoding = extractEncoding(connection, contentType);
        final Map<String, List<String>> headers = connection.getHeaderFields();
        try {
          inputStream = connection.getInputStream();
          outputStream = new ByteArrayOutputStream();
          streamUtil.copy(inputStream, outputStream);
          final HttpDownloadResult httpDownloadResult =
              new HttpDownloadResultImpl(
                  url,
                  duration.getTime(),
                  outputStream.toByteArray(),
                  contentType,
                  contentEncoding,
                  headers,
                  responseCode);
          logger.trace("downloadUrl " + url + " finished");
          return httpDownloadResult;
        } catch (FileNotFoundException e) {
          final HttpDownloadResult httpDownloadResult =
              new HttpDownloadResultImpl(
                  url,
                  duration.getTime(),
                  null,
                  contentType,
                  contentEncoding,
                  headers,
                  responseCode);
          logger.trace("downloadUrl " + url + " failed");
          return httpDownloadResult;
        }
      } catch (IOException e) {
        final HttpDownloadResult httpDownloadResult =
            new HttpDownloadResultImpl(
                url, duration.getTime(), null, null, null, null, responseCode);
        logger.trace("downloadUrl " + url + " failed");
        return httpDownloadResult;
      }
    } finally {
      if (inputStream != null) inputStream.close();
      if (outputStream != null) outputStream.close();
    }
  }
Example #2
0
  @Override
  public HttpDownloadResult postUrlSecure(
      final URL url,
      final Integer timeout,
      final Map<String, String> parameter,
      final HttpHeader httpHeader)
      throws HttpDownloaderException {
    try {
      final Duration duration = durationUtil.getDuration();

      // Construct data
      final StringWriter data = new StringWriter();
      boolean first = true;
      for (final Entry<String, String> e : parameter.entrySet()) {
        if (!first) {
          data.append("&");
        } else {
          first = false;
        }
        data.append(URLEncoder.encode(e.getKey(), "UTF-8"));
        data.append("=");
        data.append(URLEncoder.encode(e.getValue(), "UTF-8"));
      }

      // Send data
      final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
      connection.setConnectTimeout(timeout);
      connection.setReadTimeout(timeout);
      connection.setRequestProperty("User-Agent", USERAGENT);
      if (httpHeader != null) {
        for (final String httpHeaderField : httpHeader.getKeys()) {
          connection.setRequestProperty(httpHeaderField, httpHeader.getValue(httpHeaderField));
        }
      }
      connection.setDoOutput(true);
      connection.connect();
      final OutputStreamWriter wr = new OutputStreamWriter(connection.getOutputStream());
      wr.write(data.toString());
      wr.flush();
      final InputStream inputStream = connection.getInputStream();
      final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
      streamUtil.copy(inputStream, outputStream);
      final byte[] content = outputStream.toByteArray();
      final String contentType = connection.getContentType();
      final Map<String, List<String>> headers = connection.getHeaderFields();
      final Encoding contentEncoding = extractEncoding(connection, contentType);
      final int responseCode = connection.getResponseCode();
      final HttpDownloadResult httpDownloadResult =
          new HttpDownloadResultImpl(
              url,
              duration.getTime(),
              content,
              contentType,
              contentEncoding,
              headers,
              responseCode);
      logger.trace("downloadUrl finished");
      return httpDownloadResult;
    } catch (final IOException e) {
      logger.error(e.getClass().getSimpleName(), e);
      throw new HttpDownloaderException(e.getClass().getSimpleName(), e);
    }
  }