protected HttpDownloadResult doDownloadUrl(
final URL url,
final Integer timeout,
final String username,
final String password,
final HttpHeader httpHeader)
throws IOException {
logger.trace("downloadUrl started");
final Duration duration = durationUtil.getDuration();
InputStream inputStream = null;
ByteArrayOutputStream outputStream = null;
try {
final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
if (username != null && password != null || url.getUserInfo() != null) {
final String stringUserIdPassword =
url.getUserInfo() != null ? url.getUserInfo() : username + ":" + password;
final String base64UserIdPassword =
base64Util.encode(stringUserIdPassword.getBytes("ASCII"));
connection.setRequestProperty("Authorization", "Basic " + base64UserIdPassword);
}
// connection.setConnectTimeout(timeout);
// connection.setReadTimeout(timeout);
connection.setRequestProperty("User-Agent", USERAGENT);
if (httpHeader != null) {
for (final String httpHeaderField : httpHeader.getKeys()) {
connection.setRequestProperty(httpHeaderField, httpHeader.getValue(httpHeaderField));
}
}
connection.connect();
final int responseCode;
try {
responseCode = connection.getResponseCode();
} catch (RuntimeException e) {
final HttpDownloadResult httpDownloadResult =
new HttpDownloadResultImpl(url, duration.getTime(), null, null, null, null, -1);
logger.trace("downloadUrl " + url + " failed");
return httpDownloadResult;
}
try {
final String contentType = connection.getContentType();
final Encoding contentEncoding = extractEncoding(connection, contentType);
final Map<String, List<String>> headers = connection.getHeaderFields();
try {
inputStream = connection.getInputStream();
outputStream = new ByteArrayOutputStream();
streamUtil.copy(inputStream, outputStream);
final HttpDownloadResult httpDownloadResult =
new HttpDownloadResultImpl(
url,
duration.getTime(),
outputStream.toByteArray(),
contentType,
contentEncoding,
headers,
responseCode);
logger.trace("downloadUrl " + url + " finished");
return httpDownloadResult;
} catch (FileNotFoundException e) {
final HttpDownloadResult httpDownloadResult =
new HttpDownloadResultImpl(
url,
duration.getTime(),
null,
contentType,
contentEncoding,
headers,
responseCode);
logger.trace("downloadUrl " + url + " failed");
return httpDownloadResult;
}
} catch (IOException e) {
final HttpDownloadResult httpDownloadResult =
new HttpDownloadResultImpl(
url, duration.getTime(), null, null, null, null, responseCode);
logger.trace("downloadUrl " + url + " failed");
return httpDownloadResult;
}
} finally {
if (inputStream != null) inputStream.close();
if (outputStream != null) outputStream.close();
}
}
@Override
public HttpDownloadResult postUrlSecure(
final URL url,
final Integer timeout,
final Map<String, String> parameter,
final HttpHeader httpHeader)
throws HttpDownloaderException {
try {
final Duration duration = durationUtil.getDuration();
// Construct data
final StringWriter data = new StringWriter();
boolean first = true;
for (final Entry<String, String> e : parameter.entrySet()) {
if (!first) {
data.append("&");
} else {
first = false;
}
data.append(URLEncoder.encode(e.getKey(), "UTF-8"));
data.append("=");
data.append(URLEncoder.encode(e.getValue(), "UTF-8"));
}
// Send data
final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(timeout);
connection.setReadTimeout(timeout);
connection.setRequestProperty("User-Agent", USERAGENT);
if (httpHeader != null) {
for (final String httpHeaderField : httpHeader.getKeys()) {
connection.setRequestProperty(httpHeaderField, httpHeader.getValue(httpHeaderField));
}
}
connection.setDoOutput(true);
connection.connect();
final OutputStreamWriter wr = new OutputStreamWriter(connection.getOutputStream());
wr.write(data.toString());
wr.flush();
final InputStream inputStream = connection.getInputStream();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
streamUtil.copy(inputStream, outputStream);
final byte[] content = outputStream.toByteArray();
final String contentType = connection.getContentType();
final Map<String, List<String>> headers = connection.getHeaderFields();
final Encoding contentEncoding = extractEncoding(connection, contentType);
final int responseCode = connection.getResponseCode();
final HttpDownloadResult httpDownloadResult =
new HttpDownloadResultImpl(
url,
duration.getTime(),
content,
contentType,
contentEncoding,
headers,
responseCode);
logger.trace("downloadUrl finished");
return httpDownloadResult;
} catch (final IOException e) {
logger.error(e.getClass().getSimpleName(), e);
throw new HttpDownloaderException(e.getClass().getSimpleName(), e);
}
}