@Override
public void run() throws Exception {
super.run();
logger.info("Starting download in TASK " + fileURL);
checkUrl();
login();
HttpMethod method = getGetMethod(fileURL);
if (makeRedirectedRequest(method)) {
checkProblems();
checkNameAndSize();
if (isFolder()) {
parseFolder();
} else {
method =
getMethodBuilder()
.setReferer(fileURL)
.setActionFromTextBetween("<a id=\"btnLink\" href=\"", "\"")
.toGetMethod();
if (!method.getURI().toString().contains("/download/")) {
if (makeRedirectedRequest(method)) {
checkProblems();
method =
getMethodBuilder()
.setReferer(method.getURI().toString())
.setActionFromAHrefWhereATagContains("Download file")
.toGetMethod();
downloadTask.sleep(
PlugUtils.getNumberBetween(
getContentAsString(), "id=\"secondsLeft\" value=\"", "\"")
+ 1);
} else {
checkProblems();
throw new ServiceConnectionProblemException();
}
}
if (!tryDownloadAndSaveFile(method)) {
checkProblems();
throw new ServiceConnectionProblemException("Error starting download");
}
}
} else {
checkProblems();
throw new ServiceConnectionProblemException();
}
}
private String post(final HttpMethod method) throws IOException, ForbiddenException {
method.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
method.setRequestHeader("Accept-Encoding", "gzip");
InputStream is = null;
try {
this.m_clientManager.executeMethod(method);
final int statusCode = method.getStatusCode();
final StatusLine statusLine = method.getStatusLine();
final Header encoding = method.getResponseHeader("Content-Encoding");
if (encoding != null && encoding.getValue().equals("gzip")) {
m_log.debug("Unzipping body...");
is = new GZIPInputStream(method.getResponseBodyAsStream());
} else {
is = method.getResponseBodyAsStream();
}
final String body = IOUtils.toString(is);
if (StringUtils.isBlank(body)) {
// Could easily be a post request, which would not have a body.
m_log.debug("No response body. Post request?");
}
if (statusCode == HttpStatus.SC_FORBIDDEN) {
final String msg = "NO 200 OK: " + method.getURI() + "\n" + statusLine + "\n" + body;
m_log.warn(msg);
throw new ForbiddenException(msg);
}
if (statusCode != HttpStatus.SC_OK) {
final String msg = "NO 200 OK: " + method.getURI() + "\n" + statusLine + "\n" + body;
m_log.warn(msg);
throw new IOException(msg);
} else {
m_log.debug("Got 200 response...");
}
return body;
} finally {
IOUtils.closeQuietly(is);
method.releaseConnection();
}
}
private void logHttpResponse(HttpMethod method, String strResp) {
try {
if (logger.isDebugEnabled()) {
logger.debug("/n/n============= HTTP Response Start =============");
logger.debug("HTTP Response URL ==>/n" + method.getURI().toString());
logger.debug("HTTP Response Headers ==>/n" + getHttpResponseHeader(method));
logger.debug("HTTP Response Cookies ==>/n" + getHttpCookie());
logger.debug("HTTP Response Body ==>/n" + strResp);
logger.debug("============= HTTP Response End =============/n/n");
}
} catch (URIException e) {
logger.error("URIException", e);
}
}
// ------------- log methods ----------------//
private void logHttpGetRequest(HttpMethod method) {
try {
if (logger.isDebugEnabled()) {
logger.debug("/n/n============= HTTP Request Start =============");
logger.debug("HTTP Get Request URL ==>/n" + method.getURI().toString());
logger.debug("HTTP Get Request Headers ==>/n" + getHttpRequestHeader(method));
logger.debug("HTTP Get Request Cookies ==>/n" + getHttpCookie());
logger.debug("HTTP Get Request QueryString ==>/n" + method.getQueryString());
logger.debug("============= HTTP Request End =============/n/n");
}
} catch (URIException e) {
logger.error(e);
}
}
/**
* parameter와 queryString 를 가져온다.
*
* @param method
* @return
*/
private String getHttpInfoDumy(HttpMethod method) {
NameValuePair[] params = null;
String methodType = "GET";
String reqBody = null;
if (method instanceof PostMethod) {
params = ((PostMethod) method).getParameters();
methodType = "POST";
StringRequestEntity sre = (StringRequestEntity) ((PostMethod) method).getRequestEntity();
reqBody = sre.getContent();
}
StringBuffer sb = new StringBuffer();
try {
sb.append("#### getHttpInfoDumy ####");
sb.append("\n## " + methodType + " [" + method.getURI() + "], hscd[" + this.hashCode() + "]");
} catch (URIException e) {
sb.append("\n## getParamsQueryStr- URIException " + e.getMessage() + "]");
return sb.toString();
}
if (method.getQueryString() != null && method.getQueryString().length() > 0)
sb.append("\n" + "## queryString[" + method.getQueryString() + "]");
if (params != null) {
for (int i = 0; i < params.length; i++) {
NameValuePair param = params[i];
sb.append(
"\n"
+ "## POST body param["
+ i
+ "], name["
+ param.getName()
+ "], value["
+ param.getValue()
+ "]");
}
}
if (reqBody != null) {
sb.append("\n" + "## POST body String [" + reqBody + "]");
}
sb.append("\n##########");
return sb.toString();
}
@NotNull
public String executeMethod(@NotNull HttpMethod method) throws Exception {
LOG.debug("URI: " + method.getURI());
int statusCode;
String entityContent;
try {
statusCode = getHttpClient().executeMethod(method);
LOG.debug("Status code: " + statusCode);
// may be null if 204 No Content received
final InputStream stream = method.getResponseBodyAsStream();
entityContent = stream == null ? "" : StreamUtil.readText(stream, CharsetToolkit.UTF8);
LOG.debug(entityContent);
} finally {
method.releaseConnection();
}
// besides SC_OK, can also be SC_NO_CONTENT in issue transition requests
// see: JiraRestApi#setTaskStatus
// if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NO_CONTENT) {
if (statusCode >= 200 && statusCode < 300) {
return entityContent;
} else if (method.getResponseHeader("Content-Type") != null) {
Header header = method.getResponseHeader("Content-Type");
if (header.getValue().startsWith("application/json")) {
JsonObject object = GSON.fromJson(entityContent, JsonObject.class);
if (object.has("errorMessages")) {
String reason = StringUtil.join(object.getAsJsonArray("errorMessages"), " ");
// something meaningful to user, e.g. invalid field name in JQL query
LOG.warn(reason);
throw new Exception("Request failed. Reason: " + reason);
}
}
}
if (method.getResponseHeader("X-Authentication-Denied-Reason") != null) {
Header header = method.getResponseHeader("X-Authentication-Denied-Reason");
// only in JIRA >= 5.x.x
if (header.getValue().startsWith("CAPTCHA_CHALLENGE")) {
throw new Exception("Login failed. Enter captcha in web-interface.");
}
}
if (statusCode == HttpStatus.SC_UNAUTHORIZED) {
throw new Exception(LOGIN_FAILED_CHECK_YOUR_PERMISSIONS);
}
String statusText = HttpStatus.getStatusText(method.getStatusCode());
throw new Exception(
String.format("Request failed with HTTP error: %d %s", statusCode, statusText));
}
@Test
public void shouldReplaceMessageVariableWithValue() throws URIException {
Request request = new Request();
request.setUrlPath("http://smshost.com/sms/send");
Map<String, String> queryParameters =
new HashMap<String, String>() {
{
put("message", "$message");
}
};
request.setQueryParameters(queryParameters);
SmsHttpTemplate smsHttpTemplate = createSmsHttpTemplate(request);
HttpMethod httpMethod = smsHttpTemplate.generateRequestFor(Arrays.asList("123"), "foobar");
assertEquals("http://smshost.com/sms/send?message=foobar", httpMethod.getURI().getURI());
}
public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, String payload) {
// http is not used.
// payload is not used.
boolean result = false;
Map formItems = null;
try {
formItems = getFormItems(curi);
} catch (AttributeNotFoundException e1) {
logger.severe("Failed get of form items for " + curi);
}
if (formItems == null || formItems.size() <= 0) {
try {
logger.severe("No form items for " + method.getURI());
} catch (URIException e) {
logger.severe("No form items and exception getting uri: " + e.getMessage());
}
return result;
}
NameValuePair[] data = new NameValuePair[formItems.size()];
int index = 0;
String key = null;
for (Iterator i = formItems.keySet().iterator(); i.hasNext(); ) {
key = (String) i.next();
data[index++] = new NameValuePair(key, (String) formItems.get(key));
}
if (method instanceof PostMethod) {
((PostMethod) method).setRequestBody(data);
result = true;
} else if (method instanceof GetMethod) {
// Append these values to the query string.
// Get current query string, then add data, then get it again
// only this time its our data only... then append.
HttpMethodBase hmb = (HttpMethodBase) method;
String currentQuery = hmb.getQueryString();
hmb.setQueryString(data);
String newQuery = hmb.getQueryString();
hmb.setQueryString(((currentQuery != null) ? currentQuery : "") + "&" + newQuery);
result = true;
} else {
logger.severe("Unknown method type: " + method);
}
return result;
}
protected Exception populateHttpOperationFailedException(
Exchange exchange, HttpMethod method, int responseCode)
throws IOException, ClassNotFoundException {
Exception answer;
String uri = method.getURI().toString();
String statusText =
method.getStatusLine() != null ? method.getStatusLine().getReasonPhrase() : null;
Map<String, String> headers = extractResponseHeaders(method.getResponseHeaders());
Object responseBody = extractResponseBody(method, exchange);
if (transferException && responseBody != null && responseBody instanceof Exception) {
// if the response was a serialized exception then use that
return (Exception) responseBody;
}
// make a defensive copy of the response body in the exception so its detached from the cache
String copy = null;
if (responseBody != null) {
copy =
exchange.getContext().getTypeConverter().convertTo(String.class, exchange, responseBody);
}
if (responseCode >= 300 && responseCode < 400) {
String redirectLocation;
Header locationHeader = method.getResponseHeader("location");
if (locationHeader != null) {
redirectLocation = locationHeader.getValue();
answer =
new HttpOperationFailedException(
uri, responseCode, statusText, redirectLocation, headers, copy);
} else {
// no redirect location
answer =
new HttpOperationFailedException(uri, responseCode, statusText, null, headers, copy);
}
} else {
// internal server error (error code 500)
answer = new HttpOperationFailedException(uri, responseCode, statusText, null, headers, copy);
}
return answer;
}
private String getResponseHost(HttpMethod method) {
try {
String hostRegexp = "http://([^/]+)/.*";
Pattern p = Pattern.compile(hostRegexp);
String url = method.getURI().toString();
Matcher matcher = p.matcher(url);
if (!matcher.find()) {
String msg =
"ExceptionUtil.PROTOCOL_MSG" + "[Type=ResponseHost,Regexp=" + p.pattern() + "]";
logger.error(msg);
}
if (logger.isDebugEnabled()) {
logger.debug("Host=" + matcher.group(1));
}
return matcher.group(1);
} catch (Exception e) {
logger.error(e);
}
return "";
}
@Test
public void shouldReplaceReciepientsVariableWithValue() throws URIException {
Request request = new Request();
request.setUrlPath("http://smshost.com/sms/send");
request.setRecipientsSeparator(",");
Map<String, String> queryParameters =
new HashMap<String, String>() {
{
put("recipients", "$recipients");
}
};
request.setQueryParameters(queryParameters);
SmsHttpTemplate smsHttpTemplate = createSmsHttpTemplate(request);
HttpMethod httpMethod =
smsHttpTemplate.generateRequestFor(Arrays.asList("123", "456", "789"), "some message");
assertEquals(
"http://smshost.com/sms/send?recipients=123,456,789", httpMethod.getURI().getURI());
}
private String send(HttpMethod method, boolean sslExceptionIgnore) throws HttpNetAgentException {
String responseBody = null;
HttpClient hc = null;
try {
this.hashCode();
String reqLog =
">> " + (method instanceof PostMethod ? "POST" : "GET") + " [" + method.getURI() + "]";
if (method instanceof PostMethod) {
NameValuePair[] params = ((PostMethod) method).getParameters();
for (int i = 0; i < params.length; i++) {
NameValuePair param = params[i];
reqLog += "\n" + (param.getName() + "=" + param.getValue());
}
}
reqLog +=
", hscd["
+ this.hashCode()
+ "], sslExcIgnore["
+ sslExceptionIgnore
+ "], hs["
+ this.hashCode()
+ "]";
logger.info(reqLog);
logger.info(getHttpInfoDumy(method));
HttpConnectionManager httpConnMgr = new SimpleHttpConnectionManager();
HttpConnectionManagerParams httpConnMgrParams = new HttpConnectionManagerParams();
// Connection Timeout 설정
httpConnMgrParams.setConnectionTimeout(netTimeoutConn);
// Socket Timeout 설정
httpConnMgrParams.setSoTimeout(netTimeoutSock);
httpConnMgr.setParams(httpConnMgrParams);
hc = new HttpClient(httpConnMgr);
/*
if (sslExceptionIgnore){
try {
Protocol.registerProtocol("https", new Protocol("https", new EasySSLProtocolSocketFactory(), 443));
} catch (GeneralSecurityException e) {
e.printStackTrace();
}
}
*/
int status = hc.executeMethod(method);
if (200 == status) {
responseBody = method.getResponseBodyAsString();
logger.info("<< responseBody [" + responseBody + "], hs[" + this.hashCode() + "]");
if (responseBody == null) responseBody = "";
} else {
logger.error(
"<< http status Not Success ["
+ status
+ "], hs["
+ this.hashCode()
+ "],"
+ getHttpInfoDumy(method));
throw new HttpNetAgentException(
HttpNetAgentException.HTTP_HAEDER_NOT_SUCCESS, "Http Status[" + status + "]");
}
} catch (HttpException e) {
logger.error(
"<< HttpException msg ["
+ e.getMessage()
+ "], hs["
+ this.hashCode()
+ "],"
+ getHttpInfoDumy(method),
e);
throw new HttpNetAgentException(HttpNetAgentException.HTTP_NET_ERROR, e.getMessage());
} catch (IOException e) {
logger.error(
"<< IOException msg ["
+ e.getMessage()
+ "], hs["
+ this.hashCode()
+ "],"
+ getHttpInfoDumy(method),
e);
throw new HttpNetAgentException(HttpNetAgentException.HTTP_NET_ERROR, e.getMessage());
}
return responseBody;
}
public NamedList<Object> request(final SolrRequest request, ResponseParser processor)
throws SolrServerException, IOException {
HttpMethod method = null;
InputStream is = null;
SolrParams params = request.getParams();
Collection<ContentStream> streams = requestWriter.getContentStreams(request);
String path = requestWriter.getPath(request);
if (path == null || !path.startsWith("/")) {
path = "/select";
}
ResponseParser parser = request.getResponseParser();
if (parser == null) {
parser = _parser;
}
// The parser 'wt=' and 'version=' params are used instead of the original params
ModifiableSolrParams wparams = new ModifiableSolrParams();
wparams.set(CommonParams.WT, parser.getWriterType());
wparams.set(CommonParams.VERSION, parser.getVersion());
if (params == null) {
params = wparams;
} else {
params = new DefaultSolrParams(wparams, params);
}
if (_invariantParams != null) {
params = new DefaultSolrParams(_invariantParams, params);
}
int tries = _maxRetries + 1;
try {
while (tries-- > 0) {
// Note: since we aren't do intermittent time keeping
// ourselves, the potential non-timeout latency could be as
// much as tries-times (plus scheduling effects) the given
// timeAllowed.
try {
if (SolrRequest.METHOD.GET == request.getMethod()) {
if (streams != null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
}
method = new GetMethod(_baseURL + path + ClientUtils.toQueryString(params, false));
} else if (SolrRequest.METHOD.POST == request.getMethod()) {
String url = _baseURL + path;
boolean isMultipart = (streams != null && streams.size() > 1);
if (streams == null || isMultipart) {
PostMethod post = new PostMethod(url);
post.getParams().setContentCharset("UTF-8");
if (!this.useMultiPartPost && !isMultipart) {
post.addRequestHeader(
"Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
}
List<Part> parts = new LinkedList<Part>();
Iterator<String> iter = params.getParameterNamesIterator();
while (iter.hasNext()) {
String p = iter.next();
String[] vals = params.getParams(p);
if (vals != null) {
for (String v : vals) {
if (this.useMultiPartPost || isMultipart) {
parts.add(new StringPart(p, v, "UTF-8"));
} else {
post.addParameter(p, v);
}
}
}
}
if (isMultipart) {
int i = 0;
for (ContentStream content : streams) {
final ContentStream c = content;
String charSet = null;
String transferEncoding = null;
parts.add(
new PartBase(c.getName(), c.getContentType(), charSet, transferEncoding) {
@Override
protected long lengthOfData() throws IOException {
return c.getSize();
}
@Override
protected void sendData(OutputStream out) throws IOException {
InputStream in = c.getStream();
try {
IOUtils.copy(in, out);
} finally {
in.close();
}
}
});
}
}
if (parts.size() > 0) {
post.setRequestEntity(
new MultipartRequestEntity(
parts.toArray(new Part[parts.size()]), post.getParams()));
}
method = post;
}
// It is has one stream, it is the post body, put the params in the URL
else {
String pstr = ClientUtils.toQueryString(params, false);
PostMethod post = new PostMethod(url + pstr);
// Single stream as body
// Using a loop just to get the first one
final ContentStream[] contentStream = new ContentStream[1];
for (ContentStream content : streams) {
contentStream[0] = content;
break;
}
if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
post.setRequestEntity(
new RequestEntity() {
public long getContentLength() {
return -1;
}
public String getContentType() {
return contentStream[0].getContentType();
}
public boolean isRepeatable() {
return false;
}
public void writeRequest(OutputStream outputStream) throws IOException {
((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream);
}
});
} else {
is = contentStream[0].getStream();
post.setRequestEntity(
new InputStreamRequestEntity(is, contentStream[0].getContentType()));
}
method = post;
}
} else {
throw new SolrServerException("Unsupported method: " + request.getMethod());
}
} catch (NoHttpResponseException r) {
// This is generally safe to retry on
method.releaseConnection();
method = null;
if (is != null) {
is.close();
}
// If out of tries then just rethrow (as normal error).
if ((tries < 1)) {
throw r;
}
// log.warn( "Caught: " + r + ". Retrying..." );
}
}
} catch (IOException ex) {
throw new SolrServerException("error reading streams", ex);
}
method.setFollowRedirects(_followRedirects);
method.addRequestHeader("User-Agent", AGENT);
if (_allowCompression) {
method.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate"));
}
try {
// Execute the method.
// System.out.println( "EXECUTE:"+method.getURI() );
int statusCode = _httpClient.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
StringBuilder msg = new StringBuilder();
msg.append(method.getStatusLine().getReasonPhrase());
msg.append("\n\n");
msg.append(method.getStatusText());
msg.append("\n\n");
msg.append("request: " + method.getURI());
throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8"));
}
// Read the contents
String charset = "UTF-8";
if (method instanceof HttpMethodBase) {
charset = ((HttpMethodBase) method).getResponseCharSet();
}
InputStream respBody = method.getResponseBodyAsStream();
// Jakarta Commons HTTPClient doesn't handle any
// compression natively. Handle gzip or deflate
// here if applicable.
if (_allowCompression) {
Header contentEncodingHeader = method.getResponseHeader("Content-Encoding");
if (contentEncodingHeader != null) {
String contentEncoding = contentEncodingHeader.getValue();
if (contentEncoding.contains("gzip")) {
// log.debug( "wrapping response in GZIPInputStream" );
respBody = new GZIPInputStream(respBody);
} else if (contentEncoding.contains("deflate")) {
// log.debug( "wrapping response in InflaterInputStream" );
respBody = new InflaterInputStream(respBody);
}
} else {
Header contentTypeHeader = method.getResponseHeader("Content-Type");
if (contentTypeHeader != null) {
String contentType = contentTypeHeader.getValue();
if (contentType != null) {
if (contentType.startsWith("application/x-gzip-compressed")) {
// log.debug( "wrapping response in GZIPInputStream" );
respBody = new GZIPInputStream(respBody);
} else if (contentType.startsWith("application/x-deflate")) {
// log.debug( "wrapping response in InflaterInputStream" );
respBody = new InflaterInputStream(respBody);
}
}
}
}
}
return processor.processResponse(respBody, charset);
} catch (HttpException e) {
throw new SolrServerException(e);
} catch (IOException e) {
throw new SolrServerException(e);
} finally {
method.releaseConnection();
if (is != null) {
is.close();
}
}
}
public void process(Exchange exchange) throws Exception {
// if we bridge endpoint then we need to skip matching headers with the HTTP_QUERY to avoid
// sending
// duplicated headers to the receiver, so use this skipRequestHeaders as the list of headers to
// skip
Map<String, Object> skipRequestHeaders = null;
if (getEndpoint().isBridgeEndpoint()) {
exchange.setProperty(Exchange.SKIP_GZIP_ENCODING, Boolean.TRUE);
String queryString = exchange.getIn().getHeader(Exchange.HTTP_QUERY, String.class);
if (queryString != null) {
skipRequestHeaders = URISupport.parseQuery(queryString);
}
// Need to remove the Host key as it should be not used
exchange.getIn().getHeaders().remove("host");
}
HttpMethod method = createMethod(exchange);
Message in = exchange.getIn();
String httpProtocolVersion = in.getHeader(Exchange.HTTP_PROTOCOL_VERSION, String.class);
if (httpProtocolVersion != null) {
// set the HTTP protocol version
HttpMethodParams params = method.getParams();
params.setVersion(HttpVersion.parse(httpProtocolVersion));
}
HeaderFilterStrategy strategy = getEndpoint().getHeaderFilterStrategy();
// propagate headers as HTTP headers
for (Map.Entry<String, Object> entry : in.getHeaders().entrySet()) {
String key = entry.getKey();
Object headerValue = in.getHeader(key);
if (headerValue != null) {
// use an iterator as there can be multiple values. (must not use a delimiter, and allow
// empty values)
final Iterator<?> it = ObjectHelper.createIterator(headerValue, null, true);
// the value to add as request header
final List<String> values = new ArrayList<String>();
// if its a multi value then check each value if we can add it and for multi values they
// should be combined into a single value
while (it.hasNext()) {
String value =
exchange.getContext().getTypeConverter().convertTo(String.class, it.next());
// we should not add headers for the parameters in the uri if we bridge the endpoint
// as then we would duplicate headers on both the endpoint uri, and in HTTP headers as
// well
if (skipRequestHeaders != null && skipRequestHeaders.containsKey(key)) {
continue;
}
if (value != null
&& strategy != null
&& !strategy.applyFilterToCamelHeaders(key, value, exchange)) {
values.add(value);
}
}
// add the value(s) as a http request header
if (values.size() > 0) {
// use the default toString of a ArrayList to create in the form [xxx, yyy]
// if multi valued, for a single value, then just output the value as is
String s = values.size() > 1 ? values.toString() : values.get(0);
method.addRequestHeader(key, s);
}
}
}
// lets store the result in the output message.
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Executing http {} method: {}", method.getName(), method.getURI().toString());
}
int responseCode = executeMethod(method);
LOG.debug("Http responseCode: {}", responseCode);
if (!throwException) {
// if we do not use failed exception then populate response for all response codes
populateResponse(exchange, method, in, strategy, responseCode);
} else {
if (responseCode >= 100 && responseCode < 300) {
// only populate response for OK response
populateResponse(exchange, method, in, strategy, responseCode);
} else {
// operation failed so populate exception to throw
throw populateHttpOperationFailedException(exchange, method, responseCode);
}
}
} finally {
method.releaseConnection();
}
}
