/**
* Builds a String representation of the URI with cleaned parameters, that can be used when
* checking if an URI was already visited. The URI provided as a parameter should be already
* cleaned and canonicalized, so it should be build with a result from {@link
* #getCanonicalURL(String)}.
*
* <p>When building the URI representation, the same format should be used for all the cases, as
* it may affect the number of times the pages are visited and reported if the option
* HandleParametersOption is changed while the spider is running.
*
* @param uri the uri
* @param handleParameters the handle parameters option
* @param handleODataParametersVisited Should we handle specific OData parameters
* @return the string representation of the URI
* @throws URIException the URI exception
*/
public static String buildCleanedParametersURIRepresentation(
org.apache.commons.httpclient.URI uri,
SpiderParam.HandleParametersOption handleParameters,
boolean handleODataParametersVisited)
throws URIException {
// If the option is set to use all the information, just use the default string representation
if (handleParameters.equals(HandleParametersOption.USE_ALL)) {
return uri.toString();
}
// If the option is set to ignore parameters completely, ignore the query completely
if (handleParameters.equals(HandleParametersOption.IGNORE_COMPLETELY)) {
return createBaseUriWithCleanedPath(uri, handleParameters, handleODataParametersVisited);
}
// If the option is set to ignore the value, we get the parameters and we only add their name to
// the
// query
if (handleParameters.equals(HandleParametersOption.IGNORE_VALUE)) {
StringBuilder retVal =
new StringBuilder(
createBaseUriWithCleanedPath(uri, handleParameters, handleODataParametersVisited));
String cleanedQuery = getCleanedQuery(uri.getEscapedQuery());
// Add the parameters' names to the uri representation.
if (cleanedQuery.length() > 0) {
retVal.append('?').append(cleanedQuery);
}
return retVal.toString();
}
// Should not be reached
return uri.toString();
}
@SuppressWarnings("deprecation")
@Override
public void filterHttpRequest(SubmitContext context, HttpRequestInterface<?> request) {
HttpRequestBase httpMethod =
(HttpRequestBase) context.getProperty(BaseHttpRequestTransport.HTTP_METHOD);
String path = PropertyExpander.expandProperties(context, request.getPath());
StringBuffer query = new StringBuffer();
String encoding =
System.getProperty("soapui.request.encoding", StringUtils.unquote(request.getEncoding()));
StringToStringMap responseProperties =
(StringToStringMap) context.getProperty(BaseHttpRequestTransport.RESPONSE_PROPERTIES);
MimeMultipart formMp =
"multipart/form-data".equals(request.getMediaType())
&& httpMethod instanceof HttpEntityEnclosingRequestBase
? new MimeMultipart()
: null;
RestParamsPropertyHolder params = request.getParams();
for (int c = 0; c < params.getPropertyCount(); c++) {
RestParamProperty param = params.getPropertyAt(c);
String value = PropertyExpander.expandProperties(context, param.getValue());
responseProperties.put(param.getName(), value);
List<String> valueParts =
sendEmptyParameters(request) || (!StringUtils.hasContent(value) && param.getRequired())
? RestUtils.splitMultipleParametersEmptyIncluded(
value, request.getMultiValueDelimiter())
: RestUtils.splitMultipleParameters(value, request.getMultiValueDelimiter());
// skip HEADER and TEMPLATE parameter encoding (TEMPLATE is encoded by
// the URI handling further down)
if (value != null
&& param.getStyle() != ParameterStyle.HEADER
&& param.getStyle() != ParameterStyle.TEMPLATE
&& !param.isDisableUrlEncoding()) {
try {
if (StringUtils.hasContent(encoding)) {
value = URLEncoder.encode(value, encoding);
for (int i = 0; i < valueParts.size(); i++)
valueParts.set(i, URLEncoder.encode(valueParts.get(i), encoding));
} else {
value = URLEncoder.encode(value);
for (int i = 0; i < valueParts.size(); i++)
valueParts.set(i, URLEncoder.encode(valueParts.get(i)));
}
} catch (UnsupportedEncodingException e1) {
SoapUI.logError(e1);
value = URLEncoder.encode(value);
for (int i = 0; i < valueParts.size(); i++)
valueParts.set(i, URLEncoder.encode(valueParts.get(i)));
}
// URLEncoder replaces space with "+", but we want "%20".
value = value.replaceAll("\\+", "%20");
for (int i = 0; i < valueParts.size(); i++)
valueParts.set(i, valueParts.get(i).replaceAll("\\+", "%20"));
}
if (param.getStyle() == ParameterStyle.QUERY && !sendEmptyParameters(request)) {
if (!StringUtils.hasContent(value) && !param.getRequired()) continue;
}
switch (param.getStyle()) {
case HEADER:
for (String valuePart : valueParts) httpMethod.addHeader(param.getName(), valuePart);
break;
case QUERY:
if (formMp == null || !request.isPostQueryString()) {
for (String valuePart : valueParts) {
if (query.length() > 0) query.append('&');
query.append(URLEncoder.encode(param.getName()));
query.append('=');
if (StringUtils.hasContent(valuePart)) query.append(valuePart);
}
} else {
try {
addFormMultipart(
request, formMp, param.getName(), responseProperties.get(param.getName()));
} catch (MessagingException e) {
SoapUI.logError(e);
}
}
break;
case TEMPLATE:
try {
value =
getEncodedValue(
value,
encoding,
param.isDisableUrlEncoding(),
request.getSettings().getBoolean(HttpSettings.ENCODED_URLS));
path = path.replaceAll("\\{" + param.getName() + "\\}", value == null ? "" : value);
} catch (UnsupportedEncodingException e) {
SoapUI.logError(e);
}
break;
case MATRIX:
try {
value =
getEncodedValue(
value,
encoding,
param.isDisableUrlEncoding(),
request.getSettings().getBoolean(HttpSettings.ENCODED_URLS));
} catch (UnsupportedEncodingException e) {
SoapUI.logError(e);
}
if (param.getType().equals(XmlBoolean.type.getName())) {
if (value.toUpperCase().equals("TRUE") || value.equals("1")) {
path += ";" + param.getName();
}
} else {
path += ";" + param.getName();
if (StringUtils.hasContent(value)) {
path += "=" + value;
}
}
break;
case PLAIN:
break;
}
}
if (request.getSettings().getBoolean(HttpSettings.FORWARD_SLASHES))
path = PathUtils.fixForwardSlashesInPath(path);
if (PathUtils.isHttpPath(path)) {
try {
// URI(String) automatically URLencodes the input, so we need to
// decode it first...
URI uri = new URI(path, request.getSettings().getBoolean(HttpSettings.ENCODED_URLS));
context.setProperty(BaseHttpRequestTransport.REQUEST_URI, uri);
java.net.URI oldUri = httpMethod.getURI();
httpMethod.setURI(
HttpUtils.createUri(
oldUri.getScheme(),
oldUri.getRawUserInfo(),
oldUri.getHost(),
oldUri.getPort(),
oldUri.getRawPath(),
uri.getEscapedQuery(),
oldUri.getRawFragment()));
} catch (Exception e) {
SoapUI.logError(e);
}
} else if (StringUtils.hasContent(path)) {
try {
java.net.URI oldUri = httpMethod.getURI();
String pathToSet =
StringUtils.hasContent(oldUri.getRawPath()) && !"/".equals(oldUri.getRawPath())
? oldUri.getRawPath() + path
: path;
java.net.URI newUri =
URIUtils.createURI(
oldUri.getScheme(),
oldUri.getHost(),
oldUri.getPort(),
pathToSet,
oldUri.getQuery(),
oldUri.getFragment());
httpMethod.setURI(newUri);
context.setProperty(
BaseHttpRequestTransport.REQUEST_URI,
new URI(
newUri.toString(), request.getSettings().getBoolean(HttpSettings.ENCODED_URLS)));
} catch (Exception e) {
SoapUI.logError(e);
}
}
if (query.length() > 0 && !request.isPostQueryString()) {
try {
java.net.URI oldUri = httpMethod.getURI();
httpMethod.setURI(
URIUtils.createURI(
oldUri.getScheme(),
oldUri.getHost(),
oldUri.getPort(),
oldUri.getRawPath(),
query.toString(),
oldUri.getFragment()));
} catch (Exception e) {
SoapUI.logError(e);
}
}
if (request instanceof RestRequest) {
String acceptEncoding = ((RestRequest) request).getAccept();
if (StringUtils.hasContent(acceptEncoding)) {
httpMethod.setHeader("Accept", acceptEncoding);
}
}
if (formMp != null) {
// create request message
try {
if (request.hasRequestBody() && httpMethod instanceof HttpEntityEnclosingRequest) {
String requestContent =
PropertyExpander.expandProperties(
context, request.getRequestContent(), request.isEntitizeProperties());
if (StringUtils.hasContent(requestContent)) {
initRootPart(request, requestContent, formMp);
}
}
for (Attachment attachment : request.getAttachments()) {
MimeBodyPart part = new PreencodedMimeBodyPart("binary");
if (attachment instanceof FileAttachment<?>) {
String name = attachment.getName();
if (StringUtils.hasContent(attachment.getContentID())
&& !name.equals(attachment.getContentID())) name = attachment.getContentID();
part.setDisposition(
"form-data; name=\"" + name + "\"; filename=\"" + attachment.getName() + "\"");
} else part.setDisposition("form-data; name=\"" + attachment.getName() + "\"");
part.setDataHandler(new DataHandler(new AttachmentDataSource(attachment)));
formMp.addBodyPart(part);
}
MimeMessage message = new MimeMessage(AttachmentUtils.JAVAMAIL_SESSION);
message.setContent(formMp);
message.saveChanges();
RestRequestMimeMessageRequestEntity mimeMessageRequestEntity =
new RestRequestMimeMessageRequestEntity(message, request);
((HttpEntityEnclosingRequest) httpMethod).setEntity(mimeMessageRequestEntity);
httpMethod.setHeader("Content-Type", mimeMessageRequestEntity.getContentType().getValue());
httpMethod.setHeader("MIME-Version", "1.0");
} catch (Throwable e) {
SoapUI.logError(e);
}
} else if (request.hasRequestBody() && httpMethod instanceof HttpEntityEnclosingRequest) {
if (StringUtils.hasContent(request.getMediaType()))
httpMethod.setHeader(
"Content-Type", getContentTypeHeader(request.getMediaType(), encoding));
if (request.isPostQueryString()) {
try {
((HttpEntityEnclosingRequest) httpMethod).setEntity(new StringEntity(query.toString()));
} catch (UnsupportedEncodingException e) {
SoapUI.logError(e);
}
} else {
String requestContent =
PropertyExpander.expandProperties(
context, request.getRequestContent(), request.isEntitizeProperties());
List<Attachment> attachments = new ArrayList<Attachment>();
for (Attachment attachment : request.getAttachments()) {
if (attachment.getContentType().equals(request.getMediaType())) {
attachments.add(attachment);
}
}
if (StringUtils.hasContent(requestContent) && attachments.isEmpty()) {
try {
byte[] content =
encoding == null ? requestContent.getBytes() : requestContent.getBytes(encoding);
((HttpEntityEnclosingRequest) httpMethod).setEntity(new ByteArrayEntity(content));
} catch (UnsupportedEncodingException e) {
((HttpEntityEnclosingRequest) httpMethod)
.setEntity(new ByteArrayEntity(requestContent.getBytes()));
}
} else if (attachments.size() > 0) {
try {
MimeMultipart mp = null;
if (StringUtils.hasContent(requestContent)) {
mp = new MimeMultipart();
initRootPart(request, requestContent, mp);
} else if (attachments.size() == 1) {
((HttpEntityEnclosingRequest) httpMethod)
.setEntity(new InputStreamEntity(attachments.get(0).getInputStream(), -1));
httpMethod.setHeader(
"Content-Type", getContentTypeHeader(request.getMediaType(), encoding));
}
if (((HttpEntityEnclosingRequest) httpMethod).getEntity() == null) {
if (mp == null) mp = new MimeMultipart();
// init mimeparts
AttachmentUtils.addMimeParts(request, attachments, mp, new StringToStringMap());
// create request message
MimeMessage message = new MimeMessage(AttachmentUtils.JAVAMAIL_SESSION);
message.setContent(mp);
message.saveChanges();
RestRequestMimeMessageRequestEntity mimeMessageRequestEntity =
new RestRequestMimeMessageRequestEntity(message, request);
((HttpEntityEnclosingRequest) httpMethod).setEntity(mimeMessageRequestEntity);
httpMethod.setHeader(
"Content-Type",
getContentTypeHeader(
mimeMessageRequestEntity.getContentType().getValue(), encoding));
httpMethod.setHeader("MIME-Version", "1.0");
}
} catch (Exception e) {
SoapUI.logError(e);
}
}
}
}
}
/**
* @param since last modified time to use
* @param req
* @param url if null, ignored
* @param redirCount number of redirs we've done
*/
public static HttpData getDataOnce(
HttpServletRequest req,
HttpServletResponse res,
long since,
String surl,
int redirCount,
int timeout)
throws IOException, HttpException, DataSourceException, MalformedURLException {
HttpMethodBase request = null;
HostConfiguration hcfg = new HostConfiguration();
/*
[todo hqm 2006-02-01] Anyone know why this code was here? It is setting
the mime type to something which just confuses the DHTML parser.
if (res != null) {
res.setContentType("application/x-www-form-urlencoded;charset=UTF-8");
}
*/
try {
// TODO: [2002-01-09 bloch] cope with cache-control
// response headers (no-store, no-cache, must-revalidate,
// proxy-revalidate).
if (surl == null) {
surl = getURL(req);
}
if (surl == null || surl.equals("")) {
throw new MalformedURLException(
/* (non-Javadoc)
* @i18n.test
* @org-mes="url is empty or null"
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-312"));
}
String reqType = "";
String headers = "";
if (req != null) {
reqType = req.getParameter("reqtype");
headers = req.getParameter("headers");
}
boolean isPost = false;
mLogger.debug("reqtype = " + reqType);
if (reqType != null && reqType.equals("POST")) {
request = new LZPostMethod();
request.setRequestHeader("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8");
isPost = true;
mLogger.debug("setting POST req method");
} else if (reqType != null && reqType.equals("PUT")) {
request = new LZPutMethod();
// todo [hqm 2007] treat PUT like POST?
isPost = true;
mLogger.debug("setting PUT req method");
} else if (reqType != null && reqType.equals("DELETE")) {
request = new LZDeleteMethod();
mLogger.debug("setting DELETE req method");
} else {
mLogger.debug("setting GET (default) req method");
request = new LZGetMethod();
}
request.setHttp11(mUseHttp11);
// Proxy the request headers
if (req != null) {
LZHttpUtils.proxyRequestHeaders(req, request);
}
// Set headers from query string
if (headers != null && headers.length() > 0) {
StringTokenizer st = new StringTokenizer(headers, "\n");
while (st.hasMoreTokens()) {
String h = st.nextToken();
int i = h.indexOf(":");
if (i > -1) {
String n = h.substring(0, i);
String v = h.substring(i + 2, h.length());
request.setRequestHeader(n, v);
mLogger.debug(
/* (non-Javadoc)
* @i18n.test
* @org-mes="setting header " + p[0] + "=" + p[1]
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-359", new Object[] {n, v}));
}
}
}
mLogger.debug("Parsing url");
URI uri = LZHttpUtils.newURI(surl);
try {
hcfg.setHost(uri);
} catch (Exception e) {
throw new MalformedURLException(
/* (non-Javadoc)
* @i18n.test
* @org-mes="can't form uri from " + p[0]
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-376", new Object[] {surl}));
}
// This gets us the url-encoded (escaped) path and query string
String path = uri.getEscapedPath();
String query = uri.getEscapedQuery();
mLogger.debug(
/* (non-Javadoc)
* @i18n.test
* @org-mes="encoded path: " + p[0]
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-389", new Object[] {path}));
mLogger.debug(
/* (non-Javadoc)
* @i18n.test
* @org-mes="encoded query: " + p[0]
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-397", new Object[] {query}));
// This call takes a decoded (unescaped) path
request.setPath(path);
boolean hasQuery = (query != null && query.length() > 0);
String rawcontent = null;
// Newer rawpost protocol puts lzpostbody as a separate
// top level query arg in the request.
rawcontent = req.getParameter("lzpostbody");
if (isPost) {
// Older rawpost protocol put the "lzpostbody" arg
// embedded in the "url" args's query args
if (rawcontent == null && hasQuery) {
rawcontent = findQueryArg("lzpostbody", query);
}
if (rawcontent != null) {
// Get the unescaped query string
((EntityEnclosingMethod) request).setRequestBody(rawcontent);
} else if (hasQuery) {
StringTokenizer st = new StringTokenizer(query, "&");
while (st.hasMoreTokens()) {
String it = st.nextToken();
int i = it.indexOf("=");
if (i > 0) {
String n = it.substring(0, i);
String v = it.substring(i + 1, it.length());
// POST encodes values during request
((PostMethod) request).addParameter(n, URLDecoder.decode(v, "UTF-8"));
} else {
mLogger.warn(
/* (non-Javadoc)
* @i18n.test
* @org-mes="ignoring bad token (missing '=' char) in query string: " + p[0]
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-429", new Object[] {it}));
}
}
}
} else {
// This call takes an encoded (escaped) query string
request.setQueryString(query);
}
// Put in the If-Modified-Since headers
if (since != -1) {
String lms = LZHttpUtils.getDateString(since);
request.setRequestHeader(LZHttpUtils.IF_MODIFIED_SINCE, lms);
mLogger.debug(
/* (non-Javadoc)
* @i18n.test
* @org-mes="proxying lms: " + p[0]
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-450", new Object[] {lms}));
}
mLogger.debug(
/* (non-Javadoc)
* @i18n.test
* @org-mes="setting up http client"
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-460"));
HttpClient htc = null;
if (mConnectionMgr != null) {
htc = new HttpClient(mConnectionMgr);
} else {
htc = new HttpClient();
}
htc.setHostConfiguration(hcfg);
// This is the data timeout
mLogger.debug(
/* (non-Javadoc)
* @i18n.test
* @org-mes="timeout set to " + p[0]
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-478", new Object[] {new Integer(timeout)}));
htc.setTimeout(timeout);
// Set connection timeout the same
htc.setConnectionTimeout(mConnectionTimeout);
// Set timeout for getting a connection
htc.setHttpConnectionFactoryTimeout(mConnectionPoolTimeout);
// TODO: [2003-03-05 bloch] this should be more configurable (per app?)
if (!isPost) {
request.setFollowRedirects(mFollowRedirects > 0);
}
long t1 = System.currentTimeMillis();
mLogger.debug("starting remote request");
int rc = htc.executeMethod(hcfg, request);
String status = HttpStatus.getStatusText(rc);
if (status == null) {
status = "" + rc;
}
mLogger.debug(
/* (non-Javadoc)
* @i18n.test
* @org-mes="remote response status: " + p[0]
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-504", new Object[] {status}));
HttpData data = null;
if (isRedirect(rc) && mFollowRedirects > redirCount) {
String loc = request.getResponseHeader("Location").toString();
String hostURI = loc.substring(loc.indexOf(": ") + 2, loc.length());
mLogger.info(
/* (non-Javadoc)
* @i18n.test
* @org-mes="Following URL from redirect: " + p[0]
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-517", new Object[] {hostURI}));
long t2 = System.currentTimeMillis();
if (timeout > 0) {
timeout -= (t2 - t1);
if (timeout < 0) {
throw new InterruptedIOException(
/* (non-Javadoc)
* @i18n.test
* @org-mes=p[0] + " timed out after redirecting to " + p[1]
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-529", new Object[] {surl, loc}));
}
}
data = getDataOnce(req, res, since, hostURI, redirCount++, timeout);
} else {
data = new HttpData(request, rc);
}
if (req != null && res != null) {
// proxy response headers
LZHttpUtils.proxyResponseHeaders(request, res, req.isSecure());
}
return data;
} catch (ConnectTimeoutException ce) {
// Transduce to an InterrupedIOException, since lps takes these to be timeouts.
if (request != null) {
request.releaseConnection();
}
throw new InterruptedIOException(
/* (non-Javadoc)
* @i18n.test
* @org-mes="connecting to " + p[0] + ":" + p[1] + " timed out beyond " + p[2] + " msecs."
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(),
"051018-557",
new Object[] {
hcfg.getHost(), new Integer(hcfg.getPort()), new Integer(mConnectionTimeout)
}));
} catch (HttpRecoverableException hre) {
if (request != null) {
request.releaseConnection();
}
throw hre;
} catch (HttpException e) {
if (request != null) {
request.releaseConnection();
}
throw e;
} catch (IOException ie) {
if (request != null) {
request.releaseConnection();
}
throw ie;
} catch (RuntimeException e) {
if (request != null) {
request.releaseConnection();
}
throw e;
}
}