void doPerf(String writerName, SolrQueryRequest req, int encIter, int decIter) throws Exception {
SolrQueryResponse rsp = getResponse(req);
QueryResponseWriter w = h.getCore().getQueryResponseWriter(writerName);
ByteArrayOutputStream out = null;
System.gc();
long start = System.currentTimeMillis();
for (int i = 0; i < encIter; i++) {
if (w instanceof BinaryQueryResponseWriter) {
BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter) w;
out = new ByteArrayOutputStream();
binWriter.write(out, req, rsp);
out.close();
} else {
out = new ByteArrayOutputStream();
// to be fair, from my previous tests, much of the performance will be sucked up
// by java's UTF-8 encoding/decoding, not the actual writing
Writer writer = new OutputStreamWriter(out, "UTF-8");
w.write(writer, req, rsp);
writer.close();
}
}
long encodeTime = Math.max(System.currentTimeMillis() - start, 1);
byte[] arr = out.toByteArray();
start = System.currentTimeMillis();
writerName = writerName.intern();
for (int i = 0; i < decIter; i++) {
ResponseParser rp = null;
if (writerName == "xml") {
rp = new XMLResponseParser();
} else if (writerName == "javabin") {
rp = new BinaryResponseParser();
} else {
break;
}
ByteArrayInputStream in = new ByteArrayInputStream(arr);
rp.processResponse(in, "UTF-8");
}
long decodeTime = Math.max(System.currentTimeMillis() - start, 1);
log.info(
"writer "
+ writerName
+ ", size="
+ out.size()
+ ", encodeRate="
+ (encodeTime == 1 ? "N/A" : "" + (encIter * 1000L / encodeTime))
+ ", decodeRate="
+ (decodeTime == 1 ? "N/A" : "" + (decIter * 1000L / decodeTime)));
req.close();
SolrRequestInfo.clearRequestInfo();
}
public NamedList<Object> request(final SolrRequest request, final ResponseParser processor)
throws SolrServerException, IOException {
HttpRequestBase method = null;
InputStream is = null;
SolrParams params = request.getParams();
Collection<ContentStream> streams = requestWriter.getContentStreams(request);
String path = requestWriter.getPath(request);
if (path == null || !path.startsWith("/")) {
path = DEFAULT_PATH;
}
ResponseParser parser = request.getResponseParser();
if (parser == null) {
parser = this.parser;
}
// The parser 'wt=' and 'version=' params are used instead of the original
// params
ModifiableSolrParams wparams = new ModifiableSolrParams(params);
if (parser != null) {
wparams.set(CommonParams.WT, parser.getWriterType());
wparams.set(CommonParams.VERSION, parser.getVersion());
}
if (invariantParams != null) {
wparams.add(invariantParams);
}
int tries = maxRetries + 1;
try {
while (tries-- > 0) {
// Note: since we aren't do intermittent time keeping
// ourselves, the potential non-timeout latency could be as
// much as tries-times (plus scheduling effects) the given
// timeAllowed.
try {
if (SolrRequest.METHOD.GET == request.getMethod()) {
if (streams != null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
}
method = new HttpGet(baseUrl + path + ClientUtils.toQueryString(wparams, false));
} else if (SolrRequest.METHOD.POST == request.getMethod()) {
String url = baseUrl + path;
boolean hasNullStreamName = false;
if (streams != null) {
for (ContentStream cs : streams) {
if (cs.getName() == null) {
hasNullStreamName = true;
break;
}
}
}
boolean isMultipart =
(this.useMultiPartPost || (streams != null && streams.size() > 1))
&& !hasNullStreamName;
// only send this list of params as query string params
ModifiableSolrParams queryParams = new ModifiableSolrParams();
for (String param : this.queryParams) {
String[] value = wparams.getParams(param);
if (value != null) {
for (String v : value) {
queryParams.add(param, v);
}
wparams.remove(param);
}
}
LinkedList<NameValuePair> postParams = new LinkedList<NameValuePair>();
if (streams == null || isMultipart) {
HttpPost post = new HttpPost(url + ClientUtils.toQueryString(queryParams, false));
post.setHeader("Content-Charset", "UTF-8");
if (!isMultipart) {
post.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
}
List<FormBodyPart> parts = new LinkedList<FormBodyPart>();
Iterator<String> iter = wparams.getParameterNamesIterator();
while (iter.hasNext()) {
String p = iter.next();
String[] vals = wparams.getParams(p);
if (vals != null) {
for (String v : vals) {
if (isMultipart) {
parts.add(new FormBodyPart(p, new StringBody(v, Charset.forName("UTF-8"))));
} else {
postParams.add(new BasicNameValuePair(p, v));
}
}
}
}
if (isMultipart && streams != null) {
for (ContentStream content : streams) {
String contentType = content.getContentType();
if (contentType == null) {
contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; // default
}
String name = content.getName();
if (name == null) {
name = "";
}
parts.add(
new FormBodyPart(
name,
new InputStreamBody(
content.getStream(), contentType, content.getName())));
}
}
if (parts.size() > 0) {
MultipartEntity entity = new MultipartEntity(HttpMultipartMode.STRICT);
for (FormBodyPart p : parts) {
entity.addPart(p);
}
post.setEntity(entity);
} else {
// not using multipart
post.setEntity(new UrlEncodedFormEntity(postParams, "UTF-8"));
}
method = post;
}
// It is has one stream, it is the post body, put the params in the URL
else {
String pstr = ClientUtils.toQueryString(wparams, false);
HttpPost post = new HttpPost(url + pstr);
// Single stream as body
// Using a loop just to get the first one
final ContentStream[] contentStream = new ContentStream[1];
for (ContentStream content : streams) {
contentStream[0] = content;
break;
}
if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
post.setEntity(
new InputStreamEntity(contentStream[0].getStream(), -1) {
@Override
public Header getContentType() {
return new BasicHeader("Content-Type", contentStream[0].getContentType());
}
@Override
public boolean isRepeatable() {
return false;
}
});
} else {
post.setEntity(
new InputStreamEntity(contentStream[0].getStream(), -1) {
@Override
public Header getContentType() {
return new BasicHeader("Content-Type", contentStream[0].getContentType());
}
@Override
public boolean isRepeatable() {
return false;
}
});
}
method = post;
}
} else {
throw new SolrServerException("Unsupported method: " + request.getMethod());
}
} catch (NoHttpResponseException r) {
method = null;
if (is != null) {
is.close();
}
// If out of tries then just rethrow (as normal error).
if (tries < 1) {
throw r;
}
}
}
} catch (IOException ex) {
throw new SolrServerException("error reading streams", ex);
}
// XXX client already has this set, is this needed?
method.getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, followRedirects);
method.addHeader("User-Agent", AGENT);
InputStream respBody = null;
boolean shouldClose = true;
boolean success = false;
try {
// Execute the method.
final HttpResponse response = httpClient.execute(method);
int httpStatus = response.getStatusLine().getStatusCode();
// Read the contents
respBody = response.getEntity().getContent();
Header ctHeader = response.getLastHeader("content-type");
String contentType;
if (ctHeader != null) {
contentType = ctHeader.getValue();
} else {
contentType = "";
}
// handle some http level checks before trying to parse the response
switch (httpStatus) {
case HttpStatus.SC_OK:
case HttpStatus.SC_BAD_REQUEST:
case HttpStatus.SC_CONFLICT: // 409
break;
case HttpStatus.SC_MOVED_PERMANENTLY:
case HttpStatus.SC_MOVED_TEMPORARILY:
if (!followRedirects) {
throw new SolrServerException(
"Server at " + getBaseURL() + " sent back a redirect (" + httpStatus + ").");
}
break;
default:
if (processor == null) {
throw new RemoteSolrException(
httpStatus,
"Server at "
+ getBaseURL()
+ " returned non ok status:"
+ httpStatus
+ ", message:"
+ response.getStatusLine().getReasonPhrase(),
null);
}
}
if (processor == null) {
// no processor specified, return raw stream
NamedList<Object> rsp = new NamedList<Object>();
rsp.add("stream", respBody);
// Only case where stream should not be closed
shouldClose = false;
success = true;
return rsp;
}
String procCt = processor.getContentType();
if (procCt != null) {
if (!contentType.equals(procCt)) {
// unexpected content type
String msg = "Expected content type " + procCt + " but got " + contentType + ".";
Header encodingHeader = response.getEntity().getContentEncoding();
String encoding;
if (encodingHeader != null) {
encoding = encodingHeader.getValue();
} else {
encoding = "UTF-8"; // try UTF-8
}
try {
msg = msg + " " + IOUtils.toString(respBody, encoding);
} catch (IOException e) {
throw new RemoteSolrException(
httpStatus, "Could not parse response with encoding " + encoding, e);
}
RemoteSolrException e = new RemoteSolrException(httpStatus, msg, null);
throw e;
}
}
// if(true) {
// ByteArrayOutputStream copy = new ByteArrayOutputStream();
// IOUtils.copy(respBody, copy);
// String val = new String(copy.toByteArray());
// System.out.println(">RESPONSE>"+val+"<"+val.length());
// respBody = new ByteArrayInputStream(copy.toByteArray());
// }
NamedList<Object> rsp = null;
String charset = EntityUtils.getContentCharSet(response.getEntity());
try {
rsp = processor.processResponse(respBody, charset);
} catch (Exception e) {
throw new RemoteSolrException(httpStatus, e.getMessage(), e);
}
if (httpStatus != HttpStatus.SC_OK) {
String reason = null;
try {
NamedList err = (NamedList) rsp.get("error");
if (err != null) {
reason = (String) err.get("msg");
// TODO? get the trace?
}
} catch (Exception ex) {
}
if (reason == null) {
StringBuilder msg = new StringBuilder();
msg.append(response.getStatusLine().getReasonPhrase());
msg.append("\n\n");
msg.append("request: " + method.getURI());
reason = java.net.URLDecoder.decode(msg.toString(), UTF_8);
}
throw new RemoteSolrException(httpStatus, reason, null);
}
success = true;
return rsp;
} catch (ConnectException e) {
throw new SolrServerException("Server refused connection at: " + getBaseURL(), e);
} catch (SocketTimeoutException e) {
throw new SolrServerException(
"Timeout occured while waiting response from server at: " + getBaseURL(), e);
} catch (IOException e) {
throw new SolrServerException(
"IOException occured when talking to server at: " + getBaseURL(), e);
} finally {
if (respBody != null && shouldClose) {
try {
respBody.close();
} catch (Throwable t) {
} // ignore
if (!success) {
method.abort();
}
}
}
}
public NamedList<Object> request(final SolrRequest request, ResponseParser processor)
throws SolrServerException, IOException {
HttpMethod method = null;
InputStream is = null;
SolrParams params = request.getParams();
Collection<ContentStream> streams = requestWriter.getContentStreams(request);
String path = requestWriter.getPath(request);
if (path == null || !path.startsWith("/")) {
path = "/select";
}
ResponseParser parser = request.getResponseParser();
if (parser == null) {
parser = _parser;
}
// The parser 'wt=' and 'version=' params are used instead of the original params
ModifiableSolrParams wparams = new ModifiableSolrParams();
wparams.set(CommonParams.WT, parser.getWriterType());
wparams.set(CommonParams.VERSION, parser.getVersion());
if (params == null) {
params = wparams;
} else {
params = new DefaultSolrParams(wparams, params);
}
if (_invariantParams != null) {
params = new DefaultSolrParams(_invariantParams, params);
}
int tries = _maxRetries + 1;
try {
while (tries-- > 0) {
// Note: since we aren't do intermittent time keeping
// ourselves, the potential non-timeout latency could be as
// much as tries-times (plus scheduling effects) the given
// timeAllowed.
try {
if (SolrRequest.METHOD.GET == request.getMethod()) {
if (streams != null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
}
method = new GetMethod(_baseURL + path + ClientUtils.toQueryString(params, false));
} else if (SolrRequest.METHOD.POST == request.getMethod()) {
String url = _baseURL + path;
boolean isMultipart = (streams != null && streams.size() > 1);
if (streams == null || isMultipart) {
PostMethod post = new PostMethod(url);
post.getParams().setContentCharset("UTF-8");
if (!this.useMultiPartPost && !isMultipart) {
post.addRequestHeader(
"Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
}
List<Part> parts = new LinkedList<Part>();
Iterator<String> iter = params.getParameterNamesIterator();
while (iter.hasNext()) {
String p = iter.next();
String[] vals = params.getParams(p);
if (vals != null) {
for (String v : vals) {
if (this.useMultiPartPost || isMultipart) {
parts.add(new StringPart(p, v, "UTF-8"));
} else {
post.addParameter(p, v);
}
}
}
}
if (isMultipart) {
int i = 0;
for (ContentStream content : streams) {
final ContentStream c = content;
String charSet = null;
String transferEncoding = null;
parts.add(
new PartBase(c.getName(), c.getContentType(), charSet, transferEncoding) {
@Override
protected long lengthOfData() throws IOException {
return c.getSize();
}
@Override
protected void sendData(OutputStream out) throws IOException {
InputStream in = c.getStream();
try {
IOUtils.copy(in, out);
} finally {
in.close();
}
}
});
}
}
if (parts.size() > 0) {
post.setRequestEntity(
new MultipartRequestEntity(
parts.toArray(new Part[parts.size()]), post.getParams()));
}
method = post;
}
// It is has one stream, it is the post body, put the params in the URL
else {
String pstr = ClientUtils.toQueryString(params, false);
PostMethod post = new PostMethod(url + pstr);
// Single stream as body
// Using a loop just to get the first one
final ContentStream[] contentStream = new ContentStream[1];
for (ContentStream content : streams) {
contentStream[0] = content;
break;
}
if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
post.setRequestEntity(
new RequestEntity() {
public long getContentLength() {
return -1;
}
public String getContentType() {
return contentStream[0].getContentType();
}
public boolean isRepeatable() {
return false;
}
public void writeRequest(OutputStream outputStream) throws IOException {
((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream);
}
});
} else {
is = contentStream[0].getStream();
post.setRequestEntity(
new InputStreamRequestEntity(is, contentStream[0].getContentType()));
}
method = post;
}
} else {
throw new SolrServerException("Unsupported method: " + request.getMethod());
}
} catch (NoHttpResponseException r) {
// This is generally safe to retry on
method.releaseConnection();
method = null;
if (is != null) {
is.close();
}
// If out of tries then just rethrow (as normal error).
if ((tries < 1)) {
throw r;
}
// log.warn( "Caught: " + r + ". Retrying..." );
}
}
} catch (IOException ex) {
throw new SolrServerException("error reading streams", ex);
}
method.setFollowRedirects(_followRedirects);
method.addRequestHeader("User-Agent", AGENT);
if (_allowCompression) {
method.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate"));
}
try {
// Execute the method.
// System.out.println( "EXECUTE:"+method.getURI() );
int statusCode = _httpClient.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
StringBuilder msg = new StringBuilder();
msg.append(method.getStatusLine().getReasonPhrase());
msg.append("\n\n");
msg.append(method.getStatusText());
msg.append("\n\n");
msg.append("request: " + method.getURI());
throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8"));
}
// Read the contents
String charset = "UTF-8";
if (method instanceof HttpMethodBase) {
charset = ((HttpMethodBase) method).getResponseCharSet();
}
InputStream respBody = method.getResponseBodyAsStream();
// Jakarta Commons HTTPClient doesn't handle any
// compression natively. Handle gzip or deflate
// here if applicable.
if (_allowCompression) {
Header contentEncodingHeader = method.getResponseHeader("Content-Encoding");
if (contentEncodingHeader != null) {
String contentEncoding = contentEncodingHeader.getValue();
if (contentEncoding.contains("gzip")) {
// log.debug( "wrapping response in GZIPInputStream" );
respBody = new GZIPInputStream(respBody);
} else if (contentEncoding.contains("deflate")) {
// log.debug( "wrapping response in InflaterInputStream" );
respBody = new InflaterInputStream(respBody);
}
} else {
Header contentTypeHeader = method.getResponseHeader("Content-Type");
if (contentTypeHeader != null) {
String contentType = contentTypeHeader.getValue();
if (contentType != null) {
if (contentType.startsWith("application/x-gzip-compressed")) {
// log.debug( "wrapping response in GZIPInputStream" );
respBody = new GZIPInputStream(respBody);
} else if (contentType.startsWith("application/x-deflate")) {
// log.debug( "wrapping response in InflaterInputStream" );
respBody = new InflaterInputStream(respBody);
}
}
}
}
}
return processor.processResponse(respBody, charset);
} catch (HttpException e) {
throw new SolrServerException(e);
} catch (IOException e) {
throw new SolrServerException(e);
} finally {
method.releaseConnection();
if (is != null) {
is.close();
}
}
}