void doPerf(String writerName, SolrQueryRequest req, int encIter, int decIter) throws Exception {
    SolrQueryResponse rsp = getResponse(req);
    QueryResponseWriter w = h.getCore().getQueryResponseWriter(writerName);

    ByteArrayOutputStream out = null;

    System.gc();
    long start = System.currentTimeMillis();
    for (int i = 0; i < encIter; i++) {
      if (w instanceof BinaryQueryResponseWriter) {
        BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter) w;
        out = new ByteArrayOutputStream();
        binWriter.write(out, req, rsp);
        out.close();
      } else {
        out = new ByteArrayOutputStream();
        // to be fair, from my previous tests, much of the performance will be sucked up
        // by java's UTF-8 encoding/decoding, not the actual writing
        Writer writer = new OutputStreamWriter(out, "UTF-8");
        w.write(writer, req, rsp);
        writer.close();
      }
    }

    long encodeTime = Math.max(System.currentTimeMillis() - start, 1);

    byte[] arr = out.toByteArray();

    start = System.currentTimeMillis();
    writerName = writerName.intern();
    for (int i = 0; i < decIter; i++) {
      ResponseParser rp = null;
      if (writerName == "xml") {
        rp = new XMLResponseParser();
      } else if (writerName == "javabin") {
        rp = new BinaryResponseParser();
      } else {
        break;
      }
      ByteArrayInputStream in = new ByteArrayInputStream(arr);
      rp.processResponse(in, "UTF-8");
    }

    long decodeTime = Math.max(System.currentTimeMillis() - start, 1);

    log.info(
        "writer "
            + writerName
            + ", size="
            + out.size()
            + ", encodeRate="
            + (encodeTime == 1 ? "N/A" : "" + (encIter * 1000L / encodeTime))
            + ", decodeRate="
            + (decodeTime == 1 ? "N/A" : "" + (decIter * 1000L / decodeTime)));

    req.close();
    SolrRequestInfo.clearRequestInfo();
  }
Exemple #2
0
  public NamedList<Object> request(final SolrRequest request, final ResponseParser processor)
      throws SolrServerException, IOException {
    HttpRequestBase method = null;
    InputStream is = null;
    SolrParams params = request.getParams();
    Collection<ContentStream> streams = requestWriter.getContentStreams(request);
    String path = requestWriter.getPath(request);
    if (path == null || !path.startsWith("/")) {
      path = DEFAULT_PATH;
    }

    ResponseParser parser = request.getResponseParser();
    if (parser == null) {
      parser = this.parser;
    }

    // The parser 'wt=' and 'version=' params are used instead of the original
    // params
    ModifiableSolrParams wparams = new ModifiableSolrParams(params);
    if (parser != null) {
      wparams.set(CommonParams.WT, parser.getWriterType());
      wparams.set(CommonParams.VERSION, parser.getVersion());
    }
    if (invariantParams != null) {
      wparams.add(invariantParams);
    }

    int tries = maxRetries + 1;
    try {
      while (tries-- > 0) {
        // Note: since we aren't do intermittent time keeping
        // ourselves, the potential non-timeout latency could be as
        // much as tries-times (plus scheduling effects) the given
        // timeAllowed.
        try {
          if (SolrRequest.METHOD.GET == request.getMethod()) {
            if (streams != null) {
              throw new SolrException(
                  SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
            }
            method = new HttpGet(baseUrl + path + ClientUtils.toQueryString(wparams, false));
          } else if (SolrRequest.METHOD.POST == request.getMethod()) {

            String url = baseUrl + path;
            boolean hasNullStreamName = false;
            if (streams != null) {
              for (ContentStream cs : streams) {
                if (cs.getName() == null) {
                  hasNullStreamName = true;
                  break;
                }
              }
            }
            boolean isMultipart =
                (this.useMultiPartPost || (streams != null && streams.size() > 1))
                    && !hasNullStreamName;

            // only send this list of params as query string params
            ModifiableSolrParams queryParams = new ModifiableSolrParams();
            for (String param : this.queryParams) {
              String[] value = wparams.getParams(param);
              if (value != null) {
                for (String v : value) {
                  queryParams.add(param, v);
                }
                wparams.remove(param);
              }
            }

            LinkedList<NameValuePair> postParams = new LinkedList<NameValuePair>();
            if (streams == null || isMultipart) {
              HttpPost post = new HttpPost(url + ClientUtils.toQueryString(queryParams, false));
              post.setHeader("Content-Charset", "UTF-8");
              if (!isMultipart) {
                post.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
              }

              List<FormBodyPart> parts = new LinkedList<FormBodyPart>();
              Iterator<String> iter = wparams.getParameterNamesIterator();
              while (iter.hasNext()) {
                String p = iter.next();
                String[] vals = wparams.getParams(p);
                if (vals != null) {
                  for (String v : vals) {
                    if (isMultipart) {
                      parts.add(new FormBodyPart(p, new StringBody(v, Charset.forName("UTF-8"))));
                    } else {
                      postParams.add(new BasicNameValuePair(p, v));
                    }
                  }
                }
              }

              if (isMultipart && streams != null) {
                for (ContentStream content : streams) {
                  String contentType = content.getContentType();
                  if (contentType == null) {
                    contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; // default
                  }
                  String name = content.getName();
                  if (name == null) {
                    name = "";
                  }
                  parts.add(
                      new FormBodyPart(
                          name,
                          new InputStreamBody(
                              content.getStream(), contentType, content.getName())));
                }
              }

              if (parts.size() > 0) {
                MultipartEntity entity = new MultipartEntity(HttpMultipartMode.STRICT);
                for (FormBodyPart p : parts) {
                  entity.addPart(p);
                }
                post.setEntity(entity);
              } else {
                // not using multipart
                post.setEntity(new UrlEncodedFormEntity(postParams, "UTF-8"));
              }

              method = post;
            }
            // It is has one stream, it is the post body, put the params in the URL
            else {
              String pstr = ClientUtils.toQueryString(wparams, false);
              HttpPost post = new HttpPost(url + pstr);

              // Single stream as body
              // Using a loop just to get the first one
              final ContentStream[] contentStream = new ContentStream[1];
              for (ContentStream content : streams) {
                contentStream[0] = content;
                break;
              }
              if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
                post.setEntity(
                    new InputStreamEntity(contentStream[0].getStream(), -1) {
                      @Override
                      public Header getContentType() {
                        return new BasicHeader("Content-Type", contentStream[0].getContentType());
                      }

                      @Override
                      public boolean isRepeatable() {
                        return false;
                      }
                    });
              } else {
                post.setEntity(
                    new InputStreamEntity(contentStream[0].getStream(), -1) {
                      @Override
                      public Header getContentType() {
                        return new BasicHeader("Content-Type", contentStream[0].getContentType());
                      }

                      @Override
                      public boolean isRepeatable() {
                        return false;
                      }
                    });
              }
              method = post;
            }
          } else {
            throw new SolrServerException("Unsupported method: " + request.getMethod());
          }
        } catch (NoHttpResponseException r) {
          method = null;
          if (is != null) {
            is.close();
          }
          // If out of tries then just rethrow (as normal error).
          if (tries < 1) {
            throw r;
          }
        }
      }
    } catch (IOException ex) {
      throw new SolrServerException("error reading streams", ex);
    }

    // XXX client already has this set, is this needed?
    method.getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, followRedirects);
    method.addHeader("User-Agent", AGENT);

    InputStream respBody = null;
    boolean shouldClose = true;
    boolean success = false;
    try {
      // Execute the method.
      final HttpResponse response = httpClient.execute(method);
      int httpStatus = response.getStatusLine().getStatusCode();

      // Read the contents
      respBody = response.getEntity().getContent();
      Header ctHeader = response.getLastHeader("content-type");
      String contentType;
      if (ctHeader != null) {
        contentType = ctHeader.getValue();
      } else {
        contentType = "";
      }

      // handle some http level checks before trying to parse the response
      switch (httpStatus) {
        case HttpStatus.SC_OK:
        case HttpStatus.SC_BAD_REQUEST:
        case HttpStatus.SC_CONFLICT: // 409
          break;
        case HttpStatus.SC_MOVED_PERMANENTLY:
        case HttpStatus.SC_MOVED_TEMPORARILY:
          if (!followRedirects) {
            throw new SolrServerException(
                "Server at " + getBaseURL() + " sent back a redirect (" + httpStatus + ").");
          }
          break;
        default:
          if (processor == null) {
            throw new RemoteSolrException(
                httpStatus,
                "Server at "
                    + getBaseURL()
                    + " returned non ok status:"
                    + httpStatus
                    + ", message:"
                    + response.getStatusLine().getReasonPhrase(),
                null);
          }
      }
      if (processor == null) {

        // no processor specified, return raw stream
        NamedList<Object> rsp = new NamedList<Object>();
        rsp.add("stream", respBody);
        // Only case where stream should not be closed
        shouldClose = false;
        success = true;
        return rsp;
      }

      String procCt = processor.getContentType();
      if (procCt != null) {
        if (!contentType.equals(procCt)) {
          // unexpected content type
          String msg = "Expected content type " + procCt + " but got " + contentType + ".";
          Header encodingHeader = response.getEntity().getContentEncoding();
          String encoding;
          if (encodingHeader != null) {
            encoding = encodingHeader.getValue();
          } else {
            encoding = "UTF-8"; // try UTF-8
          }
          try {
            msg = msg + " " + IOUtils.toString(respBody, encoding);
          } catch (IOException e) {
            throw new RemoteSolrException(
                httpStatus, "Could not parse response with encoding " + encoding, e);
          }
          RemoteSolrException e = new RemoteSolrException(httpStatus, msg, null);
          throw e;
        }
      }

      //      if(true) {
      //        ByteArrayOutputStream copy = new ByteArrayOutputStream();
      //        IOUtils.copy(respBody, copy);
      //        String val = new String(copy.toByteArray());
      //        System.out.println(">RESPONSE>"+val+"<"+val.length());
      //        respBody = new ByteArrayInputStream(copy.toByteArray());
      //      }

      NamedList<Object> rsp = null;
      String charset = EntityUtils.getContentCharSet(response.getEntity());
      try {
        rsp = processor.processResponse(respBody, charset);
      } catch (Exception e) {
        throw new RemoteSolrException(httpStatus, e.getMessage(), e);
      }
      if (httpStatus != HttpStatus.SC_OK) {
        String reason = null;
        try {
          NamedList err = (NamedList) rsp.get("error");
          if (err != null) {
            reason = (String) err.get("msg");
            // TODO? get the trace?
          }
        } catch (Exception ex) {
        }
        if (reason == null) {
          StringBuilder msg = new StringBuilder();
          msg.append(response.getStatusLine().getReasonPhrase());
          msg.append("\n\n");
          msg.append("request: " + method.getURI());
          reason = java.net.URLDecoder.decode(msg.toString(), UTF_8);
        }
        throw new RemoteSolrException(httpStatus, reason, null);
      }
      success = true;
      return rsp;
    } catch (ConnectException e) {
      throw new SolrServerException("Server refused connection at: " + getBaseURL(), e);
    } catch (SocketTimeoutException e) {
      throw new SolrServerException(
          "Timeout occured while waiting response from server at: " + getBaseURL(), e);
    } catch (IOException e) {
      throw new SolrServerException(
          "IOException occured when talking to server at: " + getBaseURL(), e);
    } finally {
      if (respBody != null && shouldClose) {
        try {
          respBody.close();
        } catch (Throwable t) {
        } // ignore
        if (!success) {
          method.abort();
        }
      }
    }
  }
  public NamedList<Object> request(final SolrRequest request, ResponseParser processor)
      throws SolrServerException, IOException {
    HttpMethod method = null;
    InputStream is = null;
    SolrParams params = request.getParams();
    Collection<ContentStream> streams = requestWriter.getContentStreams(request);
    String path = requestWriter.getPath(request);
    if (path == null || !path.startsWith("/")) {
      path = "/select";
    }

    ResponseParser parser = request.getResponseParser();
    if (parser == null) {
      parser = _parser;
    }

    // The parser 'wt=' and 'version=' params are used instead of the original params
    ModifiableSolrParams wparams = new ModifiableSolrParams();
    wparams.set(CommonParams.WT, parser.getWriterType());
    wparams.set(CommonParams.VERSION, parser.getVersion());
    if (params == null) {
      params = wparams;
    } else {
      params = new DefaultSolrParams(wparams, params);
    }

    if (_invariantParams != null) {
      params = new DefaultSolrParams(_invariantParams, params);
    }

    int tries = _maxRetries + 1;
    try {
      while (tries-- > 0) {
        // Note: since we aren't do intermittent time keeping
        // ourselves, the potential non-timeout latency could be as
        // much as tries-times (plus scheduling effects) the given
        // timeAllowed.
        try {
          if (SolrRequest.METHOD.GET == request.getMethod()) {
            if (streams != null) {
              throw new SolrException(
                  SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
            }
            method = new GetMethod(_baseURL + path + ClientUtils.toQueryString(params, false));
          } else if (SolrRequest.METHOD.POST == request.getMethod()) {

            String url = _baseURL + path;
            boolean isMultipart = (streams != null && streams.size() > 1);

            if (streams == null || isMultipart) {
              PostMethod post = new PostMethod(url);
              post.getParams().setContentCharset("UTF-8");
              if (!this.useMultiPartPost && !isMultipart) {
                post.addRequestHeader(
                    "Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
              }

              List<Part> parts = new LinkedList<Part>();
              Iterator<String> iter = params.getParameterNamesIterator();
              while (iter.hasNext()) {
                String p = iter.next();
                String[] vals = params.getParams(p);
                if (vals != null) {
                  for (String v : vals) {
                    if (this.useMultiPartPost || isMultipart) {
                      parts.add(new StringPart(p, v, "UTF-8"));
                    } else {
                      post.addParameter(p, v);
                    }
                  }
                }
              }

              if (isMultipart) {
                int i = 0;
                for (ContentStream content : streams) {
                  final ContentStream c = content;

                  String charSet = null;
                  String transferEncoding = null;
                  parts.add(
                      new PartBase(c.getName(), c.getContentType(), charSet, transferEncoding) {
                        @Override
                        protected long lengthOfData() throws IOException {
                          return c.getSize();
                        }

                        @Override
                        protected void sendData(OutputStream out) throws IOException {
                          InputStream in = c.getStream();
                          try {
                            IOUtils.copy(in, out);
                          } finally {
                            in.close();
                          }
                        }
                      });
                }
              }
              if (parts.size() > 0) {
                post.setRequestEntity(
                    new MultipartRequestEntity(
                        parts.toArray(new Part[parts.size()]), post.getParams()));
              }

              method = post;
            }
            // It is has one stream, it is the post body, put the params in the URL
            else {
              String pstr = ClientUtils.toQueryString(params, false);
              PostMethod post = new PostMethod(url + pstr);

              // Single stream as body
              // Using a loop just to get the first one
              final ContentStream[] contentStream = new ContentStream[1];
              for (ContentStream content : streams) {
                contentStream[0] = content;
                break;
              }
              if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
                post.setRequestEntity(
                    new RequestEntity() {
                      public long getContentLength() {
                        return -1;
                      }

                      public String getContentType() {
                        return contentStream[0].getContentType();
                      }

                      public boolean isRepeatable() {
                        return false;
                      }

                      public void writeRequest(OutputStream outputStream) throws IOException {
                        ((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream);
                      }
                    });

              } else {
                is = contentStream[0].getStream();
                post.setRequestEntity(
                    new InputStreamRequestEntity(is, contentStream[0].getContentType()));
              }
              method = post;
            }
          } else {
            throw new SolrServerException("Unsupported method: " + request.getMethod());
          }
        } catch (NoHttpResponseException r) {
          // This is generally safe to retry on
          method.releaseConnection();
          method = null;
          if (is != null) {
            is.close();
          }
          // If out of tries then just rethrow (as normal error).
          if ((tries < 1)) {
            throw r;
          }
          // log.warn( "Caught: " + r + ". Retrying..." );
        }
      }
    } catch (IOException ex) {
      throw new SolrServerException("error reading streams", ex);
    }

    method.setFollowRedirects(_followRedirects);
    method.addRequestHeader("User-Agent", AGENT);
    if (_allowCompression) {
      method.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate"));
    }

    try {
      // Execute the method.
      // System.out.println( "EXECUTE:"+method.getURI() );

      int statusCode = _httpClient.executeMethod(method);
      if (statusCode != HttpStatus.SC_OK) {
        StringBuilder msg = new StringBuilder();
        msg.append(method.getStatusLine().getReasonPhrase());
        msg.append("\n\n");
        msg.append(method.getStatusText());
        msg.append("\n\n");
        msg.append("request: " + method.getURI());
        throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8"));
      }

      // Read the contents
      String charset = "UTF-8";
      if (method instanceof HttpMethodBase) {
        charset = ((HttpMethodBase) method).getResponseCharSet();
      }
      InputStream respBody = method.getResponseBodyAsStream();
      // Jakarta Commons HTTPClient doesn't handle any
      // compression natively.  Handle gzip or deflate
      // here if applicable.
      if (_allowCompression) {
        Header contentEncodingHeader = method.getResponseHeader("Content-Encoding");
        if (contentEncodingHeader != null) {
          String contentEncoding = contentEncodingHeader.getValue();
          if (contentEncoding.contains("gzip")) {
            // log.debug( "wrapping response in GZIPInputStream" );
            respBody = new GZIPInputStream(respBody);
          } else if (contentEncoding.contains("deflate")) {
            // log.debug( "wrapping response in InflaterInputStream" );
            respBody = new InflaterInputStream(respBody);
          }
        } else {
          Header contentTypeHeader = method.getResponseHeader("Content-Type");
          if (contentTypeHeader != null) {
            String contentType = contentTypeHeader.getValue();
            if (contentType != null) {
              if (contentType.startsWith("application/x-gzip-compressed")) {
                // log.debug( "wrapping response in GZIPInputStream" );
                respBody = new GZIPInputStream(respBody);
              } else if (contentType.startsWith("application/x-deflate")) {
                // log.debug( "wrapping response in InflaterInputStream" );
                respBody = new InflaterInputStream(respBody);
              }
            }
          }
        }
      }
      return processor.processResponse(respBody, charset);
    } catch (HttpException e) {
      throw new SolrServerException(e);
    } catch (IOException e) {
      throw new SolrServerException(e);
    } finally {
      method.releaseConnection();
      if (is != null) {
        is.close();
      }
    }
  }