コード例 #1
0
  private void createRetrieveDocs(ResponseBuilder rb) {

    // TODO: in a system with nTiers > 2, we could be passed "ids" here
    // unless those requests always go to the final destination shard

    // for each shard, collect the documents for that shard.
    HashMap<String, Collection<ShardDoc>> shardMap = new HashMap<String, Collection<ShardDoc>>();
    for (ShardDoc sdoc : rb.resultIds.values()) {
      Collection<ShardDoc> shardDocs = shardMap.get(sdoc.shard);
      if (shardDocs == null) {
        shardDocs = new ArrayList<ShardDoc>();
        shardMap.put(sdoc.shard, shardDocs);
      }
      shardDocs.add(sdoc);
    }

    SchemaField uniqueField = rb.req.getSchema().getUniqueKeyField();

    // Now create a request for each shard to retrieve the stored fields
    for (Collection<ShardDoc> shardDocs : shardMap.values()) {
      ShardRequest sreq = new ShardRequest();
      sreq.purpose = ShardRequest.PURPOSE_GET_FIELDS;

      sreq.shards = new String[] {shardDocs.iterator().next().shard};

      sreq.params = new ModifiableSolrParams();

      // add original params
      sreq.params.add(rb.req.getParams());

      // no need for a sort, we already have order
      sreq.params.remove(CommonParams.SORT);

      // we already have the field sort values
      sreq.params.remove(ResponseBuilder.FIELD_SORT_VALUES);

      // make sure that the id is returned for correlation.
      String fl = sreq.params.get(CommonParams.FL);
      if (fl != null) {
        fl = fl.trim();
        // currently, "score" is synonymous with "*,score" so
        // don't add "id" if the fl is empty or "score" or it would change the meaning.
        if (fl.length() != 0 && !"score".equals(fl) && !"*".equals(fl)) {
          sreq.params.set(CommonParams.FL, fl + ',' + uniqueField.getName());
        }
      }

      ArrayList<String> ids = new ArrayList<String>(shardDocs.size());
      for (ShardDoc shardDoc : shardDocs) {
        // TODO: depending on the type, we may need more tha a simple toString()?
        ids.add(shardDoc.id.toString());
      }
      sreq.params.add(ShardParams.IDS, StrUtils.join(ids, ','));

      rb.addRequest(this, sreq);
    }
  }
コード例 #2
0
  public NamedList<Object> request(final SolrRequest request, ResponseParser processor)
      throws SolrServerException, IOException {
    HttpMethod method = null;
    InputStream is = null;
    SolrParams params = request.getParams();
    Collection<ContentStream> streams = requestWriter.getContentStreams(request);
    String path = requestWriter.getPath(request);
    if (path == null || !path.startsWith("/")) {
      path = "/select";
    }

    ResponseParser parser = request.getResponseParser();
    if (parser == null) {
      parser = _parser;
    }

    // The parser 'wt=' and 'version=' params are used instead of the original params
    ModifiableSolrParams wparams = new ModifiableSolrParams();
    wparams.set(CommonParams.WT, parser.getWriterType());
    wparams.set(CommonParams.VERSION, parser.getVersion());
    if (params == null) {
      params = wparams;
    } else {
      params = new DefaultSolrParams(wparams, params);
    }

    if (_invariantParams != null) {
      params = new DefaultSolrParams(_invariantParams, params);
    }

    int tries = _maxRetries + 1;
    try {
      while (tries-- > 0) {
        // Note: since we aren't do intermittent time keeping
        // ourselves, the potential non-timeout latency could be as
        // much as tries-times (plus scheduling effects) the given
        // timeAllowed.
        try {
          if (SolrRequest.METHOD.GET == request.getMethod()) {
            if (streams != null) {
              throw new SolrException(
                  SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
            }
            method = new GetMethod(_baseURL + path + ClientUtils.toQueryString(params, false));
          } else if (SolrRequest.METHOD.POST == request.getMethod()) {

            String url = _baseURL + path;
            boolean isMultipart = (streams != null && streams.size() > 1);

            if (streams == null || isMultipart) {
              PostMethod post = new PostMethod(url);
              post.getParams().setContentCharset("UTF-8");
              if (!this.useMultiPartPost && !isMultipart) {
                post.addRequestHeader(
                    "Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
              }

              List<Part> parts = new LinkedList<Part>();
              Iterator<String> iter = params.getParameterNamesIterator();
              while (iter.hasNext()) {
                String p = iter.next();
                String[] vals = params.getParams(p);
                if (vals != null) {
                  for (String v : vals) {
                    if (this.useMultiPartPost || isMultipart) {
                      parts.add(new StringPart(p, v, "UTF-8"));
                    } else {
                      post.addParameter(p, v);
                    }
                  }
                }
              }

              if (isMultipart) {
                int i = 0;
                for (ContentStream content : streams) {
                  final ContentStream c = content;

                  String charSet = null;
                  String transferEncoding = null;
                  parts.add(
                      new PartBase(c.getName(), c.getContentType(), charSet, transferEncoding) {
                        @Override
                        protected long lengthOfData() throws IOException {
                          return c.getSize();
                        }

                        @Override
                        protected void sendData(OutputStream out) throws IOException {
                          InputStream in = c.getStream();
                          try {
                            IOUtils.copy(in, out);
                          } finally {
                            in.close();
                          }
                        }
                      });
                }
              }
              if (parts.size() > 0) {
                post.setRequestEntity(
                    new MultipartRequestEntity(
                        parts.toArray(new Part[parts.size()]), post.getParams()));
              }

              method = post;
            }
            // It is has one stream, it is the post body, put the params in the URL
            else {
              String pstr = ClientUtils.toQueryString(params, false);
              PostMethod post = new PostMethod(url + pstr);

              // Single stream as body
              // Using a loop just to get the first one
              final ContentStream[] contentStream = new ContentStream[1];
              for (ContentStream content : streams) {
                contentStream[0] = content;
                break;
              }
              if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
                post.setRequestEntity(
                    new RequestEntity() {
                      public long getContentLength() {
                        return -1;
                      }

                      public String getContentType() {
                        return contentStream[0].getContentType();
                      }

                      public boolean isRepeatable() {
                        return false;
                      }

                      public void writeRequest(OutputStream outputStream) throws IOException {
                        ((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream);
                      }
                    });

              } else {
                is = contentStream[0].getStream();
                post.setRequestEntity(
                    new InputStreamRequestEntity(is, contentStream[0].getContentType()));
              }
              method = post;
            }
          } else {
            throw new SolrServerException("Unsupported method: " + request.getMethod());
          }
        } catch (NoHttpResponseException r) {
          // This is generally safe to retry on
          method.releaseConnection();
          method = null;
          if (is != null) {
            is.close();
          }
          // If out of tries then just rethrow (as normal error).
          if ((tries < 1)) {
            throw r;
          }
          // log.warn( "Caught: " + r + ". Retrying..." );
        }
      }
    } catch (IOException ex) {
      throw new SolrServerException("error reading streams", ex);
    }

    method.setFollowRedirects(_followRedirects);
    method.addRequestHeader("User-Agent", AGENT);
    if (_allowCompression) {
      method.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate"));
    }

    try {
      // Execute the method.
      // System.out.println( "EXECUTE:"+method.getURI() );

      int statusCode = _httpClient.executeMethod(method);
      if (statusCode != HttpStatus.SC_OK) {
        StringBuilder msg = new StringBuilder();
        msg.append(method.getStatusLine().getReasonPhrase());
        msg.append("\n\n");
        msg.append(method.getStatusText());
        msg.append("\n\n");
        msg.append("request: " + method.getURI());
        throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8"));
      }

      // Read the contents
      String charset = "UTF-8";
      if (method instanceof HttpMethodBase) {
        charset = ((HttpMethodBase) method).getResponseCharSet();
      }
      InputStream respBody = method.getResponseBodyAsStream();
      // Jakarta Commons HTTPClient doesn't handle any
      // compression natively.  Handle gzip or deflate
      // here if applicable.
      if (_allowCompression) {
        Header contentEncodingHeader = method.getResponseHeader("Content-Encoding");
        if (contentEncodingHeader != null) {
          String contentEncoding = contentEncodingHeader.getValue();
          if (contentEncoding.contains("gzip")) {
            // log.debug( "wrapping response in GZIPInputStream" );
            respBody = new GZIPInputStream(respBody);
          } else if (contentEncoding.contains("deflate")) {
            // log.debug( "wrapping response in InflaterInputStream" );
            respBody = new InflaterInputStream(respBody);
          }
        } else {
          Header contentTypeHeader = method.getResponseHeader("Content-Type");
          if (contentTypeHeader != null) {
            String contentType = contentTypeHeader.getValue();
            if (contentType != null) {
              if (contentType.startsWith("application/x-gzip-compressed")) {
                // log.debug( "wrapping response in GZIPInputStream" );
                respBody = new GZIPInputStream(respBody);
              } else if (contentType.startsWith("application/x-deflate")) {
                // log.debug( "wrapping response in InflaterInputStream" );
                respBody = new InflaterInputStream(respBody);
              }
            }
          }
        }
      }
      return processor.processResponse(respBody, charset);
    } catch (HttpException e) {
      throw new SolrServerException(e);
    } catch (IOException e) {
      throw new SolrServerException(e);
    } finally {
      method.releaseConnection();
      if (is != null) {
        is.close();
      }
    }
  }