/** Generate external process results (if they have not already been generated). */ @Override public void prepare(ResponseBuilder rb) throws IOException { SolrParams params = rb.req.getParams(); if (!params.getBool(getName(), false)) { return; } XJoinResults<?> results = (XJoinResults<?>) rb.req.getContext().get(getResultsTag()); if (results != null) { return; } // generate external process results, by passing 'external' prefixed parameters // from the query string to our factory String prefix = getName() + "." + XJoinParameters.EXTERNAL_PREFIX + "."; ModifiableSolrParams externalParams = new ModifiableSolrParams(); for (Iterator<String> it = params.getParameterNamesIterator(); it.hasNext(); ) { String name = it.next(); if (name.startsWith(prefix)) { externalParams.set(name.substring(prefix.length()), params.get(name)); } } results = factory.getResults(externalParams); rb.req.getContext().put(getResultsTag(), results); }
public static String toQueryString(SolrParams params, boolean xml) { StringBuilder sb = new StringBuilder(128); try { String amp = xml ? "&" : "&"; boolean first = true; Iterator<String> names = params.getParameterNamesIterator(); while (names.hasNext()) { String key = names.next(); String[] valarr = params.getParams(key); if (valarr == null) { sb.append(first ? "?" : amp); sb.append(key); first = false; } else { for (String val : valarr) { sb.append(first ? "?" : amp); sb.append(key); if (val != null) { sb.append('='); sb.append(URLEncoder.encode(val, "UTF-8")); } first = false; } } } } catch (IOException e) { throw new RuntimeException(e); } // can't happen return sb.toString(); }
/** * Serialize Solr parameters into a String. * * @param params the parameters to serialize * @return a String serialization of the parameters */ public static String getCacheKey(SolrParams params) { StringBuilder out = new StringBuilder(); out.append('{'); Iterator<String> parameterNames = params.getParameterNamesIterator(); while (parameterNames.hasNext()) { String parameter = parameterNames.next(); out.append(parameter) .append(":[") .append(StringUtils.join(params.getParams(parameter), ", ")) .append("]\n"); } out.append('}'); return out.toString(); }
/** * Handle 'CREATE' action. * * @param req * @param rsp * @return true if a modification has resulted that requires persistance of the CoreContainer * configuration. * @throws SolrException in case of a configuration error. */ protected boolean handleCreateAction(SolrQueryRequest req, SolrQueryResponse rsp) throws SolrException { try { SolrParams params = req.getParams(); String name = params.get(CoreAdminParams.NAME); CoreDescriptor dcore = new CoreDescriptor(coreContainer, name, params.get(CoreAdminParams.INSTANCE_DIR)); // fillup optional parameters String opts = params.get(CoreAdminParams.CONFIG); if (opts != null) dcore.setConfigName(opts); opts = params.get(CoreAdminParams.SCHEMA); if (opts != null) dcore.setSchemaName(opts); opts = params.get(CoreAdminParams.DATA_DIR); if (opts != null) dcore.setDataDir(opts); // Process all property.name=value parameters and set them as name=value core properties Properties coreProperties = new Properties(); Iterator<String> parameterNamesIterator = params.getParameterNamesIterator(); while (parameterNamesIterator.hasNext()) { String parameterName = parameterNamesIterator.next(); if (parameterName.startsWith(CoreAdminParams.PROPERTY_PREFIX)) { String parameterValue = params.get(parameterName); String propertyName = parameterName.substring(CoreAdminParams.PROPERTY_PREFIX.length()); // skip prefix coreProperties.put(propertyName, parameterValue); } } dcore.setCoreProperties(coreProperties); SolrCore core = coreContainer.create(dcore); coreContainer.register(name, core, false); rsp.add("core", core.getName()); return coreContainer.isPersistent(); } catch (Exception ex) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Error executing default implementation of CREATE", ex); } }
/** create the FieldAdders that control how each field is indexed */ void prepareFields() { // Possible future optimization: for really rapid incremental indexing // from a POST, one could cache all of this setup info based on the params. // The link from FieldAdder to this would need to be severed for that to happen. adders = new CSVLoaderBase.FieldAdder[fieldnames.length]; String skipStr = params.get(SKIP); List<String> skipFields = skipStr == null ? null : StrUtils.splitSmart(skipStr, ','); CSVLoaderBase.FieldAdder adder = new CSVLoaderBase.FieldAdder(); CSVLoaderBase.FieldAdder adderKeepEmpty = new CSVLoaderBase.FieldAdderEmpty(); for (int i = 0; i < fieldnames.length; i++) { String fname = fieldnames[i]; // to skip a field, leave the entries in fields and addrs null if (fname.length() == 0 || (skipFields != null && skipFields.contains(fname))) continue; boolean keepEmpty = params.getFieldBool(fname, EMPTY, false); adders[i] = keepEmpty ? adderKeepEmpty : adder; // Order that operations are applied: split -> trim -> map -> add // so create in reverse order. // Creation of FieldAdders could be optimized and shared among fields String[] fmap = params.getFieldParams(fname, MAP); if (fmap != null) { for (String mapRule : fmap) { String[] mapArgs = colonSplit.split(mapRule, -1); if (mapArgs.length != 2) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Map rules must be of the form 'from:to' ,got '" + mapRule + "'"); adders[i] = new CSVLoaderBase.FieldMapperSingle(mapArgs[0], mapArgs[1], adders[i]); } } if (params.getFieldBool(fname, TRIM, false)) { adders[i] = new CSVLoaderBase.FieldTrimmer(adders[i]); } if (params.getFieldBool(fname, SPLIT, false)) { String sepStr = params.getFieldParam(fname, SEPARATOR); char fsep = sepStr == null || sepStr.length() == 0 ? ',' : sepStr.charAt(0); String encStr = params.getFieldParam(fname, ENCAPSULATOR); char fenc = encStr == null || encStr.length() == 0 ? (char) -2 : encStr.charAt(0); String escStr = params.getFieldParam(fname, ESCAPE); char fesc = escStr == null || escStr.length() == 0 ? CSVStrategy.ESCAPE_DISABLED : escStr.charAt(0); CSVStrategy fstrat = new CSVStrategy( fsep, fenc, CSVStrategy.COMMENTS_DISABLED, fesc, false, false, false, false); adders[i] = new CSVLoaderBase.FieldSplitter(fstrat, adders[i]); } } // look for any literal fields - literal.foo=xyzzy Iterator<String> paramNames = params.getParameterNamesIterator(); while (paramNames.hasNext()) { String pname = paramNames.next(); if (!pname.startsWith(LITERALS_PREFIX)) continue; String name = pname.substring(LITERALS_PREFIX.length()); literals.put(name, params.get(pname)); } }
public NamedList<Object> request(final SolrRequest request, ResponseParser processor) throws SolrServerException, IOException { HttpMethod method = null; InputStream is = null; SolrParams params = request.getParams(); Collection<ContentStream> streams = requestWriter.getContentStreams(request); String path = requestWriter.getPath(request); if (path == null || !path.startsWith("/")) { path = "/select"; } ResponseParser parser = request.getResponseParser(); if (parser == null) { parser = _parser; } // The parser 'wt=' and 'version=' params are used instead of the original params ModifiableSolrParams wparams = new ModifiableSolrParams(); wparams.set(CommonParams.WT, parser.getWriterType()); wparams.set(CommonParams.VERSION, parser.getVersion()); if (params == null) { params = wparams; } else { params = new DefaultSolrParams(wparams, params); } if (_invariantParams != null) { params = new DefaultSolrParams(_invariantParams, params); } int tries = _maxRetries + 1; try { while (tries-- > 0) { // Note: since we aren't do intermittent time keeping // ourselves, the potential non-timeout latency could be as // much as tries-times (plus scheduling effects) the given // timeAllowed. try { if (SolrRequest.METHOD.GET == request.getMethod()) { if (streams != null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!"); } method = new GetMethod(_baseURL + path + ClientUtils.toQueryString(params, false)); } else if (SolrRequest.METHOD.POST == request.getMethod()) { String url = _baseURL + path; boolean isMultipart = (streams != null && streams.size() > 1); if (streams == null || isMultipart) { PostMethod post = new PostMethod(url); post.getParams().setContentCharset("UTF-8"); if (!this.useMultiPartPost && !isMultipart) { post.addRequestHeader( "Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); } List<Part> parts = new LinkedList<Part>(); Iterator<String> iter = params.getParameterNamesIterator(); while (iter.hasNext()) { String p = iter.next(); String[] vals = params.getParams(p); if (vals != null) { for (String v : vals) { if (this.useMultiPartPost || isMultipart) { parts.add(new StringPart(p, v, "UTF-8")); } else { post.addParameter(p, v); } } } } if (isMultipart) { int i = 0; for (ContentStream content : streams) { final ContentStream c = content; String charSet = null; String transferEncoding = null; parts.add( new PartBase(c.getName(), c.getContentType(), charSet, transferEncoding) { @Override protected long lengthOfData() throws IOException { return c.getSize(); } @Override protected void sendData(OutputStream out) throws IOException { InputStream in = c.getStream(); try { IOUtils.copy(in, out); } finally { in.close(); } } }); } } if (parts.size() > 0) { post.setRequestEntity( new MultipartRequestEntity( parts.toArray(new Part[parts.size()]), post.getParams())); } method = post; } // It is has one stream, it is the post body, put the params in the URL else { String pstr = ClientUtils.toQueryString(params, false); PostMethod post = new PostMethod(url + pstr); // Single stream as body // Using a loop just to get the first one final ContentStream[] contentStream = new ContentStream[1]; for (ContentStream content : streams) { contentStream[0] = content; break; } if (contentStream[0] instanceof RequestWriter.LazyContentStream) { post.setRequestEntity( new RequestEntity() { public long getContentLength() { return -1; } public String getContentType() { return contentStream[0].getContentType(); } public boolean isRepeatable() { return false; } public void writeRequest(OutputStream outputStream) throws IOException { ((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream); } }); } else { is = contentStream[0].getStream(); post.setRequestEntity( new InputStreamRequestEntity(is, contentStream[0].getContentType())); } method = post; } } else { throw new SolrServerException("Unsupported method: " + request.getMethod()); } } catch (NoHttpResponseException r) { // This is generally safe to retry on method.releaseConnection(); method = null; if (is != null) { is.close(); } // If out of tries then just rethrow (as normal error). if ((tries < 1)) { throw r; } // log.warn( "Caught: " + r + ". Retrying..." ); } } } catch (IOException ex) { throw new SolrServerException("error reading streams", ex); } method.setFollowRedirects(_followRedirects); method.addRequestHeader("User-Agent", AGENT); if (_allowCompression) { method.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate")); } try { // Execute the method. // System.out.println( "EXECUTE:"+method.getURI() ); int statusCode = _httpClient.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { StringBuilder msg = new StringBuilder(); msg.append(method.getStatusLine().getReasonPhrase()); msg.append("\n\n"); msg.append(method.getStatusText()); msg.append("\n\n"); msg.append("request: " + method.getURI()); throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8")); } // Read the contents String charset = "UTF-8"; if (method instanceof HttpMethodBase) { charset = ((HttpMethodBase) method).getResponseCharSet(); } InputStream respBody = method.getResponseBodyAsStream(); // Jakarta Commons HTTPClient doesn't handle any // compression natively. Handle gzip or deflate // here if applicable. if (_allowCompression) { Header contentEncodingHeader = method.getResponseHeader("Content-Encoding"); if (contentEncodingHeader != null) { String contentEncoding = contentEncodingHeader.getValue(); if (contentEncoding.contains("gzip")) { // log.debug( "wrapping response in GZIPInputStream" ); respBody = new GZIPInputStream(respBody); } else if (contentEncoding.contains("deflate")) { // log.debug( "wrapping response in InflaterInputStream" ); respBody = new InflaterInputStream(respBody); } } else { Header contentTypeHeader = method.getResponseHeader("Content-Type"); if (contentTypeHeader != null) { String contentType = contentTypeHeader.getValue(); if (contentType != null) { if (contentType.startsWith("application/x-gzip-compressed")) { // log.debug( "wrapping response in GZIPInputStream" ); respBody = new GZIPInputStream(respBody); } else if (contentType.startsWith("application/x-deflate")) { // log.debug( "wrapping response in InflaterInputStream" ); respBody = new InflaterInputStream(respBody); } } } } } return processor.processResponse(respBody, charset); } catch (HttpException e) { throw new SolrServerException(e); } catch (IOException e) { throw new SolrServerException(e); } finally { method.releaseConnection(); if (is != null) { is.close(); } } }