Example #1
0
  private void setupHttpClientForAuthPlugin(Object authcPlugin) {
    if (authcPlugin instanceof HttpClientBuilderPlugin) {
      // Setup HttpClient for internode communication
      SolrHttpClientBuilder builder =
          ((HttpClientBuilderPlugin) authcPlugin)
              .getHttpClientBuilder(HttpClientUtil.getHttpClientBuilder());

      // The default http client of the core container's shardHandlerFactory has already been
      // created and
      // configured using the default httpclient configurer. We need to reconfigure it using the
      // plugin's
      // http client configurer to set it up for internode communication.
      log.info("Reconfiguring HttpClient settings.");

      SolrHttpClientContextBuilder httpClientBuilder = new SolrHttpClientContextBuilder();
      if (builder.getCredentialsProviderProvider() != null) {
        httpClientBuilder.setDefaultCredentialsProvider(
            new CredentialsProviderProvider() {

              @Override
              public CredentialsProvider getCredentialsProvider() {
                return builder.getCredentialsProviderProvider().getCredentialsProvider();
              }
            });
      }
      if (builder.getAuthSchemeRegistryProvider() != null) {
        httpClientBuilder.setAuthSchemeRegistryProvider(
            new AuthSchemeRegistryProvider() {

              @Override
              public Lookup<AuthSchemeProvider> getAuthSchemeRegistry() {
                return builder.getAuthSchemeRegistryProvider().getAuthSchemeRegistry();
              }
            });
      }

      HttpClientUtil.setHttpClientRequestContextBuilder(httpClientBuilder);

    } else {
      if (pkiAuthenticationPlugin != null) {
        // this happened due to an authc plugin reload. no need to register the pkiAuthc plugin
        // again
        if (pkiAuthenticationPlugin.isInterceptorRegistered()) return;
        log.info("PKIAuthenticationPlugin is managing internode requests");
        setupHttpClientForAuthPlugin(pkiAuthenticationPlugin);
        pkiAuthenticationPlugin.setInterceptorRegistered();
      }
    }
  }
 public FullThrottleStopableIndexingThread(
     List<SolrServer> clients, String id, boolean doDeletes) {
   super(id, doDeletes);
   setName("FullThrottleStopableIndexingThread");
   setDaemon(true);
   this.clients = clients;
   HttpClientUtil.setConnectionTimeout(httpClient, 15000);
   HttpClientUtil.setSoTimeout(httpClient, 15000);
   suss =
       new ConcurrentUpdateSolrServer(
           ((HttpSolrServer) clients.get(0)).getBaseURL(), httpClient, 8, 2) {
         @Override
         public void handleError(Throwable ex) {
           log.warn("suss error", ex);
         }
       };
 }
Example #3
0
  public HttpSolrServer createSolrFileStoreClient() throws PropertyNotDefinedException {
    HttpSolrServer server = null;

    server =
        new HttpSolrServer(
            AndrophsyProperties.getInstance().getProperty(AndrophsyConstants.SOLR_FILESTORE_URL));
    HttpClientUtil.setBasicAuth(
        (DefaultHttpClient) server.getHttpClient(),
        AndrophsyProperties.getInstance().getProperty(AndrophsyConstants.SOLR_USERNAME),
        AndrophsyProperties.getInstance().getProperty(AndrophsyConstants.SOLR_PASSWORD));

    return server;
  }
  public FusionPipelineClient(
      String endpointUrl, String fusionUser, String fusionPass, String fusionRealm)
      throws MalformedURLException {

    globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BEST_MATCH).build();
    cookieStore = new BasicCookieStore();

    this.fusionUser = fusionUser;
    this.fusionPass = fusionPass;
    this.fusionRealm = fusionRealm;

    // build the HttpClient to be used for all requests
    HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
    httpClientBuilder.setDefaultRequestConfig(globalConfig).setDefaultCookieStore(cookieStore);

    if (fusionUser != null && fusionRealm == null)
      httpClientBuilder.addInterceptorFirst(
          new PreEmptiveBasicAuthenticator(fusionUser, fusionPass));

    httpClient = httpClientBuilder.build();
    HttpClientUtil.setMaxConnections(httpClient, 500);
    HttpClientUtil.setMaxConnectionsPerHost(httpClient, 100);

    originalEndpoints = Arrays.asList(endpointUrl.split(","));
    try {
      sessions = establishSessions(originalEndpoints, fusionUser, fusionPass, fusionRealm);
    } catch (Exception exc) {
      if (exc instanceof RuntimeException) {
        throw (RuntimeException) exc;
      } else {
        throw new RuntimeException(exc);
      }
    }

    random = new Random();
    jsonObjectMapper = new ObjectMapper();

    requestCounter = new AtomicInteger(0);
  }
Example #5
0
  private void remoteQuery(String coreUrl, HttpServletResponse resp) throws IOException {
    HttpRequestBase method = null;
    HttpEntity httpEntity = null;
    try {
      String urlstr = coreUrl + queryParams.toQueryString();

      boolean isPostOrPutRequest = "POST".equals(req.getMethod()) || "PUT".equals(req.getMethod());
      if ("GET".equals(req.getMethod())) {
        method = new HttpGet(urlstr);
      } else if ("HEAD".equals(req.getMethod())) {
        method = new HttpHead(urlstr);
      } else if (isPostOrPutRequest) {
        HttpEntityEnclosingRequestBase entityRequest =
            "POST".equals(req.getMethod()) ? new HttpPost(urlstr) : new HttpPut(urlstr);
        InputStream in =
            new CloseShieldInputStream(req.getInputStream()); // Prevent close of container streams
        HttpEntity entity = new InputStreamEntity(in, req.getContentLength());
        entityRequest.setEntity(entity);
        method = entityRequest;
      } else if ("DELETE".equals(req.getMethod())) {
        method = new HttpDelete(urlstr);
      } else {
        throw new SolrException(
            SolrException.ErrorCode.SERVER_ERROR, "Unexpected method type: " + req.getMethod());
      }

      for (Enumeration<String> e = req.getHeaderNames(); e.hasMoreElements(); ) {
        String headerName = e.nextElement();
        if (!"host".equalsIgnoreCase(headerName)
            && !"authorization".equalsIgnoreCase(headerName)
            && !"accept".equalsIgnoreCase(headerName)) {
          method.addHeader(headerName, req.getHeader(headerName));
        }
      }
      // These headers not supported for HttpEntityEnclosingRequests
      if (method instanceof HttpEntityEnclosingRequest) {
        method.removeHeaders(TRANSFER_ENCODING_HEADER);
        method.removeHeaders(CONTENT_LENGTH_HEADER);
      }

      final HttpResponse response =
          solrDispatchFilter.httpClient.execute(
              method, HttpClientUtil.createNewHttpClientRequestContext());
      int httpStatus = response.getStatusLine().getStatusCode();
      httpEntity = response.getEntity();

      resp.setStatus(httpStatus);
      for (HeaderIterator responseHeaders = response.headerIterator();
          responseHeaders.hasNext(); ) {
        Header header = responseHeaders.nextHeader();

        // We pull out these two headers below because they can cause chunked
        // encoding issues with Tomcat
        if (header != null
            && !header.getName().equalsIgnoreCase(TRANSFER_ENCODING_HEADER)
            && !header.getName().equalsIgnoreCase(CONNECTION_HEADER)) {
          resp.addHeader(header.getName(), header.getValue());
        }
      }

      if (httpEntity != null) {
        if (httpEntity.getContentEncoding() != null)
          resp.setCharacterEncoding(httpEntity.getContentEncoding().getValue());
        if (httpEntity.getContentType() != null)
          resp.setContentType(httpEntity.getContentType().getValue());

        InputStream is = httpEntity.getContent();
        OutputStream os = resp.getOutputStream();

        IOUtils.copyLarge(is, os);
      }

    } catch (IOException e) {
      sendError(
          new SolrException(
              SolrException.ErrorCode.SERVER_ERROR,
              "Error trying to proxy request for url: " + coreUrl,
              e));
    } finally {
      Utils.consumeFully(httpEntity);
    }
  }
  class FullThrottleStopableIndexingThread extends StopableIndexingThread {
    private HttpClient httpClient = HttpClientUtil.createClient(null);
    private volatile boolean stop = false;
    int clientIndex = 0;
    private ConcurrentUpdateSolrServer suss;
    private List<SolrServer> clients;
    private AtomicInteger fails = new AtomicInteger();

    public FullThrottleStopableIndexingThread(
        List<SolrServer> clients, String id, boolean doDeletes) {
      super(id, doDeletes);
      setName("FullThrottleStopableIndexingThread");
      setDaemon(true);
      this.clients = clients;
      HttpClientUtil.setConnectionTimeout(httpClient, 15000);
      HttpClientUtil.setSoTimeout(httpClient, 15000);
      suss =
          new ConcurrentUpdateSolrServer(
              ((HttpSolrServer) clients.get(0)).getBaseURL(), httpClient, 8, 2) {
            @Override
            public void handleError(Throwable ex) {
              log.warn("suss error", ex);
            }
          };
    }

    @Override
    public void run() {
      int i = 0;
      int numDeletes = 0;
      int numAdds = 0;

      while (true && !stop) {
        String id = this.id + "-" + i;
        ++i;

        if (doDeletes && random().nextBoolean() && deletes.size() > 0) {
          String delete = deletes.remove(0);
          try {
            numDeletes++;
            suss.deleteById(delete);
          } catch (Exception e) {
            changeUrlOnError(e);
            // System.err.println("REQUEST FAILED:");
            // e.printStackTrace();
            fails.incrementAndGet();
          }
        }

        try {
          numAdds++;
          if (numAdds > 4000) continue;
          SolrInputDocument doc =
              getDoc(
                  "id",
                  id,
                  i1,
                  50,
                  t1,
                  "Saxon heptarchies that used to rip around so in old times and raise Cain.  My, you ought to seen old Henry the Eight when he was in bloom.  He WAS a blossom.  He used to marry a new wife every day, and chop off her head next morning.  And he would do it just as indifferent as if ");
          suss.add(doc);
        } catch (Exception e) {
          changeUrlOnError(e);
          // System.err.println("REQUEST FAILED:");
          // e.printStackTrace();
          fails.incrementAndGet();
        }

        if (doDeletes && random().nextBoolean()) {
          deletes.add(id);
        }
      }

      System.err.println(
          "FT added docs:" + numAdds + " with " + fails + " fails" + " deletes:" + numDeletes);
    }

    private void changeUrlOnError(Exception e) {
      if (e instanceof ConnectException) {
        clientIndex++;
        if (clientIndex > clients.size() - 1) {
          clientIndex = 0;
        }
        suss.shutdownNow();
        suss =
            new ConcurrentUpdateSolrServer(
                ((HttpSolrServer) clients.get(clientIndex)).getBaseURL(), httpClient, 30, 3) {
              @Override
              public void handleError(Throwable ex) {
                log.warn("suss error", ex);
              }
            };
      }
    }

    @Override
    public void safeStop() {
      stop = true;
      suss.blockUntilFinished();
      suss.shutdownNow();
      httpClient.getConnectionManager().shutdown();
    }

    @Override
    public int getFailCount() {
      return fails.get();
    }

    @Override
    public Set<String> getAddFails() {
      throw new UnsupportedOperationException();
    }

    @Override
    public Set<String> getDeleteFails() {
      throw new UnsupportedOperationException();
    }
  };