Example #1
0
 private void extractRemotePath(String corename, String origCorename, int idx)
     throws UnsupportedEncodingException, KeeperException, InterruptedException {
   if (core == null && idx > 0) {
     coreUrl = getRemotCoreUrl(corename, origCorename);
     // don't proxy for internal update requests
     invalidStates = checkStateIsValid(queryParams.get(CloudSolrClient.STATE_VERSION));
     if (coreUrl != null
         && queryParams.get(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM) == null) {
       path = path.substring(idx);
       if (invalidStates != null) {
         // it does not make sense to send the request to a remote node
         throw new SolrException(
             SolrException.ErrorCode.INVALID_STATE,
             new String(Utils.toJSON(invalidStates), org.apache.lucene.util.IOUtils.UTF_8));
       }
       action = REMOTEQUERY;
     } else {
       if (!retry) {
         // we couldn't find a core to work with, try reloading aliases
         // TODO: it would be nice if admin ui elements skipped this...
         ZkStateReader reader = cores.getZkController().getZkStateReader();
         reader.updateAliases();
         action = RETRY;
       }
     }
   }
 }
Example #2
0
  private synchronized void initializeAuthorizationPlugin(Map<String, Object> authorizationConf) {
    authorizationConf = Utils.getDeepCopy(authorizationConf, 4);
    // Initialize the Authorization module
    SecurityPluginHolder<AuthorizationPlugin> old = authorizationPlugin;
    SecurityPluginHolder<AuthorizationPlugin> authorizationPlugin = null;
    if (authorizationConf != null) {
      String klas = (String) authorizationConf.get("class");
      if (klas == null) {
        throw new SolrException(
            ErrorCode.SERVER_ERROR, "class is required for authorization plugin");
      }
      if (old != null && old.getZnodeVersion() == readVersion(authorizationConf)) {
        return;
      }
      log.info("Initializing authorization plugin: " + klas);
      authorizationPlugin =
          new SecurityPluginHolder<>(
              readVersion(authorizationConf),
              getResourceLoader().newInstance(klas, AuthorizationPlugin.class));

      // Read and pass the authorization context to the plugin
      authorizationPlugin.plugin.init(authorizationConf);
    } else {
      log.info("Security conf doesn't exist. Skipping setup for authorization module.");
    }
    this.authorizationPlugin = authorizationPlugin;
    if (old != null) {
      try {
        old.plugin.close();
      } catch (Exception e) {
      }
    }
  }
Example #3
0
  private synchronized void initializeAuthenticationPlugin(
      Map<String, Object> authenticationConfig) {
    authenticationConfig = Utils.getDeepCopy(authenticationConfig, 4);
    String pluginClassName = null;
    if (authenticationConfig != null) {
      if (authenticationConfig.containsKey("class")) {
        pluginClassName = String.valueOf(authenticationConfig.get("class"));
      } else {
        throw new SolrException(
            ErrorCode.SERVER_ERROR, "No 'class' specified for authentication in ZK.");
      }
    }

    if (pluginClassName != null) {
      log.info("Authentication plugin class obtained from ZK: " + pluginClassName);
    } else if (System.getProperty(AUTHENTICATION_PLUGIN_PROP) != null) {
      pluginClassName = System.getProperty(AUTHENTICATION_PLUGIN_PROP);
      log.info(
          "Authentication plugin class obtained from system property '"
              + AUTHENTICATION_PLUGIN_PROP
              + "': "
              + pluginClassName);
    } else {
      log.info("No authentication plugin used.");
    }
    SecurityPluginHolder<AuthenticationPlugin> old = authenticationPlugin;
    SecurityPluginHolder<AuthenticationPlugin> authenticationPlugin = null;

    // Initialize the plugin
    if (pluginClassName != null) {
      authenticationPlugin =
          new SecurityPluginHolder<>(
              readVersion(authenticationConfig),
              getResourceLoader()
                  .newInstance(
                      pluginClassName,
                      AuthenticationPlugin.class,
                      null,
                      new Class[] {CoreContainer.class},
                      new Object[] {this}));
    }
    if (authenticationPlugin != null) {
      authenticationPlugin.plugin.init(authenticationConfig);
      setupHttpClientForAuthPlugin(authenticationPlugin.plugin);
    }
    this.authenticationPlugin = authenticationPlugin;
    try {
      if (old != null) old.plugin.close();
    } catch (Exception e) {
      /*do nothing*/
    }
  }
  public static Map testForResponseElement(
      RestTestHarness harness,
      String testServerBaseUrl,
      String uri,
      CloudSolrClient cloudSolrClient,
      List<String> jsonPath,
      Object expected,
      long maxTimeoutSeconds)
      throws Exception {

    boolean success = false;
    long startTime = System.nanoTime();
    Map m = null;

    while (TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS)
        < maxTimeoutSeconds) {
      try {
        m =
            testServerBaseUrl == null
                ? getRespMap(uri, harness)
                : TestSolrConfigHandlerConcurrent.getAsMap(
                    testServerBaseUrl + uri, cloudSolrClient);
      } catch (Exception e) {
        Thread.sleep(100);
        continue;
      }
      if (Objects.equals(expected, Utils.getObjectByPath(m, false, jsonPath))) {
        success = true;
        break;
      }
      Thread.sleep(100);
    }
    assertTrue(
        StrUtils.formatString(
            "Could not get expected value  ''{0}'' for path ''{1}'' full output: {2},  from server:  {3}",
            expected, StrUtils.join(jsonPath, '/'), getAsString(m), testServerBaseUrl),
        success);

    return m;
  }
Example #5
0
  private void remoteQuery(String coreUrl, HttpServletResponse resp) throws IOException {
    HttpRequestBase method = null;
    HttpEntity httpEntity = null;
    try {
      String urlstr = coreUrl + queryParams.toQueryString();

      boolean isPostOrPutRequest = "POST".equals(req.getMethod()) || "PUT".equals(req.getMethod());
      if ("GET".equals(req.getMethod())) {
        method = new HttpGet(urlstr);
      } else if ("HEAD".equals(req.getMethod())) {
        method = new HttpHead(urlstr);
      } else if (isPostOrPutRequest) {
        HttpEntityEnclosingRequestBase entityRequest =
            "POST".equals(req.getMethod()) ? new HttpPost(urlstr) : new HttpPut(urlstr);
        InputStream in =
            new CloseShieldInputStream(req.getInputStream()); // Prevent close of container streams
        HttpEntity entity = new InputStreamEntity(in, req.getContentLength());
        entityRequest.setEntity(entity);
        method = entityRequest;
      } else if ("DELETE".equals(req.getMethod())) {
        method = new HttpDelete(urlstr);
      } else {
        throw new SolrException(
            SolrException.ErrorCode.SERVER_ERROR, "Unexpected method type: " + req.getMethod());
      }

      for (Enumeration<String> e = req.getHeaderNames(); e.hasMoreElements(); ) {
        String headerName = e.nextElement();
        if (!"host".equalsIgnoreCase(headerName)
            && !"authorization".equalsIgnoreCase(headerName)
            && !"accept".equalsIgnoreCase(headerName)) {
          method.addHeader(headerName, req.getHeader(headerName));
        }
      }
      // These headers not supported for HttpEntityEnclosingRequests
      if (method instanceof HttpEntityEnclosingRequest) {
        method.removeHeaders(TRANSFER_ENCODING_HEADER);
        method.removeHeaders(CONTENT_LENGTH_HEADER);
      }

      final HttpResponse response =
          solrDispatchFilter.httpClient.execute(
              method, HttpClientUtil.createNewHttpClientRequestContext());
      int httpStatus = response.getStatusLine().getStatusCode();
      httpEntity = response.getEntity();

      resp.setStatus(httpStatus);
      for (HeaderIterator responseHeaders = response.headerIterator();
          responseHeaders.hasNext(); ) {
        Header header = responseHeaders.nextHeader();

        // We pull out these two headers below because they can cause chunked
        // encoding issues with Tomcat
        if (header != null
            && !header.getName().equalsIgnoreCase(TRANSFER_ENCODING_HEADER)
            && !header.getName().equalsIgnoreCase(CONNECTION_HEADER)) {
          resp.addHeader(header.getName(), header.getValue());
        }
      }

      if (httpEntity != null) {
        if (httpEntity.getContentEncoding() != null)
          resp.setCharacterEncoding(httpEntity.getContentEncoding().getValue());
        if (httpEntity.getContentType() != null)
          resp.setContentType(httpEntity.getContentType().getValue());

        InputStream is = httpEntity.getContent();
        OutputStream os = resp.getOutputStream();

        IOUtils.copyLarge(is, os);
      }

    } catch (IOException e) {
      sendError(
          new SolrException(
              SolrException.ErrorCode.SERVER_ERROR,
              "Error trying to proxy request for url: " + coreUrl,
              e));
    } finally {
      Utils.consumeFully(httpEntity);
    }
  }
  @Test
  public void testStoreAndRead() throws Exception {
    Map<String, DocCollection> collectionStates = new HashMap<>();
    Set<String> liveNodes = new HashSet<>();
    liveNodes.add("node1");
    liveNodes.add("node2");

    Map<String, Slice> slices = new HashMap<>();
    Map<String, Replica> sliceToProps = new HashMap<>();
    Map<String, Object> props = new HashMap<>();

    props.put("prop1", "value");
    props.put("prop2", "value2");
    Replica replica = new Replica("node1", props);
    sliceToProps.put("node1", replica);
    Slice slice = new Slice("shard1", sliceToProps, null);
    slices.put("shard1", slice);
    Slice slice2 = new Slice("shard2", sliceToProps, null);
    slices.put("shard2", slice2);
    collectionStates.put(
        "collection1", new DocCollection("collection1", slices, null, DocRouter.DEFAULT));
    collectionStates.put(
        "collection2", new DocCollection("collection2", slices, null, DocRouter.DEFAULT));
    ZkStateReader zkStateReaderMock = getMockZkStateReader(collectionStates.keySet());

    ClusterState clusterState = new ClusterState(-1, liveNodes, collectionStates);
    byte[] bytes = Utils.toJSON(clusterState);
    // System.out.println("#################### " + new String(bytes));
    ClusterState loadedClusterState = ClusterState.load(-1, bytes, liveNodes);

    assertEquals(
        "Provided liveNodes not used properly", 2, loadedClusterState.getLiveNodes().size());
    assertEquals("No collections found", 2, loadedClusterState.getCollectionsMap().size());
    assertEquals(
        "Properties not copied properly",
        replica.getStr("prop1"),
        loadedClusterState
            .getSlice("collection1", "shard1")
            .getReplicasMap()
            .get("node1")
            .getStr("prop1"));
    assertEquals(
        "Properties not copied properly",
        replica.getStr("prop2"),
        loadedClusterState
            .getSlice("collection1", "shard1")
            .getReplicasMap()
            .get("node1")
            .getStr("prop2"));

    loadedClusterState = ClusterState.load(-1, new byte[0], liveNodes);

    assertEquals(
        "Provided liveNodes not used properly", 2, loadedClusterState.getLiveNodes().size());
    assertEquals("Should not have collections", 0, loadedClusterState.getCollectionsMap().size());

    loadedClusterState = ClusterState.load(-1, (byte[]) null, liveNodes);

    assertEquals(
        "Provided liveNodes not used properly", 2, loadedClusterState.getLiveNodes().size());
    assertEquals("Should not have collections", 0, loadedClusterState.getCollectionsMap().size());
  }
  public static void reqhandlertests(
      RestTestHarness writeHarness, String testServerBaseUrl, CloudSolrClient cloudSolrClient)
      throws Exception {
    String payload =
        "{\n"
            + "'create-requesthandler' : { 'name' : '/x', 'class': 'org.apache.solr.handler.DumpRequestHandler' , 'startup' : 'lazy'}\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);

    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config/overlay?wt=json",
        cloudSolrClient,
        Arrays.asList("overlay", "requestHandler", "/x", "startup"),
        "lazy",
        10);

    payload =
        "{\n"
            + "'update-requesthandler' : { 'name' : '/x', 'class': 'org.apache.solr.handler.DumpRequestHandler' , 'startup' : 'lazy' , 'a':'b' , 'defaults': {'def_a':'def A val', 'multival':['a','b','c']}}\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);

    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config/overlay?wt=json",
        cloudSolrClient,
        Arrays.asList("overlay", "requestHandler", "/x", "a"),
        "b",
        10);

    payload =
        "{\n"
            + "'update-requesthandler' : { 'name' : '/dump', "
            + "'initParams': 'a',"
            + "'class': 'org.apache.solr.handler.DumpRequestHandler' ,"
            + " 'defaults': {'a':'A','b':'B','c':'C'}}\n"
            + "}";

    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config/overlay?wt=json",
        cloudSolrClient,
        Arrays.asList("overlay", "requestHandler", "/dump", "defaults", "c"),
        "C",
        10);

    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/x?wt=json&getdefaults=true&json.nl=map",
        cloudSolrClient,
        Arrays.asList("getdefaults", "def_a"),
        "def A val",
        10);

    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/x?wt=json&param=multival&json.nl=map",
        cloudSolrClient,
        Arrays.asList("params", "multival"),
        Arrays.asList("a", "b", "c"),
        10);

    payload = "{\n" + "'delete-requesthandler' : '/x'" + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    boolean success = false;
    long startTime = System.nanoTime();
    int maxTimeoutSeconds = 10;
    while (TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS)
        < maxTimeoutSeconds) {
      String uri = "/config/overlay?wt=json";
      Map m =
          testServerBaseUrl == null
              ? getRespMap(uri, writeHarness)
              : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl + uri, cloudSolrClient);
      if (null
          == Utils.getObjectByPath(
              m, true, Arrays.asList("overlay", "requestHandler", "/x", "a"))) {
        success = true;
        break;
      }
      Thread.sleep(100);
    }
    assertTrue("Could not delete requestHandler  ", success);

    payload =
        "{\n"
            + "'create-queryconverter' : { 'name' : 'qc', 'class': 'org.apache.solr.spelling.SpellingQueryConverter'}\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config?wt=json",
        cloudSolrClient,
        Arrays.asList("config", "queryConverter", "qc", "class"),
        "org.apache.solr.spelling.SpellingQueryConverter",
        10);
    payload =
        "{\n"
            + "'update-queryconverter' : { 'name' : 'qc', 'class': 'org.apache.solr.spelling.SuggestQueryConverter'}\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config?wt=json",
        cloudSolrClient,
        Arrays.asList("config", "queryConverter", "qc", "class"),
        "org.apache.solr.spelling.SuggestQueryConverter",
        10);

    payload = "{\n" + "'delete-queryconverter' : 'qc'" + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config?wt=json",
        cloudSolrClient,
        Arrays.asList("config", "queryConverter", "qc"),
        null,
        10);

    payload =
        "{\n"
            + "'create-searchcomponent' : { 'name' : 'tc', 'class': 'org.apache.solr.handler.component.TermsComponent'}\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config?wt=json",
        cloudSolrClient,
        Arrays.asList("config", "searchComponent", "tc", "class"),
        "org.apache.solr.handler.component.TermsComponent",
        10);
    payload =
        "{\n"
            + "'update-searchcomponent' : { 'name' : 'tc', 'class': 'org.apache.solr.handler.component.TermVectorComponent' }\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config?wt=json",
        cloudSolrClient,
        Arrays.asList("config", "searchComponent", "tc", "class"),
        "org.apache.solr.handler.component.TermVectorComponent",
        10);

    payload = "{\n" + "'delete-searchcomponent' : 'tc'" + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config?wt=json",
        cloudSolrClient,
        Arrays.asList("config", "searchComponent", "tc"),
        null,
        10);
    // <valueSourceParser name="countUsage"
    // class="org.apache.solr.core.CountUsageValueSourceParser"/>
    payload =
        "{\n"
            + "'create-valuesourceparser' : { 'name' : 'cu', 'class': 'org.apache.solr.core.CountUsageValueSourceParser'}\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config?wt=json",
        cloudSolrClient,
        Arrays.asList("config", "valueSourceParser", "cu", "class"),
        "org.apache.solr.core.CountUsageValueSourceParser",
        10);
    //  <valueSourceParser name="nvl" class="org.apache.solr.search.function.NvlValueSourceParser">
    //    <float name="nvlFloatValue">0.0</float>
    //    </valueSourceParser>
    payload =
        "{\n"
            + "'update-valuesourceparser' : { 'name' : 'cu', 'class': 'org.apache.solr.search.function.NvlValueSourceParser'}\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config?wt=json",
        cloudSolrClient,
        Arrays.asList("config", "valueSourceParser", "cu", "class"),
        "org.apache.solr.search.function.NvlValueSourceParser",
        10);

    payload = "{\n" + "'delete-valuesourceparser' : 'cu'" + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config?wt=json",
        cloudSolrClient,
        Arrays.asList("config", "valueSourceParser", "cu"),
        null,
        10);
    //    <transformer name="mytrans2"
    // class="org.apache.solr.response.transform.ValueAugmenterFactory" >
    //    <int name="value">5</int>
    //    </transformer>
    payload =
        "{\n"
            + "'create-transformer' : { 'name' : 'mytrans', 'class': 'org.apache.solr.response.transform.ValueAugmenterFactory', 'value':'5'}\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config?wt=json",
        cloudSolrClient,
        Arrays.asList("config", "transformer", "mytrans", "class"),
        "org.apache.solr.response.transform.ValueAugmenterFactory",
        10);

    payload =
        "{\n"
            + "'update-transformer' : { 'name' : 'mytrans', 'class': 'org.apache.solr.response.transform.ValueAugmenterFactory', 'value':'6'}\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    testForResponseElement(
        writeHarness,
        testServerBaseUrl,
        "/config?wt=json",
        cloudSolrClient,
        Arrays.asList("config", "transformer", "mytrans", "value"),
        "6",
        10);

    payload =
        "{\n"
            + "'delete-transformer' : 'mytrans',"
            + "'create-initparams' : { 'name' : 'hello', 'key':'val'}\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    Map map =
        testForResponseElement(
            writeHarness,
            testServerBaseUrl,
            "/config?wt=json",
            cloudSolrClient,
            Arrays.asList("config", "transformer", "mytrans"),
            null,
            10);

    List l = (List) Utils.getObjectByPath(map, false, Arrays.asList("config", "initParams"));
    assertNotNull("no object /config/initParams : " + TestBlobHandler.getAsString(map), l);
    assertEquals(1, l.size());
    assertEquals("val", ((Map) l.get(0)).get("key"));

    payload =
        "{\n"
            + "    'add-searchcomponent': {\n"
            + "        'name': 'myspellcheck',\n"
            + "        'class': 'solr.SpellCheckComponent',\n"
            + "        'queryAnalyzerFieldType': 'text_general',\n"
            + "        'spellchecker': {\n"
            + "            'name': 'default',\n"
            + "            'field': '_text_',\n"
            + "            'class': 'solr.DirectSolrSpellChecker'\n"
            + "        }\n"
            + "    }\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    map =
        testForResponseElement(
            writeHarness,
            testServerBaseUrl,
            "/config?wt=json",
            cloudSolrClient,
            Arrays.asList("config", "searchComponent", "myspellcheck", "spellchecker", "class"),
            "solr.DirectSolrSpellChecker",
            10);

    payload =
        "{\n"
            + "    'add-requesthandler': {\n"
            + "        name : '/dump100',\n"
            + "        class : 'org.apache.solr.handler.DumpRequestHandler',"
            + "        suggester: [{name: s1,lookupImpl: FuzzyLookupFactory, dictionaryImpl : DocumentDictionaryFactory},"
            + "                    {name: s2,lookupImpl: FuzzyLookupFactory , dictionaryImpl : DocumentExpressionDictionaryFactory}]"
            + "    }\n"
            + "}";
    runConfigCommand(writeHarness, "/config?wt=json", payload);
    map =
        testForResponseElement(
            writeHarness,
            testServerBaseUrl,
            "/config?wt=json",
            cloudSolrClient,
            Arrays.asList("config", "requestHandler", "/dump100", "class"),
            "org.apache.solr.handler.DumpRequestHandler",
            10);

    map = getRespMap("/dump100?wt=json&json.nl=arrmap&initArgs=true", writeHarness);
    List initArgs = (List) map.get("initArgs");
    assertEquals(2, initArgs.size());
    assertTrue(((Map) initArgs.get(0)).containsKey("suggester"));
    assertTrue(((Map) initArgs.get(1)).containsKey("suggester"));
    System.out.println(map);
  }