コード例 #1
0
ファイル: ResourceLoaderTest.java プロジェクト: bozydar/solr
  public void testBOMMarkers() throws Exception {
    final String fileWithBom = "stopwithbom.txt";
    SolrResourceLoader loader = new SolrResourceLoader("solr/collection1");

    // preliminary sanity check
    InputStream bomStream = loader.openResource(fileWithBom);
    try {
      final byte[] bomExpected = new byte[] {-17, -69, -65};
      final byte[] firstBytes = new byte[3];

      assertEquals(
          "Should have been able to read 3 bytes from bomStream", 3, bomStream.read(firstBytes));

      assertTrue(
          "This test only works if "
              + fileWithBom
              + " contains a BOM -- it appears someone removed it.",
          Arrays.equals(bomExpected, firstBytes));
    } finally {
      try {
        bomStream.close();
      } catch (Exception e) {
        /* IGNORE */
      }
    }

    // now make sure getLines skips the BOM...
    List<String> lines = loader.getLines(fileWithBom);
    assertEquals(1, lines.size());
    assertEquals("BOMsAreEvil", lines.get(0));
  }
コード例 #2
0
ファイル: ResourceLoaderTest.java プロジェクト: bozydar/solr
  public void testInstanceDir() throws Exception {
    SolrResourceLoader loader = new SolrResourceLoader(null);
    String instDir = loader.getInstanceDir();
    assertTrue(instDir + " is not equal to " + "solr/", instDir.equals("solr/") == true);

    loader = new SolrResourceLoader("solr");
    instDir = loader.getInstanceDir();
    assertTrue(
        instDir + " is not equal to " + "solr/", instDir.equals("solr" + File.separator) == true);
  }
コード例 #3
0
ファイル: Config.java プロジェクト: PATRIC3/p3_solr
  /**
   * Builds a config:
   *
   * <p>Note that the 'name' parameter is used to obtain a valid input stream if no valid one is
   * provided through 'is'. If no valid stream is provided, a valid SolrResourceLoader instance
   * should be provided through 'loader' so the resource can be opened (@see
   * SolrResourceLoader#openResource); if no SolrResourceLoader instance is provided, a default one
   * will be created.
   *
   * <p>Consider passing a non-null 'name' parameter in all use-cases since it is used for logging
   * &amp; exception reporting.
   *
   * @param loader the resource loader used to obtain an input stream if 'is' is null
   * @param name the resource name used if the input stream 'is' is null
   * @param is the resource as a SAX InputSource
   * @param prefix an optional prefix that will be preprended to all non-absolute xpath expressions
   */
  public Config(
      SolrResourceLoader loader,
      String name,
      InputSource is,
      String prefix,
      boolean substituteProps)
      throws ParserConfigurationException, IOException, SAXException {
    if (loader == null) {
      loader = new SolrResourceLoader(SolrResourceLoader.locateSolrHome());
    }
    this.loader = loader;
    this.name = name;
    this.prefix = (prefix != null && !prefix.endsWith("/")) ? prefix + '/' : prefix;
    try {
      javax.xml.parsers.DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();

      if (is == null) {
        InputStream in = loader.openConfig(name);
        if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {
          zkVersion = ((ZkSolrResourceLoader.ZkByteArrayInputStream) in).getStat().getVersion();
          log.info("loaded config {} with version {} ", name, zkVersion);
        }
        is = new InputSource(in);
        is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(name));
      }

      // only enable xinclude, if a SystemId is available
      if (is.getSystemId() != null) {
        try {
          dbf.setXIncludeAware(true);
          dbf.setNamespaceAware(true);
        } catch (UnsupportedOperationException e) {
          log.warn(name + " XML parser doesn't support XInclude option");
        }
      }

      final DocumentBuilder db = dbf.newDocumentBuilder();
      db.setEntityResolver(new SystemIdResolver(loader));
      db.setErrorHandler(xmllog);
      try {
        doc = db.parse(is);
        origDoc = copyDoc(doc);
      } finally {
        // some XML parsers are broken and don't close the byte stream (but they should according to
        // spec)
        IOUtils.closeQuietly(is.getByteStream());
      }
      if (substituteProps) {
        DOMUtil.substituteProperties(doc, getSubstituteProperties());
      }
    } catch (ParserConfigurationException | SAXException | TransformerException e) {
      SolrException.log(log, "Exception during parsing file: " + name, e);
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
    }
  }
コード例 #4
0
ファイル: ResourceLoaderTest.java プロジェクト: bozydar/solr
 public void testWrongEncoding() throws Exception {
   String wrongEncoding = "stopwordsWrongEncoding.txt";
   SolrResourceLoader loader = new SolrResourceLoader("solr/collection1");
   // ensure we get our exception
   try {
     List<String> lines = loader.getLines(wrongEncoding);
     fail();
   } catch (SolrException expected) {
     assertTrue(expected.getCause() instanceof CharacterCodingException);
   }
 }
コード例 #5
0
ファイル: SolrConfig.java プロジェクト: hunglsx/lucene-solr
  public static ConfigOverlay getConfigOverlay(SolrResourceLoader loader) {
    InputStream in = null;
    InputStreamReader isr = null;
    try {
      try {
        in = loader.openResource(ConfigOverlay.RESOURCE_NAME);
      } catch (IOException e) {
        // TODO: we should be explicitly looking for file not found exceptions
        // and logging if it's not the expected IOException
        // hopefully no problem, assume no overlay.json file
        return new ConfigOverlay(Collections.EMPTY_MAP, -1);
      }

      int version = 0; // will be always 0 for file based resourceLoader
      if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {
        version = ((ZkSolrResourceLoader.ZkByteArrayInputStream) in).getStat().getVersion();
        log.info("config overlay loaded . version : {} ", version);
      }
      isr = new InputStreamReader(in, StandardCharsets.UTF_8);
      Map m = (Map) ObjectBuilder.getVal(new JSONParser(isr));
      return new ConfigOverlay(m, version);
    } catch (Exception e) {
      throw new SolrException(ErrorCode.SERVER_ERROR, "Error reading config overlay", e);
    } finally {
      IOUtils.closeQuietly(isr);
      IOUtils.closeQuietly(in);
    }
  }
コード例 #6
0
  @Override
  public String getDataHome(CoreDescriptor cd) throws IOException {
    if (hdfsDataDir == null) {
      throw new SolrException(
          ErrorCode.SERVER_ERROR,
          "You must set the "
              + this.getClass().getSimpleName()
              + " param "
              + HDFS_HOME
              + " for relative dataDir paths to work");
    }

    // by default, we go off the instance directory
    String path;
    if (cd.getCloudDescriptor() != null) {
      path =
          URLEncoder.encode(cd.getCloudDescriptor().getCollectionName(), "UTF-8")
              + "/"
              + URLEncoder.encode(cd.getCloudDescriptor().getCoreNodeName(), "UTF-8");
    } else {
      path = cd.getName();
    }

    return normalize(
        SolrResourceLoader.normalizeDir(
            ZkController.trimLeadingAndTrailingSlashes(hdfsDataDir)
                + "/"
                + path
                + "/"
                + cd.getDataDir()));
  }
コード例 #7
0
 public CoreContainer(
     NodeConfig config, Properties properties, CoresLocator locator, boolean asyncSolrCoreLoad) {
   this.loader = config.getSolrResourceLoader();
   this.solrHome = loader.getInstancePath().toString();
   this.cfg = checkNotNull(config);
   this.coresLocator = locator;
   this.containerProperties = new Properties(properties);
   this.asyncSolrCoreLoad = asyncSolrCoreLoad;
 }
コード例 #8
0
ファイル: SolrConfig.java プロジェクト: hunglsx/lucene-solr
 public static SolrConfig readFromResourceLoader(SolrResourceLoader loader, String name) {
   try {
     return new SolrConfig(loader, name, null);
   } catch (Exception e) {
     String resource;
     if (loader instanceof ZkSolrResourceLoader) {
       resource = name;
     } else {
       resource = Paths.get(loader.getConfigDir()).resolve(name).toString();
     }
     throw new SolrException(
         ErrorCode.SERVER_ERROR, "Error loading solr config from " + resource, e);
   }
 }
コード例 #9
0
ファイル: SolrConfig.java プロジェクト: shaie/lucene-solr
  private void initLibs() {
    NodeList nodes = (NodeList) evaluate("lib", XPathConstants.NODESET);
    if (nodes == null || nodes.getLength() == 0) return;

    log.info("Adding specified lib dirs to ClassLoader");
    SolrResourceLoader loader = getResourceLoader();

    try {
      for (int i = 0; i < nodes.getLength(); i++) {
        Node node = nodes.item(i);

        String baseDir = DOMUtil.getAttr(node, "dir");
        String path = DOMUtil.getAttr(node, "path");
        if (null != baseDir) {
          // :TODO: add support for a simpler 'glob' mutually exclusive of regex
          String regex = DOMUtil.getAttr(node, "regex");
          FileFilter filter = (null == regex) ? null : new RegexFileFilter(regex);
          loader.addToClassLoader(baseDir, filter, false);
        } else if (null != path) {
          final File file = FileUtils.resolvePath(new File(loader.getInstanceDir()), path);
          loader.addToClassLoader(
              file.getParent(),
              new FileFilter() {
                @Override
                public boolean accept(File pathname) {
                  return pathname.equals(file);
                }
              },
              false);
        } else {
          throw new RuntimeException("lib: missing mandatory attributes: 'dir' or 'path'");
        }
      }
    } finally {
      loader.reloadLuceneSPI();
    }
  }
コード例 #10
0
ファイル: ResourceLoaderTest.java プロジェクト: bozydar/solr
  public void testAwareCompatibility() {
    SolrResourceLoader loader = new SolrResourceLoader(".");

    Class clazz = ResourceLoaderAware.class;
    // Check ResourceLoaderAware valid objects
    loader.assertAwareCompatibility(clazz, new NGramFilterFactory(new HashMap<String, String>()));
    loader.assertAwareCompatibility(
        clazz, new KeywordTokenizerFactory(new HashMap<String, String>()));

    // Make sure it throws an error for invalid objects
    Object[] invalid =
        new Object[] {
          // new NGramTokenFilter( null ),
          "hello", new Float(12.3f), new LukeRequestHandler(), new JSONResponseWriter()
        };
    for (Object obj : invalid) {
      try {
        loader.assertAwareCompatibility(clazz, obj);
        Assert.fail("Should be invalid class: " + obj + " FOR " + clazz);
      } catch (SolrException ex) {
      } // OK
    }

    clazz = SolrCoreAware.class;
    // Check ResourceLoaderAware valid objects
    loader.assertAwareCompatibility(clazz, new LukeRequestHandler());
    loader.assertAwareCompatibility(clazz, new FacetComponent());
    loader.assertAwareCompatibility(clazz, new JSONResponseWriter());

    // Make sure it throws an error for invalid objects
    invalid =
        new Object[] {
          new NGramFilterFactory(new HashMap<String, String>()),
          "hello",
          new Float(12.3f),
          new KeywordTokenizerFactory(new HashMap<String, String>())
        };
    for (Object obj : invalid) {
      try {
        loader.assertAwareCompatibility(clazz, obj);
        Assert.fail("Should be invalid class: " + obj + " FOR " + clazz);
      } catch (SolrException ex) {
      } // OK
    }
  }
コード例 #11
0
ファイル: SolrConfig.java プロジェクト: hunglsx/lucene-solr
  private void initLibs() {
    NodeList nodes = (NodeList) evaluate("lib", XPathConstants.NODESET);
    if (nodes == null || nodes.getLength() == 0) return;

    log.info("Adding specified lib dirs to ClassLoader");
    SolrResourceLoader loader = getResourceLoader();
    List<URL> urls = new ArrayList<>();

    for (int i = 0; i < nodes.getLength(); i++) {
      Node node = nodes.item(i);
      String baseDir = DOMUtil.getAttr(node, "dir");
      String path = DOMUtil.getAttr(node, PATH);
      if (null != baseDir) {
        // :TODO: add support for a simpler 'glob' mutually exclusive of regex
        Path dir = loader.getInstancePath().resolve(baseDir);
        String regex = DOMUtil.getAttr(node, "regex");
        try {
          if (regex == null) urls.addAll(SolrResourceLoader.getURLs(dir));
          else urls.addAll(SolrResourceLoader.getFilteredURLs(dir, regex));
        } catch (IOException e) {
          log.warn(
              "Couldn't add files from {} filtered by {} to classpath: {}",
              dir,
              regex,
              e.getMessage());
        }
      } else if (null != path) {
        final Path dir = loader.getInstancePath().resolve(path);
        try {
          urls.add(dir.toUri().toURL());
        } catch (MalformedURLException e) {
          log.warn("Couldn't add file {} to classpath: {}", dir, e.getMessage());
        }
      } else {
        throw new RuntimeException("lib: missing mandatory attributes: 'dir' or 'path'");
      }
    }

    if (urls.size() > 0) {
      loader.addToClassLoader(urls);
      loader.reloadLuceneSPI();
    }
  }
コード例 #12
0
  /** Load the cores defined for this CoreContainer */
  public void load() {
    log.info("Loading cores into CoreContainer [instanceDir={}]", loader.getInstancePath());

    // add the sharedLib to the shared resource loader before initializing cfg based plugins
    String libDir = cfg.getSharedLibDirectory();
    if (libDir != null) {
      Path libPath = loader.getInstancePath().resolve(libDir);
      try {
        loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));
        loader.reloadLuceneSPI();
      } catch (IOException e) {
        log.warn("Couldn't add files from {} to classpath: {}", libPath, e.getMessage());
      }
    }

    shardHandlerFactory =
        ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader);

    updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig());

    solrCores.allocateLazyCores(cfg.getTransientCacheSize(), loader);

    logging = LogWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader);

    hostName = cfg.getNodeName();

    zkSys.initZooKeeper(this, solrHome, cfg.getCloudConfig());
    if (isZooKeeperAware())
      pkiAuthenticationPlugin =
          new PKIAuthenticationPlugin(this, zkSys.getZkController().getNodeName());

    ZkStateReader.ConfigData securityConfig =
        isZooKeeperAware()
            ? getZkController().getZkStateReader().getSecurityProps(false)
            : new ZkStateReader.ConfigData(EMPTY_MAP, -1);
    initializeAuthorizationPlugin((Map<String, Object>) securityConfig.data.get("authorization"));
    initializeAuthenticationPlugin((Map<String, Object>) securityConfig.data.get("authentication"));

    this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins());

    containerHandlers.put(ZK_PATH, new ZookeeperInfoHandler(this));
    securityConfHandler = new SecurityConfHandler(this);
    collectionsHandler = createHandler(cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
    containerHandlers.put(COLLECTIONS_HANDLER_PATH, collectionsHandler);
    infoHandler = createHandler(cfg.getInfoHandlerClass(), InfoHandler.class);
    containerHandlers.put(INFO_HANDLER_PATH, infoHandler);
    coreAdminHandler = createHandler(cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
    containerHandlers.put(CORES_HANDLER_PATH, coreAdminHandler);
    configSetsHandler = createHandler(cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class);
    containerHandlers.put(CONFIGSETS_HANDLER_PATH, configSetsHandler);
    containerHandlers.put(AUTHZ_PATH, securityConfHandler);
    containerHandlers.put(AUTHC_PATH, securityConfHandler);
    if (pkiAuthenticationPlugin != null)
      containerHandlers.put(
          PKIAuthenticationPlugin.PATH, pkiAuthenticationPlugin.getRequestHandler());

    coreConfigService = ConfigSetService.createConfigSetService(cfg, loader, zkSys.zkController);

    containerProperties.putAll(cfg.getSolrProperties());

    // setup executor to load cores in parallel
    ExecutorService coreLoadExecutor =
        ExecutorUtil.newMDCAwareFixedThreadPool(
            cfg.getCoreLoadThreadCount(
                isZooKeeperAware()
                    ? DEFAULT_CORE_LOAD_THREADS_IN_CLOUD
                    : DEFAULT_CORE_LOAD_THREADS),
            new DefaultSolrThreadFactory("coreLoadExecutor"));
    final List<Future<SolrCore>> futures = new ArrayList<>();
    try {
      List<CoreDescriptor> cds = coresLocator.discover(this);
      if (isZooKeeperAware()) {
        // sort the cores if it is in SolrCloud. In standalone node the order does not matter
        CoreSorter coreComparator = new CoreSorter().init(this);
        cds = new ArrayList<>(cds); // make a copy
        Collections.sort(cds, coreComparator::compare);
      }
      checkForDuplicateCoreNames(cds);

      for (final CoreDescriptor cd : cds) {
        if (cd.isTransient() || !cd.isLoadOnStartup()) {
          solrCores.putDynamicDescriptor(cd.getName(), cd);
        } else if (asyncSolrCoreLoad) {
          solrCores.markCoreAsLoading(cd);
        }
        if (cd.isLoadOnStartup()) {
          futures.add(
              coreLoadExecutor.submit(
                  () -> {
                    SolrCore core;
                    try {
                      if (zkSys.getZkController() != null) {
                        zkSys.getZkController().throwErrorIfReplicaReplaced(cd);
                      }

                      core = create(cd, false);
                    } finally {
                      if (asyncSolrCoreLoad) {
                        solrCores.markCoreAsNotLoading(cd);
                      }
                    }
                    try {
                      zkSys.registerInZk(core, true);
                    } catch (RuntimeException e) {
                      SolrException.log(log, "Error registering SolrCore", e);
                    }
                    return core;
                  }));
        }
      }

      // Start the background thread
      backgroundCloser = new CloserThread(this, solrCores, cfg);
      backgroundCloser.start();

    } finally {
      if (asyncSolrCoreLoad && futures != null) {

        coreContainerWorkExecutor.submit(
            (Runnable)
                () -> {
                  try {
                    for (Future<SolrCore> future : futures) {
                      try {
                        future.get();
                      } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                      } catch (ExecutionException e) {
                        log.error("Error waiting for SolrCore to be created", e);
                      }
                    }
                  } finally {
                    ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor);
                  }
                });
      } else {
        ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor);
      }
    }

    if (isZooKeeperAware()) {
      zkSys.getZkController().checkOverseerDesignate();
    }
  }
コード例 #13
0
 /**
  * Create a new CoreContainer using the given SolrResourceLoader. The container's cores are not
  * loaded.
  *
  * @param loader the SolrResourceLoader
  * @see #load()
  */
 public CoreContainer(SolrResourceLoader loader) {
   this(SolrXmlConfig.fromSolrHome(loader, loader.getInstancePath()));
 }
コード例 #14
0
 /**
  * Create a new CoreContainer using system properties to detect the solr home directory. The
  * container's cores are not loaded.
  *
  * @see #load()
  */
 public CoreContainer() {
   this(new SolrResourceLoader(SolrResourceLoader.locateSolrHome()));
 }
コード例 #15
0
 protected <T> T createHandler(String handlerClass, Class<T> clazz) {
   return loader.newInstance(
       handlerClass, clazz, null, new Class[] {CoreContainer.class}, new Object[] {this});
 }
コード例 #16
0
ファイル: Config.java プロジェクト: PATRIC3/p3_solr
 protected Properties getSubstituteProperties() {
   return loader.getCoreProperties();
 }