public void inform(SolrCore core) {
   List<PluginInfo> children = info.getChildren("highlighting");
   if (children.isEmpty()) {
     PluginInfo pluginInfo =
         core.getSolrConfig()
             .getPluginInfo(
                 SolrHighlighter.class.getName()); // TODO deprecated configuration remove later
     if (pluginInfo != null) {
       highlighter =
           core.createInitInstance(
               pluginInfo, SolrHighlighter.class, null, DefaultSolrHighlighter.class.getName());
       highlighter.initalize(core.getSolrConfig());
     } else {
       DefaultSolrHighlighter defHighlighter = new DefaultSolrHighlighter(core);
       defHighlighter.init(PluginInfo.EMPTY_INFO);
       highlighter = defHighlighter;
     }
   } else {
     highlighter =
         core.createInitInstance(
             children.get(0), SolrHighlighter.class, null, DefaultSolrHighlighter.class.getName());
   }
 }
  private static Runnable getListener(SolrCore core, ZkSolrResourceLoader zkSolrResourceLoader) {
    final String coreName = core.getName();
    final CoreContainer cc = core.getCoreDescriptor().getCoreContainer();
    final String overlayPath =
        zkSolrResourceLoader.getConfigSetZkPath() + "/" + ConfigOverlay.RESOURCE_NAME;
    final String solrConfigPath =
        zkSolrResourceLoader.getConfigSetZkPath() + "/" + core.getSolrConfig().getName();
    String schemaRes = null;
    if (core.getLatestSchema().isMutable()
        && core.getLatestSchema() instanceof ManagedIndexSchema) {
      ManagedIndexSchema mis = (ManagedIndexSchema) core.getLatestSchema();
      schemaRes = mis.getResourceName();
    }
    final String managedSchmaResourcePath =
        schemaRes == null ? null : zkSolrResourceLoader.getConfigSetZkPath() + "/" + schemaRes;
    return new Runnable() {
      @Override
      public void run() {
        log.info("config update listener called for core {}", coreName);
        SolrZkClient zkClient = cc.getZkController().getZkClient();
        int solrConfigversion, overlayVersion, managedSchemaVersion = 0;
        SolrConfig cfg = null;
        try (SolrCore core = cc.getCore(coreName)) {
          if (core.isClosed()) return;
          cfg = core.getSolrConfig();
          solrConfigversion = core.getSolrConfig().getOverlay().getZnodeVersion();
          overlayVersion = core.getSolrConfig().getZnodeVersion();
          if (managedSchmaResourcePath != null) {
            managedSchemaVersion =
                ((ManagedIndexSchema) core.getLatestSchema()).getSchemaZkVersion();
          }
        }
        if (cfg != null) {
          cfg.refreshRequestParams();
        }

        if (checkStale(zkClient, overlayPath, solrConfigversion)
            || checkStale(zkClient, solrConfigPath, overlayVersion)
            || checkStale(zkClient, managedSchmaResourcePath, managedSchemaVersion)) {
          log.info("core reload {}", coreName);
          cc.reload(coreName);
        }
      }
    };
  }
 /** Initialize using an explicit SolrCore */
 public DirectSolrConnection(SolrCore c) {
   core = c;
   parser = new SolrRequestParsers(c.getSolrConfig());
 }
  private void init() throws Exception {
    // The states of client that is invalid in this request
    Aliases aliases = null;
    String corename = "";
    String origCorename = null;
    // set a request timer which can be reused by requests if needed
    req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimerTree());
    // put the core container in request attribute
    req.setAttribute("org.apache.solr.CoreContainer", cores);
    path = req.getServletPath();
    if (req.getPathInfo() != null) {
      // this lets you handle /update/commit when /update is a servlet
      path += req.getPathInfo();
    }
    // check for management path
    String alternate = cores.getManagementPath();
    if (alternate != null && path.startsWith(alternate)) {
      path = path.substring(0, alternate.length());
    }
    // unused feature ?
    int idx = path.indexOf(':');
    if (idx > 0) {
      // save the portion after the ':' for a 'handler' path parameter
      path = path.substring(0, idx);
    }

    boolean usingAliases = false;

    // Check for container handlers
    handler = cores.getRequestHandler(path);
    if (handler != null) {
      solrReq = SolrRequestParsers.DEFAULT.parse(null, path, req);
      solrReq.getContext().put(CoreContainer.class.getName(), cores);
      requestType = RequestType.ADMIN;
      action = ADMIN;
      return;
    } else {
      // otherwise, we should find a core from the path
      idx = path.indexOf("/", 1);
      if (idx > 1) {
        // try to get the corename as a request parameter first
        corename = path.substring(1, idx);

        // look at aliases
        if (cores.isZooKeeperAware()) {
          origCorename = corename;
          ZkStateReader reader = cores.getZkController().getZkStateReader();
          aliases = reader.getAliases();
          if (aliases != null && aliases.collectionAliasSize() > 0) {
            usingAliases = true;
            String alias = aliases.getCollectionAlias(corename);
            if (alias != null) {
              collectionsList = StrUtils.splitSmart(alias, ",", true);
              corename = collectionsList.get(0);
            }
          }
        }

        core = cores.getCore(corename);
        if (core != null) {
          path = path.substring(idx);
        } else if (cores.isCoreLoading(
            corename)) { // extra mem barriers, so don't look at this before trying to get core
          throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is loading");
        } else {
          // the core may have just finished loading
          core = cores.getCore(corename);
          if (core != null) {
            path = path.substring(idx);
          }
        }
      }
      if (core == null) {
        if (!cores.isZooKeeperAware()) {
          core = cores.getCore("");
        }
      }
    }

    if (core == null && cores.isZooKeeperAware()) {
      // we couldn't find the core - lets make sure a collection was not specified instead
      core = getCoreByCollection(corename);
      if (core != null) {
        // we found a core, update the path
        path = path.substring(idx);
        if (collectionsList == null) collectionsList = new ArrayList<>();
        collectionsList.add(corename);
      }

      // if we couldn't find it locally, look on other nodes
      extractRemotePath(corename, origCorename, idx);
      if (action != null) return;
    }

    // With a valid core...
    if (core != null) {
      MDCLoggingContext.setCore(core);
      config = core.getSolrConfig();
      // get or create/cache the parser for the core
      SolrRequestParsers parser = config.getRequestParsers();

      // Determine the handler from the url path if not set
      // (we might already have selected the cores handler)
      extractHandlerFromURLPath(parser);
      if (action != null) return;

      // With a valid handler and a valid core...
      if (handler != null) {
        // if not a /select, create the request
        if (solrReq == null) {
          solrReq = parser.parse(core, path, req);
        }

        if (usingAliases) {
          processAliases(aliases, collectionsList);
        }

        action = PROCESS;
        return; // we are done with a valid handler
      }
    }
    log.debug("no handler or core retrieved for " + path + ", follow through...");

    action = PASSTHROUGH;
  }
Beispiel #5
0
  public void inform(SolrCore core) {

    /* The stream factory will always contain the zkUrl for the given collection
     * Adds default streams with their corresponding function names. These
     * defaults can be overridden or added to in the solrConfig in the stream
     * RequestHandler def. Example config override
     *  <lst name="streamFunctions">
     *    <str name="group">org.apache.solr.client.solrj.io.stream.ReducerStream</str>
     *    <str name="count">org.apache.solr.client.solrj.io.stream.RecordCountStream</str>
     *  </lst>
     * */

    String defaultCollection;
    String defaultZkhost;
    CoreContainer coreContainer = core.getCoreDescriptor().getCoreContainer();
    this.coreName = core.getName();

    if (coreContainer.isZooKeeperAware()) {
      defaultCollection = core.getCoreDescriptor().getCollectionName();
      defaultZkhost =
          core.getCoreDescriptor().getCoreContainer().getZkController().getZkServerAddress();
      streamFactory.withCollectionZkHost(defaultCollection, defaultZkhost);
      streamFactory.withDefaultZkHost(defaultZkhost);
      modelCache = new ModelCache(250, defaultZkhost, clientCache);
    }

    streamFactory
        // source streams
        .withFunctionName("search", CloudSolrStream.class)
        .withFunctionName("facet", FacetStream.class)
        .withFunctionName("update", UpdateStream.class)
        .withFunctionName("jdbc", JDBCStream.class)
        .withFunctionName("topic", TopicStream.class)
        .withFunctionName("commit", CommitStream.class)
        .withFunctionName("random", RandomStream.class)

        // decorator streams
        .withFunctionName("merge", MergeStream.class)
        .withFunctionName("unique", UniqueStream.class)
        .withFunctionName("top", RankStream.class)
        .withFunctionName("group", GroupOperation.class)
        .withFunctionName("reduce", ReducerStream.class)
        .withFunctionName("parallel", ParallelStream.class)
        .withFunctionName("rollup", RollupStream.class)
        .withFunctionName("stats", StatsStream.class)
        .withFunctionName("innerJoin", InnerJoinStream.class)
        .withFunctionName("leftOuterJoin", LeftOuterJoinStream.class)
        .withFunctionName("hashJoin", HashJoinStream.class)
        .withFunctionName("outerHashJoin", OuterHashJoinStream.class)
        .withFunctionName("intersect", IntersectStream.class)
        .withFunctionName("complement", ComplementStream.class)
        .withFunctionName("sort", SortStream.class)
        .withFunctionName("train", TextLogitStream.class)
        .withFunctionName("features", FeaturesSelectionStream.class)
        .withFunctionName("daemon", DaemonStream.class)
        .withFunctionName("shortestPath", ShortestPathStream.class)
        .withFunctionName("gatherNodes", GatherNodesStream.class)
        .withFunctionName("nodes", GatherNodesStream.class)
        .withFunctionName("select", SelectStream.class)
        .withFunctionName("scoreNodes", ScoreNodesStream.class)
        .withFunctionName("model", ModelStream.class)
        .withFunctionName("classify", ClassifyStream.class)
        .withFunctionName("fetch", FetchStream.class)
        .withFunctionName("executor", ExecutorStream.class)
        .withFunctionName("null", NullStream.class)
        .withFunctionName("priority", PriorityStream.class)
        // metrics
        .withFunctionName("min", MinMetric.class)
        .withFunctionName("max", MaxMetric.class)
        .withFunctionName("avg", MeanMetric.class)
        .withFunctionName("sum", SumMetric.class)
        .withFunctionName("count", CountMetric.class)

        // tuple manipulation operations
        .withFunctionName("replace", ReplaceOperation.class)
        .withFunctionName("concat", ConcatOperation.class)

        // stream reduction operations
        .withFunctionName("group", GroupOperation.class)
        .withFunctionName("distinct", DistinctOperation.class)
        .withFunctionName("having", HavingStream.class)
        .withFunctionName("and", AndOperation.class)
        .withFunctionName("or", OrOperation.class)
        .withFunctionName("not", NotOperation.class)
        .withFunctionName("gt", GreaterThanOperation.class)
        .withFunctionName("lt", LessThanOperation.class)
        .withFunctionName("eq", EqualsOperation.class)
        .withFunctionName("lteq", LessThanEqualToOperation.class)
        .withFunctionName("gteq", GreaterThanEqualToOperation.class);

    // This pulls all the overrides and additions from the config
    List<PluginInfo> pluginInfos = core.getSolrConfig().getPluginInfos(Expressible.class.getName());
    for (PluginInfo pluginInfo : pluginInfos) {
      Class<? extends Expressible> clazz =
          core.getResourceLoader().findClass(pluginInfo.className, Expressible.class);
      streamFactory.withFunctionName(pluginInfo.name, clazz);
    }

    core.addCloseHook(
        new CloseHook() {
          @Override
          public void preClose(SolrCore core) {
            // To change body of implemented methods use File | Settings | File Templates.
          }

          @Override
          public void postClose(SolrCore core) {
            clientCache.close();
          }
        });
  }
Beispiel #6
0
  public void handleRequest(RequestGetter requestGetter) {
    MDCLoggingContext.reset();
    MDCLoggingContext.setNode(cores);

    String path = requestGetter.getPath();
    solrParams = requestGetter.getSolrParams();
    SolrRequestHandler handler = null;
    String corename = "";
    String origCorename = null;
    try {
      // set a request timer which can be reused by requests if needed
      // req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimer());
      // put the core container in request attribute
      // req.setAttribute("org.apache.solr.CoreContainer", cores);
      // check for management path
      String alternate = cores.getManagementPath();
      if (alternate != null && path.startsWith(alternate)) {
        path = path.substring(0, alternate.length());
      }
      // unused feature ?
      int idx = path.indexOf(':');
      if (idx > 0) {
        // save the portion after the ':' for a 'handler' path parameter
        path = path.substring(0, idx);
      }

      boolean usingAliases = false;
      List<String> collectionsList = null;

      // Check for container handlers
      handler = cores.getRequestHandler(path);
      if (handler != null) {
        solrReq = parseSolrQueryRequest(SolrRequestParsers.DEFAULT, requestGetter);
        handleAdminRequest(handler, solrReq);
        return;
      } else {
        // otherwise, we should find a core from the path
        idx = path.indexOf("/", 1);
        if (idx > 1) {
          // try to get the corename as a request parameter first
          corename = path.substring(1, idx);

          // look at aliases
          if (cores.isZooKeeperAware()) {
            origCorename = corename;
            ZkStateReader reader = cores.getZkController().getZkStateReader();
            aliases = reader.getAliases();
            if (aliases != null && aliases.collectionAliasSize() > 0) {
              usingAliases = true;
              String alias = aliases.getCollectionAlias(corename);
              if (alias != null) {
                collectionsList = StrUtils.splitSmart(alias, ",", true);
                corename = collectionsList.get(0);
              }
            }
          }

          core = cores.getCore(corename);

          if (core != null) {
            path = path.substring(idx);
          }
        }

        // add collection name
        if (core == null && StringUtils.isNotBlank(requestGetter.getCollection())) {
          corename = requestGetter.getCollection();
          core = cores.getCore(corename);
        }

        if (core == null) {
          if (!cores.isZooKeeperAware()) {
            core = cores.getCore("");
          }
        }
      }

      if (core == null && cores.isZooKeeperAware()) {
        // we couldn't find the core - lets make sure a collection was not specified instead
        core = getCoreByCollection(cores, corename);

        if (core != null) {
          // we found a core, update the path
          path = path.substring(idx);
        }

        // try the default core
        if (core == null) {
          core = cores.getCore("");
          if (core != null) {}
        }
      }

      // With a valid core...
      if (core != null) {
        MDCLoggingContext.setCore(core);
        final SolrConfig config = core.getSolrConfig();
        // get or create/cache the parser for the core
        SolrRequestParsers parser = config.getRequestParsers();

        // Determine the handler from the url path if not set
        // (we might already have selected the cores handler)
        if (handler == null && path.length() > 1) { // don't match "" or "/" as valid path
          handler = core.getRequestHandler(path);

          if (handler == null) {
            // may be a restlet path
            // Handle /schema/* paths via Restlet
            if (path.equals("/schema") || path.startsWith("/schema/")) {
              throw new SolrException(
                  SolrException.ErrorCode.BAD_REQUEST, "unsupport /schema/**, use http solr");
            }
          }
          // no handler yet but allowed to handle select; let's check
          if (handler == null && parser.isHandleSelect()) {
            if ("/select".equals(path) || "/select/".equals(path)) {
              solrReq = parseSolrQueryRequest(parser, requestGetter);

              invalidStates =
                  checkStateIsValid(cores, solrReq.getParams().get(CloudSolrClient.STATE_VERSION));
              String qt = solrReq.getParams().get(CommonParams.QT);
              handler = core.getRequestHandler(qt);
              if (handler == null) {
                throw new SolrException(
                    SolrException.ErrorCode.BAD_REQUEST, "unknown handler: " + qt);
              }
              if (qt != null
                  && qt.startsWith("/")
                  && (handler instanceof ContentStreamHandlerBase)) {
                // For security reasons it's a bad idea to allow a leading '/', ex:
                // /select?qt=/update see SOLR-3161
                // There was no restriction from Solr 1.4 thru 3.5 and it's not supported for update
                // handlers.
                throw new SolrException(
                    SolrException.ErrorCode.BAD_REQUEST,
                    "Invalid Request Handler ('qt').  Do not use /select to access: " + qt);
              }
            }
          }
        }

        // With a valid handler and a valid core...
        if (handler != null) {
          // if not a /select, create the request
          if (solrReq == null) {
            solrReq = parseSolrQueryRequest(parser, requestGetter);
          }

          if (usingAliases) {
            processAliases(solrReq, aliases, collectionsList);
          }

          SolrQueryResponse solrRsp = new SolrQueryResponse();
          SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp));
          this.execute(handler, solrReq, solrRsp);
          QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq);
          if (invalidStates != null)
            solrReq.getContext().put(CloudSolrClient.STATE_VERSION, invalidStates);
          writeResponse(solrRsp, responseWriter, solrReq);

          return; // we are done with a valid handler
        }
      }
      logger.debug("no handler or core retrieved for {}, follow through...", path);
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST, "no handler or core retrieved for " + path);
    } catch (Throwable ex) {
      sendError(core, solrReq, ex);
      // walk the the entire cause chain to search for an Error
      Throwable t = ex;
      while (t != null) {
        if (t instanceof Error) {
          if (t != ex) {
            logger.error(
                "An Error was wrapped in another exception - please report complete stacktrace on SOLR-6161",
                ex);
          }
          throw (Error) t;
        }
        t = t.getCause();
      }
      return;
    } finally {
      try {
        if (solrReq != null) {
          logger.debug("Closing out SolrRequest: {}", solrReq);
          solrReq.close();
        }
      } finally {
        try {
          if (core != null) {
            core.close();
          }
        } finally {
          SolrRequestInfo.clearRequestInfo();
        }
      }
      MDCLoggingContext.clear();
    }
  }
  private boolean doRestore() throws Exception {

    Path backupPath = Paths.get(backupLocation).resolve(backupName);
    SimpleDateFormat dateFormat = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT);
    String restoreIndexName = "restore." + dateFormat.format(new Date());
    String restoreIndexPath = core.getDataDir() + restoreIndexName;

    Directory restoreIndexDir = null;
    Directory indexDir = null;
    try (Directory backupDir = FSDirectory.open(backupPath)) {

      final Version version =
          IndexFetcher.checkOldestVersion(SegmentInfos.readLatestCommit(backupDir));

      restoreIndexDir =
          core.getDirectoryFactory()
              .get(
                  restoreIndexPath,
                  DirectoryFactory.DirContext.DEFAULT,
                  core.getSolrConfig().indexConfig.lockType);

      // Prefer local copy.
      indexDir =
          core.getDirectoryFactory()
              .get(
                  core.getIndexDir(),
                  DirectoryFactory.DirContext.DEFAULT,
                  core.getSolrConfig().indexConfig.lockType);

      // Move all files from backupDir to restoreIndexDir
      for (String filename : backupDir.listAll()) {
        checkInterrupted();
        log.info("Copying file {} to restore directory ", filename);
        try (IndexInput indexInput = backupDir.openInput(filename, IOContext.READONCE)) {
          Long checksum = null;
          try {
            checksum = CodecUtil.retrieveChecksum(indexInput);
          } catch (Exception e) {
            log.warn("Could not read checksum from index file: " + filename, e);
          }
          long length = indexInput.length();
          IndexFetcher.CompareResult compareResult =
              IndexFetcher.compareFile(indexDir, version, filename, length, checksum);
          if (!compareResult.equal
              || (!compareResult.checkSummed
                  && (filename.endsWith(".si")
                      || filename.endsWith(".liv")
                      || filename.startsWith("segments_")))) {
            restoreIndexDir.copyFrom(backupDir, filename, filename, IOContext.READONCE);
          } else {
            // prefer local copy
            restoreIndexDir.copyFrom(indexDir, filename, filename, IOContext.READONCE);
          }
        } catch (Exception e) {
          throw new SolrException(
              SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e);
        }
      }
      log.debug("Switching directories");
      IndexFetcher.modifyIndexProps(core, restoreIndexName);

      boolean success;
      try {
        core.getUpdateHandler().newIndexWriter(false);
        openNewSearcher();
        success = true;
        log.info("Successfully restored to the backup index");
      } catch (Exception e) {
        // Rollback to the old index directory. Delete the restore index directory and mark the
        // restore as failed.
        log.warn("Could not switch to restored index. Rolling back to the current index");
        Directory dir = null;
        try {
          dir =
              core.getDirectoryFactory()
                  .get(
                      core.getDataDir(),
                      DirectoryFactory.DirContext.META_DATA,
                      core.getSolrConfig().indexConfig.lockType);
          dir.deleteFile(IndexFetcher.INDEX_PROPERTIES);
        } finally {
          if (dir != null) {
            core.getDirectoryFactory().release(dir);
          }
        }

        core.getDirectoryFactory().doneWithDirectory(restoreIndexDir);
        core.getDirectoryFactory().remove(restoreIndexDir);
        core.getUpdateHandler().newIndexWriter(false);
        openNewSearcher();
        throw new SolrException(
            SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e);
      }
      if (success) {
        core.getDirectoryFactory().doneWithDirectory(indexDir);
        core.getDirectoryFactory().remove(indexDir);
      }

      return true;
    } finally {
      if (restoreIndexDir != null) {
        core.getDirectoryFactory().release(restoreIndexDir);
      }
      if (indexDir != null) {
        core.getDirectoryFactory().release(indexDir);
      }
    }
  }