/** Return info from the index */
  private static SimpleOrderedMap<Object> getSchemaInfo(IndexSchema schema) {
    Map<String, List<String>> typeusemap = new HashMap<String, List<String>>();
    SimpleOrderedMap<Object> fields = new SimpleOrderedMap<Object>();
    SchemaField uniqueField = schema.getUniqueKeyField();
    for (SchemaField f : schema.getFields().values()) {
      populateFieldInfo(schema, typeusemap, fields, uniqueField, f);
    }

    SimpleOrderedMap<Object> dynamicFields = new SimpleOrderedMap<Object>();
    for (SchemaField f : schema.getDynamicFieldPrototypes()) {
      populateFieldInfo(schema, typeusemap, dynamicFields, uniqueField, f);
    }
    SimpleOrderedMap<Object> types = new SimpleOrderedMap<Object>();
    for (FieldType ft : schema.getFieldTypes().values()) {
      SimpleOrderedMap<Object> field = new SimpleOrderedMap<Object>();
      field.add("fields", typeusemap.get(ft.getTypeName()));
      field.add("tokenized", ft.isTokenized());
      field.add("className", ft.getClass().getName());
      field.add("indexAnalyzer", getAnalyzerInfo(ft.getAnalyzer()));
      field.add("queryAnalyzer", getAnalyzerInfo(ft.getQueryAnalyzer()));
      types.add(ft.getTypeName(), field);
    }

    SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<Object>();
    finfo.add("fields", fields);
    finfo.add("dynamicFields", dynamicFields);
    finfo.add("uniqueKeyField", null == uniqueField ? null : uniqueField.getName());
    finfo.add("defaultSearchField", schema.getDefaultSearchFieldName());
    finfo.add("types", types);
    return finfo;
  }
Example #2
0
  @Override
  public String init(NamedList config, SolrCore core) {
    LOG.info("init: " + config);
    String name = super.init(config, core);
    threshold =
        config.get(THRESHOLD_TOKEN_FREQUENCY) == null
            ? 0.0f
            : (Float) config.get(THRESHOLD_TOKEN_FREQUENCY);
    sourceLocation = (String) config.get(LOCATION);
    lookupImpl = (String) config.get(LOOKUP_IMPL);

    IndexSchema schema = core.getLatestSchema();
    suggestionAnalyzerFieldTypeName = (String) config.get(SUGGESTION_ANALYZER_FIELDTYPE);
    if (schema.getFieldTypes().containsKey(suggestionAnalyzerFieldTypeName)) {
      FieldType fieldType = schema.getFieldTypes().get(suggestionAnalyzerFieldTypeName);
      suggestionAnalyzer = fieldType.getQueryAnalyzer();
    }

    // support the old classnames without -Factory for config file backwards compatibility.
    if (lookupImpl == null
        || "org.apache.solr.spelling.suggest.jaspell.JaspellLookup".equals(lookupImpl)) {
      lookupImpl = JaspellLookupFactory.class.getName();
    } else if ("org.apache.solr.spelling.suggest.tst.TSTLookup".equals(lookupImpl)) {
      lookupImpl = TSTLookupFactory.class.getName();
    } else if ("org.apache.solr.spelling.suggest.fst.FSTLookup".equals(lookupImpl)) {
      lookupImpl = FSTLookupFactory.class.getName();
    }

    factory = core.getResourceLoader().newInstance(lookupImpl, LookupFactory.class);

    lookup = factory.create(config, core);
    String store = (String) config.get(STORE_DIR);
    if (store != null) {
      storeDir = new File(store);
      if (!storeDir.isAbsolute()) {
        storeDir = new File(core.getDataDir() + File.separator + storeDir);
      }
      if (!storeDir.exists()) {
        storeDir.mkdirs();
      } else {
        // attempt reload of the stored lookup
        try {
          lookup.load(new FileInputStream(new File(storeDir, factory.storeFileName())));
        } catch (IOException e) {
          LOG.warn("Loading stored lookup data failed", e);
        }
      }
    }
    return name;
  }
Example #3
0
 public String init(NamedList config, SolrCore core) {
   name = (String) config.get(DICTIONARY_NAME);
   if (name == null) {
     name = DEFAULT_DICTIONARY_NAME;
   }
   field = (String) config.get(FIELD);
   IndexSchema schema = core.getLatestSchema();
   if (field != null && schema.getFieldTypeNoEx(field) != null) {
     analyzer = schema.getFieldType(field).getQueryAnalyzer();
   }
   fieldTypeName = (String) config.get(FIELD_TYPE);
   if (schema.getFieldTypes().containsKey(fieldTypeName)) {
     FieldType fieldType = schema.getFieldTypes().get(fieldTypeName);
     analyzer = fieldType.getQueryAnalyzer();
   }
   if (analyzer == null) {
     analyzer = new WhitespaceAnalyzer();
   }
   return name;
 }
  @Override
  public Lookup create(NamedList params, SolrCore core) {

    // mandatory parameter
    Object fieldTypeName = params.get(AnalyzingLookupFactory.QUERY_ANALYZER);
    if (fieldTypeName == null) {
      throw new IllegalArgumentException(
          "Error in configuration: "
              + AnalyzingLookupFactory.QUERY_ANALYZER
              + " parameter is mandatory");
    }
    // retrieve index and query analyzers for the field
    FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString());
    if (ft == null) {
      throw new IllegalArgumentException(
          "Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema");
    }
    Analyzer indexAnalyzer = ft.getAnalyzer();
    Analyzer queryAnalyzer = ft.getQueryAnalyzer();

    // optional parameters
    boolean exactMatchFirst =
        (params.get(AnalyzingLookupFactory.EXACT_MATCH_FIRST) != null)
            ? Boolean.valueOf(params.get(AnalyzingLookupFactory.EXACT_MATCH_FIRST).toString())
            : true;

    boolean preserveSep =
        (params.get(AnalyzingLookupFactory.PRESERVE_SEP) != null)
            ? Boolean.valueOf(params.get(AnalyzingLookupFactory.PRESERVE_SEP).toString())
            : true;

    int options = 0;
    if (exactMatchFirst) {
      options |= FuzzySuggester.EXACT_FIRST;
    }
    if (preserveSep) {
      options |= FuzzySuggester.PRESERVE_SEP;
    }

    int maxSurfaceFormsPerAnalyzedForm =
        (params.get(AnalyzingLookupFactory.MAX_SURFACE_FORMS) != null)
            ? Integer.parseInt(params.get(AnalyzingLookupFactory.MAX_SURFACE_FORMS).toString())
            : 256;

    int maxGraphExpansions =
        (params.get(AnalyzingLookupFactory.MAX_EXPANSIONS) != null)
            ? Integer.parseInt(params.get(AnalyzingLookupFactory.MAX_EXPANSIONS).toString())
            : -1;

    boolean preservePositionIncrements =
        params.get(AnalyzingLookupFactory.PRESERVE_POSITION_INCREMENTS) != null
            ? Boolean.valueOf(
                params.get(AnalyzingLookupFactory.PRESERVE_POSITION_INCREMENTS).toString())
            : false;

    int maxEdits =
        (params.get(MAX_EDITS) != null)
            ? Integer.parseInt(params.get(MAX_EDITS).toString())
            : FuzzySuggester.DEFAULT_MAX_EDITS;

    boolean transpositions =
        (params.get(TRANSPOSITIONS) != null)
            ? Boolean.parseBoolean(params.get(TRANSPOSITIONS).toString())
            : FuzzySuggester.DEFAULT_TRANSPOSITIONS;

    int nonFuzzyPrefix =
        (params.get(NON_FUZZY_PREFIX) != null)
            ? Integer.parseInt(params.get(NON_FUZZY_PREFIX).toString())
            : FuzzySuggester.DEFAULT_NON_FUZZY_PREFIX;

    int minFuzzyLength =
        (params.get(MIN_FUZZY_LENGTH) != null)
            ? Integer.parseInt(params.get(MIN_FUZZY_LENGTH).toString())
            : FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH;

    boolean unicodeAware =
        (params.get(UNICODE_AWARE) != null)
            ? Boolean.valueOf(params.get(UNICODE_AWARE).toString())
            : FuzzySuggester.DEFAULT_UNICODE_AWARE;

    return new FuzzySuggester(
        indexAnalyzer,
        queryAnalyzer,
        options,
        maxSurfaceFormsPerAnalyzedForm,
        maxGraphExpansions,
        preservePositionIncrements,
        maxEdits,
        transpositions,
        nonFuzzyPrefix,
        minFuzzyLength,
        unicodeAware);
  }
  @Override
  public Lookup create(NamedList params, SolrCore core) {
    // mandatory parameter
    Object fieldTypeName = params.get(QUERY_ANALYZER);
    if (fieldTypeName == null) {
      throw new IllegalArgumentException(
          "Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory");
    }
    FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString());
    if (ft == null) {
      throw new IllegalArgumentException(
          "Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema");
    }

    Analyzer indexAnalyzer = ft.getIndexAnalyzer();
    Analyzer queryAnalyzer = ft.getQueryAnalyzer();

    // optional parameters

    boolean exactMatchFirst =
        params.get(EXACT_MATCH_FIRST) != null
            ? Boolean.valueOf(params.get(EXACT_MATCH_FIRST).toString())
            : true;

    boolean preserveSep =
        params.get(PRESERVE_SEP) != null
            ? Boolean.valueOf(params.get(PRESERVE_SEP).toString())
            : true;

    int flags = 0;
    if (exactMatchFirst) {
      flags |= AnalyzingSuggester.EXACT_FIRST;
    }
    if (preserveSep) {
      flags |= AnalyzingSuggester.PRESERVE_SEP;
    }

    int maxSurfaceFormsPerAnalyzedForm =
        params.get(MAX_SURFACE_FORMS) != null
            ? Integer.parseInt(params.get(MAX_SURFACE_FORMS).toString())
            : 256;

    int maxGraphExpansions =
        params.get(MAX_EXPANSIONS) != null
            ? Integer.parseInt(params.get(MAX_EXPANSIONS).toString())
            : -1;

    boolean preservePositionIncrements =
        params.get(PRESERVE_POSITION_INCREMENTS) != null
            ? Boolean.valueOf(params.get(PRESERVE_POSITION_INCREMENTS).toString())
            : false;

    return new AnalyzingSuggester(
        getTempDir(),
        "suggester",
        indexAnalyzer,
        queryAnalyzer,
        flags,
        maxSurfaceFormsPerAnalyzedForm,
        maxGraphExpansions,
        preservePositionIncrements);
  }
  @Override
  public void inform(SolrCore core) {
    IndexSchema schema = core.getLatestSchema();
    String a = initArgs.get(FIELD_TYPE);
    if (a != null) {
      FieldType ft = schema.getFieldTypes().get(a);
      if (ft == null) {
        throw new SolrException(
            SolrException.ErrorCode.SERVER_ERROR,
            "Unknown FieldType: '" + a + "' used in QueryElevationComponent");
      }
      analyzer = ft.getQueryAnalyzer();
    }

    SchemaField sf = schema.getUniqueKeyField();
    if (sf == null) {
      throw new SolrException(
          SolrException.ErrorCode.SERVER_ERROR,
          "QueryElevationComponent requires the schema to have a uniqueKeyField.");
    }
    idSchemaFT = sf.getType();
    idField = sf.getName();
    // register the EditorialMarkerFactory
    String excludeName = initArgs.get(QueryElevationParams.EXCLUDE_MARKER_FIELD_NAME, "excluded");
    if (excludeName == null || excludeName.equals("") == true) {
      excludeName = "excluded";
    }
    ExcludedMarkerFactory excludedMarkerFactory = new ExcludedMarkerFactory();
    core.addTransformerFactory(excludeName, excludedMarkerFactory);
    ElevatedMarkerFactory elevatedMarkerFactory = new ElevatedMarkerFactory();
    String markerName = initArgs.get(QueryElevationParams.EDITORIAL_MARKER_FIELD_NAME, "elevated");
    if (markerName == null || markerName.equals("") == true) {
      markerName = "elevated";
    }
    core.addTransformerFactory(markerName, elevatedMarkerFactory);
    forceElevation = initArgs.getBool(QueryElevationParams.FORCE_ELEVATION, forceElevation);
    try {
      synchronized (elevationCache) {
        elevationCache.clear();
        String f = initArgs.get(CONFIG_FILE);
        if (f == null) {
          throw new SolrException(
              SolrException.ErrorCode.SERVER_ERROR,
              "QueryElevationComponent must specify argument: '"
                  + CONFIG_FILE
                  + "' -- path to elevate.xml");
        }
        boolean exists = false;

        // check if using ZooKeeper
        ZkController zkController = core.getCoreDescriptor().getCoreContainer().getZkController();
        if (zkController != null) {
          // TODO : shouldn't have to keep reading the config name when it has been read before
          exists =
              zkController.configFileExists(
                  zkController
                      .getZkStateReader()
                      .readConfigName(
                          core.getCoreDescriptor().getCloudDescriptor().getCollectionName()),
                  f);
        } else {
          File fC = new File(core.getResourceLoader().getConfigDir(), f);
          File fD = new File(core.getDataDir(), f);
          if (fC.exists() == fD.exists()) {
            throw new SolrException(
                SolrException.ErrorCode.SERVER_ERROR,
                "QueryElevationComponent missing config file: '"
                    + f
                    + "\n"
                    + "either: "
                    + fC.getAbsolutePath()
                    + " or "
                    + fD.getAbsolutePath()
                    + " must exist, but not both.");
          }
          if (fC.exists()) {
            exists = true;
            log.info("Loading QueryElevation from: " + fC.getAbsolutePath());
            Config cfg = new Config(core.getResourceLoader(), f);
            elevationCache.put(null, loadElevationMap(cfg));
          }
        }
        // in other words, we think this is in the data dir, not the conf dir
        if (!exists) {
          // preload the first data
          RefCounted<SolrIndexSearcher> searchHolder = null;
          try {
            searchHolder = core.getNewestSearcher(false);
            IndexReader reader = searchHolder.get().getIndexReader();
            getElevationMap(reader, core);
          } finally {
            if (searchHolder != null) searchHolder.decref();
          }
        }
      }
    } catch (Exception ex) {
      throw new SolrException(
          SolrException.ErrorCode.SERVER_ERROR, "Error initializing QueryElevationComponent.", ex);
    }
  }