/** Return info from the index */
private static SimpleOrderedMap<Object> getSchemaInfo(IndexSchema schema) {
Map<String, List<String>> typeusemap = new HashMap<String, List<String>>();
SimpleOrderedMap<Object> fields = new SimpleOrderedMap<Object>();
SchemaField uniqueField = schema.getUniqueKeyField();
for (SchemaField f : schema.getFields().values()) {
populateFieldInfo(schema, typeusemap, fields, uniqueField, f);
}
SimpleOrderedMap<Object> dynamicFields = new SimpleOrderedMap<Object>();
for (SchemaField f : schema.getDynamicFieldPrototypes()) {
populateFieldInfo(schema, typeusemap, dynamicFields, uniqueField, f);
}
SimpleOrderedMap<Object> types = new SimpleOrderedMap<Object>();
for (FieldType ft : schema.getFieldTypes().values()) {
SimpleOrderedMap<Object> field = new SimpleOrderedMap<Object>();
field.add("fields", typeusemap.get(ft.getTypeName()));
field.add("tokenized", ft.isTokenized());
field.add("className", ft.getClass().getName());
field.add("indexAnalyzer", getAnalyzerInfo(ft.getAnalyzer()));
field.add("queryAnalyzer", getAnalyzerInfo(ft.getQueryAnalyzer()));
types.add(ft.getTypeName(), field);
}
SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<Object>();
finfo.add("fields", fields);
finfo.add("dynamicFields", dynamicFields);
finfo.add("uniqueKeyField", null == uniqueField ? null : uniqueField.getName());
finfo.add("defaultSearchField", schema.getDefaultSearchFieldName());
finfo.add("types", types);
return finfo;
}
@Override
public String init(NamedList config, SolrCore core) {
LOG.info("init: " + config);
String name = super.init(config, core);
threshold =
config.get(THRESHOLD_TOKEN_FREQUENCY) == null
? 0.0f
: (Float) config.get(THRESHOLD_TOKEN_FREQUENCY);
sourceLocation = (String) config.get(LOCATION);
lookupImpl = (String) config.get(LOOKUP_IMPL);
IndexSchema schema = core.getLatestSchema();
suggestionAnalyzerFieldTypeName = (String) config.get(SUGGESTION_ANALYZER_FIELDTYPE);
if (schema.getFieldTypes().containsKey(suggestionAnalyzerFieldTypeName)) {
FieldType fieldType = schema.getFieldTypes().get(suggestionAnalyzerFieldTypeName);
suggestionAnalyzer = fieldType.getQueryAnalyzer();
}
// support the old classnames without -Factory for config file backwards compatibility.
if (lookupImpl == null
|| "org.apache.solr.spelling.suggest.jaspell.JaspellLookup".equals(lookupImpl)) {
lookupImpl = JaspellLookupFactory.class.getName();
} else if ("org.apache.solr.spelling.suggest.tst.TSTLookup".equals(lookupImpl)) {
lookupImpl = TSTLookupFactory.class.getName();
} else if ("org.apache.solr.spelling.suggest.fst.FSTLookup".equals(lookupImpl)) {
lookupImpl = FSTLookupFactory.class.getName();
}
factory = core.getResourceLoader().newInstance(lookupImpl, LookupFactory.class);
lookup = factory.create(config, core);
String store = (String) config.get(STORE_DIR);
if (store != null) {
storeDir = new File(store);
if (!storeDir.isAbsolute()) {
storeDir = new File(core.getDataDir() + File.separator + storeDir);
}
if (!storeDir.exists()) {
storeDir.mkdirs();
} else {
// attempt reload of the stored lookup
try {
lookup.load(new FileInputStream(new File(storeDir, factory.storeFileName())));
} catch (IOException e) {
LOG.warn("Loading stored lookup data failed", e);
}
}
}
return name;
}
public String init(NamedList config, SolrCore core) {
name = (String) config.get(DICTIONARY_NAME);
if (name == null) {
name = DEFAULT_DICTIONARY_NAME;
}
field = (String) config.get(FIELD);
IndexSchema schema = core.getLatestSchema();
if (field != null && schema.getFieldTypeNoEx(field) != null) {
analyzer = schema.getFieldType(field).getQueryAnalyzer();
}
fieldTypeName = (String) config.get(FIELD_TYPE);
if (schema.getFieldTypes().containsKey(fieldTypeName)) {
FieldType fieldType = schema.getFieldTypes().get(fieldTypeName);
analyzer = fieldType.getQueryAnalyzer();
}
if (analyzer == null) {
analyzer = new WhitespaceAnalyzer();
}
return name;
}
@Override
public Lookup create(NamedList params, SolrCore core) {
// mandatory parameter
Object fieldTypeName = params.get(AnalyzingLookupFactory.QUERY_ANALYZER);
if (fieldTypeName == null) {
throw new IllegalArgumentException(
"Error in configuration: "
+ AnalyzingLookupFactory.QUERY_ANALYZER
+ " parameter is mandatory");
}
// retrieve index and query analyzers for the field
FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString());
if (ft == null) {
throw new IllegalArgumentException(
"Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema");
}
Analyzer indexAnalyzer = ft.getAnalyzer();
Analyzer queryAnalyzer = ft.getQueryAnalyzer();
// optional parameters
boolean exactMatchFirst =
(params.get(AnalyzingLookupFactory.EXACT_MATCH_FIRST) != null)
? Boolean.valueOf(params.get(AnalyzingLookupFactory.EXACT_MATCH_FIRST).toString())
: true;
boolean preserveSep =
(params.get(AnalyzingLookupFactory.PRESERVE_SEP) != null)
? Boolean.valueOf(params.get(AnalyzingLookupFactory.PRESERVE_SEP).toString())
: true;
int options = 0;
if (exactMatchFirst) {
options |= FuzzySuggester.EXACT_FIRST;
}
if (preserveSep) {
options |= FuzzySuggester.PRESERVE_SEP;
}
int maxSurfaceFormsPerAnalyzedForm =
(params.get(AnalyzingLookupFactory.MAX_SURFACE_FORMS) != null)
? Integer.parseInt(params.get(AnalyzingLookupFactory.MAX_SURFACE_FORMS).toString())
: 256;
int maxGraphExpansions =
(params.get(AnalyzingLookupFactory.MAX_EXPANSIONS) != null)
? Integer.parseInt(params.get(AnalyzingLookupFactory.MAX_EXPANSIONS).toString())
: -1;
boolean preservePositionIncrements =
params.get(AnalyzingLookupFactory.PRESERVE_POSITION_INCREMENTS) != null
? Boolean.valueOf(
params.get(AnalyzingLookupFactory.PRESERVE_POSITION_INCREMENTS).toString())
: false;
int maxEdits =
(params.get(MAX_EDITS) != null)
? Integer.parseInt(params.get(MAX_EDITS).toString())
: FuzzySuggester.DEFAULT_MAX_EDITS;
boolean transpositions =
(params.get(TRANSPOSITIONS) != null)
? Boolean.parseBoolean(params.get(TRANSPOSITIONS).toString())
: FuzzySuggester.DEFAULT_TRANSPOSITIONS;
int nonFuzzyPrefix =
(params.get(NON_FUZZY_PREFIX) != null)
? Integer.parseInt(params.get(NON_FUZZY_PREFIX).toString())
: FuzzySuggester.DEFAULT_NON_FUZZY_PREFIX;
int minFuzzyLength =
(params.get(MIN_FUZZY_LENGTH) != null)
? Integer.parseInt(params.get(MIN_FUZZY_LENGTH).toString())
: FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH;
boolean unicodeAware =
(params.get(UNICODE_AWARE) != null)
? Boolean.valueOf(params.get(UNICODE_AWARE).toString())
: FuzzySuggester.DEFAULT_UNICODE_AWARE;
return new FuzzySuggester(
indexAnalyzer,
queryAnalyzer,
options,
maxSurfaceFormsPerAnalyzedForm,
maxGraphExpansions,
preservePositionIncrements,
maxEdits,
transpositions,
nonFuzzyPrefix,
minFuzzyLength,
unicodeAware);
}
@Override
public Lookup create(NamedList params, SolrCore core) {
// mandatory parameter
Object fieldTypeName = params.get(QUERY_ANALYZER);
if (fieldTypeName == null) {
throw new IllegalArgumentException(
"Error in configuration: " + QUERY_ANALYZER + " parameter is mandatory");
}
FieldType ft = core.getLatestSchema().getFieldTypeByName(fieldTypeName.toString());
if (ft == null) {
throw new IllegalArgumentException(
"Error in configuration: " + fieldTypeName.toString() + " is not defined in the schema");
}
Analyzer indexAnalyzer = ft.getIndexAnalyzer();
Analyzer queryAnalyzer = ft.getQueryAnalyzer();
// optional parameters
boolean exactMatchFirst =
params.get(EXACT_MATCH_FIRST) != null
? Boolean.valueOf(params.get(EXACT_MATCH_FIRST).toString())
: true;
boolean preserveSep =
params.get(PRESERVE_SEP) != null
? Boolean.valueOf(params.get(PRESERVE_SEP).toString())
: true;
int flags = 0;
if (exactMatchFirst) {
flags |= AnalyzingSuggester.EXACT_FIRST;
}
if (preserveSep) {
flags |= AnalyzingSuggester.PRESERVE_SEP;
}
int maxSurfaceFormsPerAnalyzedForm =
params.get(MAX_SURFACE_FORMS) != null
? Integer.parseInt(params.get(MAX_SURFACE_FORMS).toString())
: 256;
int maxGraphExpansions =
params.get(MAX_EXPANSIONS) != null
? Integer.parseInt(params.get(MAX_EXPANSIONS).toString())
: -1;
boolean preservePositionIncrements =
params.get(PRESERVE_POSITION_INCREMENTS) != null
? Boolean.valueOf(params.get(PRESERVE_POSITION_INCREMENTS).toString())
: false;
return new AnalyzingSuggester(
getTempDir(),
"suggester",
indexAnalyzer,
queryAnalyzer,
flags,
maxSurfaceFormsPerAnalyzedForm,
maxGraphExpansions,
preservePositionIncrements);
}
@Override
public void inform(SolrCore core) {
IndexSchema schema = core.getLatestSchema();
String a = initArgs.get(FIELD_TYPE);
if (a != null) {
FieldType ft = schema.getFieldTypes().get(a);
if (ft == null) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"Unknown FieldType: '" + a + "' used in QueryElevationComponent");
}
analyzer = ft.getQueryAnalyzer();
}
SchemaField sf = schema.getUniqueKeyField();
if (sf == null) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"QueryElevationComponent requires the schema to have a uniqueKeyField.");
}
idSchemaFT = sf.getType();
idField = sf.getName();
// register the EditorialMarkerFactory
String excludeName = initArgs.get(QueryElevationParams.EXCLUDE_MARKER_FIELD_NAME, "excluded");
if (excludeName == null || excludeName.equals("") == true) {
excludeName = "excluded";
}
ExcludedMarkerFactory excludedMarkerFactory = new ExcludedMarkerFactory();
core.addTransformerFactory(excludeName, excludedMarkerFactory);
ElevatedMarkerFactory elevatedMarkerFactory = new ElevatedMarkerFactory();
String markerName = initArgs.get(QueryElevationParams.EDITORIAL_MARKER_FIELD_NAME, "elevated");
if (markerName == null || markerName.equals("") == true) {
markerName = "elevated";
}
core.addTransformerFactory(markerName, elevatedMarkerFactory);
forceElevation = initArgs.getBool(QueryElevationParams.FORCE_ELEVATION, forceElevation);
try {
synchronized (elevationCache) {
elevationCache.clear();
String f = initArgs.get(CONFIG_FILE);
if (f == null) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"QueryElevationComponent must specify argument: '"
+ CONFIG_FILE
+ "' -- path to elevate.xml");
}
boolean exists = false;
// check if using ZooKeeper
ZkController zkController = core.getCoreDescriptor().getCoreContainer().getZkController();
if (zkController != null) {
// TODO : shouldn't have to keep reading the config name when it has been read before
exists =
zkController.configFileExists(
zkController
.getZkStateReader()
.readConfigName(
core.getCoreDescriptor().getCloudDescriptor().getCollectionName()),
f);
} else {
File fC = new File(core.getResourceLoader().getConfigDir(), f);
File fD = new File(core.getDataDir(), f);
if (fC.exists() == fD.exists()) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"QueryElevationComponent missing config file: '"
+ f
+ "\n"
+ "either: "
+ fC.getAbsolutePath()
+ " or "
+ fD.getAbsolutePath()
+ " must exist, but not both.");
}
if (fC.exists()) {
exists = true;
log.info("Loading QueryElevation from: " + fC.getAbsolutePath());
Config cfg = new Config(core.getResourceLoader(), f);
elevationCache.put(null, loadElevationMap(cfg));
}
}
// in other words, we think this is in the data dir, not the conf dir
if (!exists) {
// preload the first data
RefCounted<SolrIndexSearcher> searchHolder = null;
try {
searchHolder = core.getNewestSearcher(false);
IndexReader reader = searchHolder.get().getIndexReader();
getElevationMap(reader, core);
} finally {
if (searchHolder != null) searchHolder.decref();
}
}
}
} catch (Exception ex) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Error initializing QueryElevationComponent.", ex);
}
}