/**
* unregisters the Engines service registration, closes the SolrCore and rests the fields. If no
* engine is registered this does nothing!
*/
private void unregisterEngine() {
// use local copies for method calls to avoid concurrency issues
ServiceRegistration engineRegistration = this.engineRegistration;
if (engineRegistration != null) {
log.info(" ... unregister Lucene FSTLinkingEngine {}", engineName);
engineRegistration.unregister();
this.engineRegistration = null; // reset the field
}
solrServerReference = null;
SolrCore solrServer = this.solrCore;
if (solrServer != null) {
log.debug(" ... unregister SolrCore {}", solrServer.getName());
solrServer.close(); // decrease the reference count!!
this.solrCore = null; // rest the field
}
// deactivate the index configuration if present
if (indexConfig != null) {
log.debug(" ... deactivate IndexingConfiguration");
indexConfig.deactivate();
// close the EntityCacheManager (if present
EntityCacheManager cacheManager = indexConfig.getEntityCacheManager();
if (cacheManager != null) {
log.debug(" ... deactivate {}", cacheManager.getClass().getSimpleName());
cacheManager.close();
}
indexConfig = null;
}
}
/**
* Extract core names.
*
* @param cores All the available cores.
* @return The core's name.
*/
private static Iterable<String> coreNames(final CoreContainer cores) {
List<String> names = new ArrayList<String>();
for (SolrCore core : cores.getCores()) {
String name = core.getName();
if (isEmpty(name)) {
// This is the default core.
name = cores.getDefaultCoreName();
}
names.add(name);
}
return names;
}
@Override
public String lookup(String key) {
if ("solr-data-dir".equals(key)) {
return core.getDataDir();
} else if ("solr-index-dir".equals(key)) {
return core.getIndexDir();
} else if ("solr-server-name".equals(key)) {
return indexRef.getServer();
} else if ("solr-core-name".equals(key)) {
return core.getName();
} else {
return bc.getProperty(key);
}
}
private static Runnable getListener(SolrCore core, ZkSolrResourceLoader zkSolrResourceLoader) {
final String coreName = core.getName();
final CoreContainer cc = core.getCoreDescriptor().getCoreContainer();
final String overlayPath =
zkSolrResourceLoader.getConfigSetZkPath() + "/" + ConfigOverlay.RESOURCE_NAME;
final String solrConfigPath =
zkSolrResourceLoader.getConfigSetZkPath() + "/" + core.getSolrConfig().getName();
String schemaRes = null;
if (core.getLatestSchema().isMutable()
&& core.getLatestSchema() instanceof ManagedIndexSchema) {
ManagedIndexSchema mis = (ManagedIndexSchema) core.getLatestSchema();
schemaRes = mis.getResourceName();
}
final String managedSchmaResourcePath =
schemaRes == null ? null : zkSolrResourceLoader.getConfigSetZkPath() + "/" + schemaRes;
return new Runnable() {
@Override
public void run() {
log.info("config update listener called for core {}", coreName);
SolrZkClient zkClient = cc.getZkController().getZkClient();
int solrConfigversion, overlayVersion, managedSchemaVersion = 0;
SolrConfig cfg = null;
try (SolrCore core = cc.getCore(coreName)) {
if (core.isClosed()) return;
cfg = core.getSolrConfig();
solrConfigversion = core.getSolrConfig().getOverlay().getZnodeVersion();
overlayVersion = core.getSolrConfig().getZnodeVersion();
if (managedSchmaResourcePath != null) {
managedSchemaVersion =
((ManagedIndexSchema) core.getLatestSchema()).getSchemaZkVersion();
}
}
if (cfg != null) {
cfg.refreshRequestParams();
}
if (checkStale(zkClient, overlayPath, solrConfigversion)
|| checkStale(zkClient, solrConfigPath, overlayVersion)
|| checkStale(zkClient, managedSchmaResourcePath, managedSchemaVersion)) {
log.info("core reload {}", coreName);
cc.reload(coreName);
}
}
};
}
/**
* Handle 'CREATE' action.
*
* @param req
* @param rsp
* @return true if a modification has resulted that requires persistance of the CoreContainer
* configuration.
* @throws SolrException in case of a configuration error.
*/
protected boolean handleCreateAction(SolrQueryRequest req, SolrQueryResponse rsp)
throws SolrException {
try {
SolrParams params = req.getParams();
String name = params.get(CoreAdminParams.NAME);
CoreDescriptor dcore =
new CoreDescriptor(coreContainer, name, params.get(CoreAdminParams.INSTANCE_DIR));
// fillup optional parameters
String opts = params.get(CoreAdminParams.CONFIG);
if (opts != null) dcore.setConfigName(opts);
opts = params.get(CoreAdminParams.SCHEMA);
if (opts != null) dcore.setSchemaName(opts);
opts = params.get(CoreAdminParams.DATA_DIR);
if (opts != null) dcore.setDataDir(opts);
// Process all property.name=value parameters and set them as name=value core properties
Properties coreProperties = new Properties();
Iterator<String> parameterNamesIterator = params.getParameterNamesIterator();
while (parameterNamesIterator.hasNext()) {
String parameterName = parameterNamesIterator.next();
if (parameterName.startsWith(CoreAdminParams.PROPERTY_PREFIX)) {
String parameterValue = params.get(parameterName);
String propertyName =
parameterName.substring(CoreAdminParams.PROPERTY_PREFIX.length()); // skip prefix
coreProperties.put(propertyName, parameterValue);
}
}
dcore.setCoreProperties(coreProperties);
SolrCore core = coreContainer.create(dcore);
coreContainer.register(name, core, false);
rsp.add("core", core.getName());
return coreContainer.isPersistent();
} catch (Exception ex) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Error executing default implementation of CREATE",
ex);
}
}
private void checkInstanceDirs(JettySolrRunner jetty) {
CoreContainer cores = ((SolrDispatchFilter) jetty.getDispatchFilter().getFilter()).getCores();
Collection<SolrCore> theCores = cores.getCores();
for (SolrCore core : theCores) {
if (!oldStyleSolrXml) {
// look for core props file
assertTrue(
"Could not find expected core.properties file",
new File((String) core.getStatistics().get("instanceDir"), "core.properties").exists());
}
assertEquals(
new File(
SolrResourceLoader.normalizeDir(
jetty.getSolrHome() + File.separator + core.getName()))
.getAbsolutePath(),
new File(
SolrResourceLoader.normalizeDir((String) core.getStatistics().get("instanceDir")))
.getAbsolutePath());
}
}
protected NamedList<Object> getCoreStatus(CoreContainer cores, String cname) throws IOException {
NamedList<Object> info = new SimpleOrderedMap<Object>();
SolrCore core = cores.getCore(cname);
if (core != null) {
try {
info.add("name", core.getName());
info.add("instanceDir", normalizePath(core.getResourceLoader().getInstanceDir()));
info.add("dataDir", normalizePath(core.getDataDir()));
info.add("startTime", new Date(core.getStartTime()));
info.add("uptime", System.currentTimeMillis() - core.getStartTime());
RefCounted<SolrIndexSearcher> searcher = core.getSearcher();
try {
info.add("index", LukeRequestHandler.getIndexInfo(searcher.get().getReader(), false));
} finally {
searcher.decref();
}
} finally {
core.close();
}
}
return info;
}
/**
* This will be called on each
*
* <ul>
* <li>update to the Component configuration (activate, deactivate)
* <li>updates on the SolrCore
* </ul>
*
* on any detected change it will update the registered EnhancementEngine.
*
* <p>This also initialises the FST configuration.
*
* @param reference the ServiceRefernece for the SolrServer or <code>null</code> in case the
* service is no longer available.
* @param server the SolrServer (or <code>null</code>
*/
protected void updateEngineRegistration(ServiceReference reference, SolrServer server) {
log.info(" ... updateEngineRegistration for {}: {}", getClass().getSimpleName(), engineName);
if (reference != null && server == null) {
server = solrServerTracker.getService(reference);
}
if (reference == null && this.indexReference == null) {
// unregisterEngine(); //unregister existing
return; // and return
}
BundleContext bundleContext = this.bundleContext;
synchronized (this) { // init one after the other in case of multiple calls
SolrCore core;
IndexConfiguration indexConfig; // the indexConfig build by this call
try { // try to init - finally unregisterEngine
if (bundleContext == null) { // already deactivated
return; // NOTE: unregistering is done in finally block
}
if (reference != null) {
if (reference.equals(this.solrServerReference)) {
// use the current core
core = solrCore;
} else { // get the SolrCore from the EmbeddedSolrServer
core = getSolrCore(server);
}
} else { // SolrCore not available
core = null;
}
if (core == null) { // no SolrCore
log.info(" - SolrCore {} present", this.solrCore == null ? "not yet" : "no longer");
return; // NOTE: unregistering is done in finally block
} // else - we do have a SolrCore
// File fstDir = new File(dataDir,"fst");
// now collect the FST configuration
indexConfig = new IndexConfiguration(fstConfig, core, fieldEncoding);
indexConfig.setTypeField(solrTypeField);
indexConfig.setRankingField(solrRankingField);
// set fields parsed in the activate method
indexConfig.setExecutorService(fstCreatorService);
indexConfig.setRedirectField(null); // TODO add support
// NOTE: the FST cofnig is processed even if the SolrCore has not changed
// because their might be config changes and/or new FST files in the
// FST directory of the SolrCore.
indexConfig.setFstDirectory(getFstDirectory(core, fstFolder));
// set the DocumentCacheFactory
if (entityCacheSize > 0) {
indexConfig.setEntityCacheManager(new FastLRUCacheManager(entityCacheSize));
} // else no entityCache is used
if (skipAltTokensConfig != null) {
indexConfig.setSkipAltTokens(skipAltTokensConfig);
}
// create a new searcher for creating FSTs
if (!indexConfig.activate()) {
log.warn(
"Processing of the FST configuration was not successfull "
+ "for any language. See WARN level loggings for more details!");
log.warn(
" ... FstLinkingEnigne wiht name {} will be registered but"
+ "be inactive as there seam to be no data for linking available"
+ "in the SolrCore {} (dir: {})",
new Object[] {engineName, core.getName(), core.getCoreDescriptor().getInstanceDir()});
} else { // some FST corpora initialised
if (log.isInfoEnabled()) { // log the initialised languages
Set<String> langSet = new HashSet<String>(indexConfig.getCorpusLanguages());
if (langSet.remove(null)) { // replace the null for the default language
langSet.add(""); // with an empty string
}
String[] langArray = langSet.toArray(new String[langSet.size()]);
Arrays.sort(langArray, String.CASE_INSENSITIVE_ORDER);
log.info(" ... initialised FST corpora for languages {}", Arrays.toString(langArray));
}
}
} finally {
// in any case (even an Exception) ensure that the current
// engine registration is unregistered and the currently used
// SolrCore is unregistered!
unregisterEngine();
}
// check if we need to create some FST files
for (CorpusInfo fstInfo : indexConfig.getCorpora()) {
// check if the fst does not exist and the fstInfo allows creation
if (!fstInfo.fst.exists() && fstInfo.allowCreation) {
// create a task on the FST corpus creation service
fstCreatorService.execute(new CorpusCreationTask(indexConfig, fstInfo));
}
}
// set the default linking corpora
String defaultLanguage = entityLinkerConfig.getDefaultLanguage();
if (defaultLanguage == null) {
defaultLanguage = ""; // FST uses an empty string for the default
}
CorpusInfo defaultCoprous = indexConfig.getCorpus(defaultLanguage);
if (defaultCoprous != null) {
log.info(
" ... set '{}' as default FST Corpus: {}", defaultCoprous.language, defaultCoprous);
indexConfig.setDefaultCorpus(defaultCoprous);
} else {
log.info(" ... no corpus for default language {} available", defaultCoprous);
}
// set the index configuration to the field;
this.indexConfig = indexConfig;
FstLinkingEngine engine =
new FstLinkingEngine(engineName, indexConfig, textProcessingConfig, entityLinkerConfig);
String[] services =
new String[] {EnhancementEngine.class.getName(), ServiceProperties.class.getName()};
log.info(" ... register {}: {}", engine.getClass().getSimpleName(), engineName);
this.engineRegistration = bundleContext.registerService(services, engine, engineMetadata);
this.solrServerReference = reference;
this.solrCore = core;
}
}
public void inform(SolrCore core) {
/* The stream factory will always contain the zkUrl for the given collection
* Adds default streams with their corresponding function names. These
* defaults can be overridden or added to in the solrConfig in the stream
* RequestHandler def. Example config override
* <lst name="streamFunctions">
* <str name="group">org.apache.solr.client.solrj.io.stream.ReducerStream</str>
* <str name="count">org.apache.solr.client.solrj.io.stream.RecordCountStream</str>
* </lst>
* */
String defaultCollection;
String defaultZkhost;
CoreContainer coreContainer = core.getCoreDescriptor().getCoreContainer();
this.coreName = core.getName();
if (coreContainer.isZooKeeperAware()) {
defaultCollection = core.getCoreDescriptor().getCollectionName();
defaultZkhost =
core.getCoreDescriptor().getCoreContainer().getZkController().getZkServerAddress();
streamFactory.withCollectionZkHost(defaultCollection, defaultZkhost);
streamFactory.withDefaultZkHost(defaultZkhost);
modelCache = new ModelCache(250, defaultZkhost, clientCache);
}
streamFactory
// source streams
.withFunctionName("search", CloudSolrStream.class)
.withFunctionName("facet", FacetStream.class)
.withFunctionName("update", UpdateStream.class)
.withFunctionName("jdbc", JDBCStream.class)
.withFunctionName("topic", TopicStream.class)
.withFunctionName("commit", CommitStream.class)
.withFunctionName("random", RandomStream.class)
// decorator streams
.withFunctionName("merge", MergeStream.class)
.withFunctionName("unique", UniqueStream.class)
.withFunctionName("top", RankStream.class)
.withFunctionName("group", GroupOperation.class)
.withFunctionName("reduce", ReducerStream.class)
.withFunctionName("parallel", ParallelStream.class)
.withFunctionName("rollup", RollupStream.class)
.withFunctionName("stats", StatsStream.class)
.withFunctionName("innerJoin", InnerJoinStream.class)
.withFunctionName("leftOuterJoin", LeftOuterJoinStream.class)
.withFunctionName("hashJoin", HashJoinStream.class)
.withFunctionName("outerHashJoin", OuterHashJoinStream.class)
.withFunctionName("intersect", IntersectStream.class)
.withFunctionName("complement", ComplementStream.class)
.withFunctionName("sort", SortStream.class)
.withFunctionName("train", TextLogitStream.class)
.withFunctionName("features", FeaturesSelectionStream.class)
.withFunctionName("daemon", DaemonStream.class)
.withFunctionName("shortestPath", ShortestPathStream.class)
.withFunctionName("gatherNodes", GatherNodesStream.class)
.withFunctionName("nodes", GatherNodesStream.class)
.withFunctionName("select", SelectStream.class)
.withFunctionName("scoreNodes", ScoreNodesStream.class)
.withFunctionName("model", ModelStream.class)
.withFunctionName("classify", ClassifyStream.class)
.withFunctionName("fetch", FetchStream.class)
.withFunctionName("executor", ExecutorStream.class)
.withFunctionName("null", NullStream.class)
.withFunctionName("priority", PriorityStream.class)
// metrics
.withFunctionName("min", MinMetric.class)
.withFunctionName("max", MaxMetric.class)
.withFunctionName("avg", MeanMetric.class)
.withFunctionName("sum", SumMetric.class)
.withFunctionName("count", CountMetric.class)
// tuple manipulation operations
.withFunctionName("replace", ReplaceOperation.class)
.withFunctionName("concat", ConcatOperation.class)
// stream reduction operations
.withFunctionName("group", GroupOperation.class)
.withFunctionName("distinct", DistinctOperation.class)
.withFunctionName("having", HavingStream.class)
.withFunctionName("and", AndOperation.class)
.withFunctionName("or", OrOperation.class)
.withFunctionName("not", NotOperation.class)
.withFunctionName("gt", GreaterThanOperation.class)
.withFunctionName("lt", LessThanOperation.class)
.withFunctionName("eq", EqualsOperation.class)
.withFunctionName("lteq", LessThanEqualToOperation.class)
.withFunctionName("gteq", GreaterThanEqualToOperation.class);
// This pulls all the overrides and additions from the config
List<PluginInfo> pluginInfos = core.getSolrConfig().getPluginInfos(Expressible.class.getName());
for (PluginInfo pluginInfo : pluginInfos) {
Class<? extends Expressible> clazz =
core.getResourceLoader().findClass(pluginInfo.className, Expressible.class);
streamFactory.withFunctionName(pluginInfo.name, clazz);
}
core.addCloseHook(
new CloseHook() {
@Override
public void preClose(SolrCore core) {
// To change body of implemented methods use File | Settings | File Templates.
}
@Override
public void postClose(SolrCore core) {
clientCache.close();
}
});
}
public String _format(LoggingEvent event) {
String message = (String) event.getMessage();
if (message == null) {
message = "";
}
StringBuilder sb = new StringBuilder(message.length() + 80);
long now = event.timeStamp;
long timeFromStart = now - startTime;
long timeSinceLast = now - lastTime;
lastTime = now;
String shortClassName =
getShortClassName(
event.getLocationInformation().getClassName(),
event.getLocationInformation().getMethodName());
/**
* * sb.append(timeFromStart).append(' ').append(timeSinceLast); sb.append(' ');
* sb.append(record.getSourceClassName()).append('.').append( record.getSourceMethodName());
* sb.append(' '); sb.append(record.getLevel()); *
*/
SolrRequestInfo requestInfo = SolrRequestInfo.getRequestInfo();
SolrQueryRequest req = requestInfo == null ? null : requestInfo.getReq();
SolrCore core = req == null ? null : req.getCore();
ZkController zkController = null;
CoreInfo info = null;
if (core != null) {
info = coreInfoMap.get(core.hashCode());
if (info == null) {
info = new CoreInfo();
info.shortId = "C" + Integer.toString(CoreInfo.maxCoreNum++);
coreInfoMap.put(core.hashCode(), info);
if (sb.length() == 0) sb.append("ASYNC ");
sb.append(" NEW_CORE " + info.shortId);
sb.append(" name=" + core.getName());
sb.append(" " + core);
}
zkController = core.getCoreDescriptor().getCoreContainer().getZkController();
if (zkController != null) {
if (info.url == null) {
info.url = zkController.getBaseUrl() + "/" + core.getName();
sb.append(" url=" + info.url + " node=" + zkController.getNodeName());
}
Map<String, Object> coreProps = getReplicaProps(zkController, core);
if (info.coreProps == null || !coreProps.equals(info.coreProps)) {
info.coreProps = coreProps;
final String corePropsString =
"coll:"
+ core.getCoreDescriptor().getCloudDescriptor().getCollectionName()
+ " core:"
+ core.getName()
+ " props:"
+ coreProps;
sb.append(" " + info.shortId + "_STATE=" + corePropsString);
}
}
}
if (sb.length() > 0) sb.append('\n');
sb.append(timeFromStart);
// sb.append("\nL").append(record.getSequenceNumber()); // log number is
// useful for sequencing when looking at multiple parts of a log file, but
// ms since start should be fine.
appendThread(sb, event);
appendMDC(sb);
// todo: should be able to get port from core container for non zk tests
if (info != null) {
sb.append(' ').append(info.shortId); // core
}
if (shortClassName.length() > 0) {
sb.append(' ').append(shortClassName);
}
if (event.getLevel() != Level.INFO) {
sb.append(' ').append(event.getLevel());
}
sb.append(' ');
appendMultiLineString(sb, message);
ThrowableInformation thInfo = event.getThrowableInformation();
if (thInfo != null) {
Throwable th = event.getThrowableInformation().getThrowable();
if (th != null) {
sb.append(' ');
String err = SolrException.toStr(th);
String ignoredMsg = SolrException.doIgnore(th, err);
if (ignoredMsg != null) {
sb.append(ignoredMsg);
} else {
sb.append(err);
}
}
}
sb.append('\n');
/**
* * Isn't core specific... prob better logged from zkController if (info != null) {
* ClusterState clusterState = zkController.getClusterState(); if (info.clusterState !=
* clusterState) { // something has changed in the matrix... sb.append(zkController.getBaseUrl()
* + " sees new ClusterState:"); } } *
*/
return sb.toString();
}
