// Determine that when the query lists are commented out of both new and
// first searchers in the config, we don't throw an NPE
@Test
public void testSearcherEvents() throws Exception {
SolrCore core = h.getCore();
SolrEventListener newSearcherListener = core.newSearcherListeners.get(0);
assertTrue(
"Not an instance of QuerySenderListener",
newSearcherListener instanceof QuerySenderListener);
QuerySenderListener qsl = (QuerySenderListener) newSearcherListener;
RefCounted<SolrIndexSearcher> currentSearcherRef = core.getSearcher();
SolrIndexSearcher currentSearcher = currentSearcherRef.get();
SolrIndexSearcher dummy = null;
qsl.newSearcher(currentSearcher, dummy); // test first Searcher (since param is null)
MockQuerySenderListenerReqHandler mock =
(MockQuerySenderListenerReqHandler) core.getRequestHandler("mock");
assertNotNull("Mock is null", mock);
assertNull("Req (firstsearcher) is not null", mock.req);
SolrIndexSearcher newSearcher =
new SolrIndexSearcher(
core,
core.getNewIndexDir(),
core.getLatestSchema(),
core.getSolrConfig().indexConfig,
"testQuerySenderNoQuery",
false,
core.getDirectoryFactory());
qsl.newSearcher(newSearcher, currentSearcher); // get newSearcher.
assertNull("Req (newsearcher) is not null", mock.req);
newSearcher.close();
currentSearcherRef.decref();
}
public void testDocValues() throws IOException {
assertU(adoc("id", "1"));
assertU(commit());
try (SolrCore core = h.getCoreInc()) {
final RefCounted<SolrIndexSearcher> searcherRef = core.openNewSearcher(true, true);
final SolrIndexSearcher searcher = searcherRef.get();
try {
final LeafReader reader = searcher.getLeafReader();
assertEquals(1, reader.numDocs());
final FieldInfos infos = reader.getFieldInfos();
assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("floatdv").getDocValuesType());
assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("intdv").getDocValuesType());
assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("doubledv").getDocValuesType());
assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("longdv").getDocValuesType());
assertEquals(DocValuesType.SORTED, infos.fieldInfo("stringdv").getDocValuesType());
assertEquals((long) Float.floatToIntBits(1), reader.getNumericDocValues("floatdv").get(0));
assertEquals(2L, reader.getNumericDocValues("intdv").get(0));
assertEquals(Double.doubleToLongBits(3), reader.getNumericDocValues("doubledv").get(0));
assertEquals(4L, reader.getNumericDocValues("longdv").get(0));
final IndexSchema schema = core.getLatestSchema();
final SchemaField floatDv = schema.getField("floatdv");
final SchemaField intDv = schema.getField("intdv");
final SchemaField doubleDv = schema.getField("doubledv");
final SchemaField longDv = schema.getField("longdv");
FunctionValues values =
floatDv
.getType()
.getValueSource(floatDv, null)
.getValues(null, searcher.getLeafReader().leaves().get(0));
assertEquals(1f, values.floatVal(0), 0f);
assertEquals(1f, values.objectVal(0));
values =
intDv
.getType()
.getValueSource(intDv, null)
.getValues(null, searcher.getLeafReader().leaves().get(0));
assertEquals(2, values.intVal(0));
assertEquals(2, values.objectVal(0));
values =
doubleDv
.getType()
.getValueSource(doubleDv, null)
.getValues(null, searcher.getLeafReader().leaves().get(0));
assertEquals(3d, values.doubleVal(0), 0d);
assertEquals(3d, values.objectVal(0));
values =
longDv
.getType()
.getValueSource(longDv, null)
.getValues(null, searcher.getLeafReader().leaves().get(0));
assertEquals(4L, values.longVal(0));
assertEquals(4L, values.objectVal(0));
} finally {
searcherRef.decref();
}
}
}
@Override
public Query rewrite(IndexReader reader) throws IOException {
SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
CoreContainer container = info.getReq().getCore().getCoreDescriptor().getCoreContainer();
final SolrCore fromCore = container.getCore(fromIndex);
if (fromCore == null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Cross-core join: no such core " + fromIndex);
}
RefCounted<SolrIndexSearcher> fromHolder = null;
fromHolder = fromCore.getRegisteredSearcher();
final Query joinQuery;
try {
joinQuery =
JoinUtil.createJoinQuery(
fromField, true, toField, fromQuery, fromHolder.get(), scoreMode);
} finally {
fromCore.close();
fromHolder.decref();
}
return joinQuery.rewrite(reader);
}
@Override
public long getSize() {
RefCounted<SolrIndexSearcher> refCountedIndexSearcher = this.core.getSearcher();
SolrIndexSearcher searcher = refCountedIndexSearcher.get();
DirectoryReader reader = searcher.getIndexReader();
long numDocs = reader.numDocs();
refCountedIndexSearcher.decref();
return numDocs;
}
public static SolrInputDocument getInputDocument(SolrCore core, BytesRef idBytes)
throws IOException {
SolrInputDocument sid = null;
RefCounted<SolrIndexSearcher> searcherHolder = null;
try {
SolrIndexSearcher searcher = null;
UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
if (ulog != null) {
Object o = ulog.lookup(idBytes);
if (o != null) {
// should currently be a List<Oper,Ver,Doc/Id>
List entry = (List) o;
assert entry.size() >= 3;
int oper = (Integer) entry.get(0) & UpdateLog.OPERATION_MASK;
switch (oper) {
case UpdateLog.ADD:
sid = (SolrInputDocument) entry.get(entry.size() - 1);
break;
case UpdateLog.DELETE:
return null;
default:
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Unknown Operation! " + oper);
}
}
}
if (sid == null) {
// didn't find it in the update log, so it should be in the newest searcher opened
if (searcher == null) {
searcherHolder = core.getRealtimeSearcher();
searcher = searcherHolder.get();
}
// SolrCore.verbose("RealTimeGet using searcher ", searcher);
SchemaField idField = core.getLatestSchema().getUniqueKeyField();
int docid = searcher.getFirstMatch(new Term(idField.getName(), idBytes));
if (docid < 0) return null;
StoredDocument luceneDocument = searcher.doc(docid);
sid = toSolrInputDocument(luceneDocument, core.getLatestSchema());
}
} finally {
if (searcherHolder != null) {
searcherHolder.decref();
}
}
return sid;
}
@Override
public void run() {
while (!core.isClosed()) {
try {
RefCounted<SolrIndexSearcher> newSearcher = null;
try {
newSearcher = core.openNewSearcher(true, true);
} finally {
if (newSearcher != null) {
newSearcher.decref();
}
}
} catch (SolrException e) {
if (!core.isClosed()) {
throw e;
}
}
}
}
boolean waitForNewSearcher(int timeout) {
long timeoutTime = System.currentTimeMillis() + timeout;
while (System.currentTimeMillis() < timeoutTime) {
if (triggered) {
// check if the new searcher has been registered yet
RefCounted<SolrIndexSearcher> registeredSearcherH = newSearcher.getCore().getSearcher();
SolrIndexSearcher registeredSearcher = registeredSearcherH.get();
registeredSearcherH.decref();
if (registeredSearcher == newSearcher) return true;
// log.info("TEST: waiting for searcher " + newSearcher + " to be registered. current=" +
// registeredSearcher);
}
try {
Thread.sleep(250);
} catch (InterruptedException e) {
}
}
return false;
}
protected NamedList<Object> getCoreStatus(CoreContainer cores, String cname) throws IOException {
NamedList<Object> info = new SimpleOrderedMap<Object>();
SolrCore core = cores.getCore(cname);
if (core != null) {
try {
info.add("name", core.getName());
info.add("instanceDir", normalizePath(core.getResourceLoader().getInstanceDir()));
info.add("dataDir", normalizePath(core.getDataDir()));
info.add("startTime", new Date(core.getStartTime()));
info.add("uptime", System.currentTimeMillis() - core.getStartTime());
RefCounted<SolrIndexSearcher> searcher = core.getSearcher();
try {
info.add("index", LukeRequestHandler.getIndexInfo(searcher.get().getReader(), false));
} finally {
searcher.decref();
}
} finally {
core.close();
}
}
return info;
}
protected boolean handleMergeAction(SolrQueryRequest req, SolrQueryResponse rsp)
throws IOException {
SolrParams params = req.getParams();
String cname = params.required().get(CoreAdminParams.CORE);
SolrCore core = coreContainer.getCore(cname);
SolrCore[] sourceCores = null;
RefCounted<SolrIndexSearcher>[] searchers = null;
// stores readers created from indexDir param values
IndexReader[] readersToBeClosed = null;
if (core != null) {
try {
String[] dirNames = params.getParams(CoreAdminParams.INDEX_DIR);
if (dirNames == null || dirNames.length == 0) {
String[] sources = params.getParams("srcCore");
if (sources == null || sources.length == 0)
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"At least one indexDir or srcCore must be specified");
sourceCores = new SolrCore[sources.length];
for (int i = 0; i < sources.length; i++) {
String source = sources[i];
SolrCore srcCore = coreContainer.getCore(source);
if (srcCore == null)
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Core: " + source + " does not exist");
sourceCores[i] = srcCore;
}
} else {
readersToBeClosed = new IndexReader[dirNames.length];
DirectoryFactory dirFactory = core.getDirectoryFactory();
for (int i = 0; i < dirNames.length; i++) {
readersToBeClosed[i] = IndexReader.open(dirFactory.open(dirNames[i]), true);
}
}
IndexReader[] readers = null;
if (readersToBeClosed != null) {
readers = readersToBeClosed;
} else {
readers = new IndexReader[sourceCores.length];
searchers = new RefCounted[sourceCores.length];
for (int i = 0; i < sourceCores.length; i++) {
SolrCore solrCore = sourceCores[i];
// record the searchers so that we can decref
searchers[i] = solrCore.getSearcher();
readers[i] = searchers[i].get().getIndexReader();
}
}
UpdateRequestProcessorChain processorChain =
core.getUpdateProcessingChain(params.get(UpdateParams.UPDATE_CHAIN));
SolrQueryRequest wrappedReq = new LocalSolrQueryRequest(core, req.getParams());
UpdateRequestProcessor processor = processorChain.createProcessor(wrappedReq, rsp);
processor.processMergeIndexes(new MergeIndexesCommand(readers));
} finally {
if (searchers != null) {
for (RefCounted<SolrIndexSearcher> searcher : searchers) {
if (searcher != null) searcher.decref();
}
}
if (sourceCores != null) {
for (SolrCore solrCore : sourceCores) {
if (solrCore != null) solrCore.close();
}
}
if (readersToBeClosed != null) IOUtils.closeWhileHandlingException(readersToBeClosed);
core.close();
}
}
return coreContainer.isPersistent();
}
@Override
public void inform(SolrCore core) {
IndexSchema schema = core.getLatestSchema();
String a = initArgs.get(FIELD_TYPE);
if (a != null) {
FieldType ft = schema.getFieldTypes().get(a);
if (ft == null) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"Unknown FieldType: '" + a + "' used in QueryElevationComponent");
}
analyzer = ft.getQueryAnalyzer();
}
SchemaField sf = schema.getUniqueKeyField();
if (sf == null) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"QueryElevationComponent requires the schema to have a uniqueKeyField.");
}
idSchemaFT = sf.getType();
idField = sf.getName();
// register the EditorialMarkerFactory
String excludeName = initArgs.get(QueryElevationParams.EXCLUDE_MARKER_FIELD_NAME, "excluded");
if (excludeName == null || excludeName.equals("") == true) {
excludeName = "excluded";
}
ExcludedMarkerFactory excludedMarkerFactory = new ExcludedMarkerFactory();
core.addTransformerFactory(excludeName, excludedMarkerFactory);
ElevatedMarkerFactory elevatedMarkerFactory = new ElevatedMarkerFactory();
String markerName = initArgs.get(QueryElevationParams.EDITORIAL_MARKER_FIELD_NAME, "elevated");
if (markerName == null || markerName.equals("") == true) {
markerName = "elevated";
}
core.addTransformerFactory(markerName, elevatedMarkerFactory);
forceElevation = initArgs.getBool(QueryElevationParams.FORCE_ELEVATION, forceElevation);
try {
synchronized (elevationCache) {
elevationCache.clear();
String f = initArgs.get(CONFIG_FILE);
if (f == null) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"QueryElevationComponent must specify argument: '"
+ CONFIG_FILE
+ "' -- path to elevate.xml");
}
boolean exists = false;
// check if using ZooKeeper
ZkController zkController = core.getCoreDescriptor().getCoreContainer().getZkController();
if (zkController != null) {
// TODO : shouldn't have to keep reading the config name when it has been read before
exists =
zkController.configFileExists(
zkController
.getZkStateReader()
.readConfigName(
core.getCoreDescriptor().getCloudDescriptor().getCollectionName()),
f);
} else {
File fC = new File(core.getResourceLoader().getConfigDir(), f);
File fD = new File(core.getDataDir(), f);
if (fC.exists() == fD.exists()) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"QueryElevationComponent missing config file: '"
+ f
+ "\n"
+ "either: "
+ fC.getAbsolutePath()
+ " or "
+ fD.getAbsolutePath()
+ " must exist, but not both.");
}
if (fC.exists()) {
exists = true;
log.info("Loading QueryElevation from: " + fC.getAbsolutePath());
Config cfg = new Config(core.getResourceLoader(), f);
elevationCache.put(null, loadElevationMap(cfg));
}
}
// in other words, we think this is in the data dir, not the conf dir
if (!exists) {
// preload the first data
RefCounted<SolrIndexSearcher> searchHolder = null;
try {
searchHolder = core.getNewestSearcher(false);
IndexReader reader = searchHolder.get().getIndexReader();
getElevationMap(reader, core);
} finally {
if (searchHolder != null) searchHolder.decref();
}
}
}
} catch (Exception ex) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Error initializing QueryElevationComponent.", ex);
}
}
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
String val = params.get("getVersions");
if (val != null) {
processGetVersions(rb);
return;
}
val = params.get("getUpdates");
if (val != null) {
processGetUpdates(rb);
return;
}
String id[] = params.getParams("id");
String ids[] = params.getParams("ids");
if (id == null && ids == null) {
return;
}
String[] allIds = id == null ? new String[0] : id;
if (ids != null) {
List<String> lst = new ArrayList<String>();
for (String s : allIds) {
lst.add(s);
}
for (String idList : ids) {
lst.addAll(StrUtils.splitSmart(idList, ",", true));
}
allIds = lst.toArray(new String[lst.size()]);
}
SolrCore core = req.getCore();
SchemaField idField = core.getLatestSchema().getUniqueKeyField();
FieldType fieldType = idField.getType();
SolrDocumentList docList = new SolrDocumentList();
UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
RefCounted<SolrIndexSearcher> searcherHolder = null;
DocTransformer transformer = rsp.getReturnFields().getTransformer();
if (transformer != null) {
TransformContext context = new TransformContext();
context.req = req;
transformer.setContext(context);
}
try {
SolrIndexSearcher searcher = null;
BytesRef idBytes = new BytesRef();
for (String idStr : allIds) {
fieldType.readableToIndexed(idStr, idBytes);
if (ulog != null) {
Object o = ulog.lookup(idBytes);
if (o != null) {
// should currently be a List<Oper,Ver,Doc/Id>
List entry = (List) o;
assert entry.size() >= 3;
int oper = (Integer) entry.get(0) & UpdateLog.OPERATION_MASK;
switch (oper) {
case UpdateLog.ADD:
SolrDocument doc =
toSolrDoc(
(SolrInputDocument) entry.get(entry.size() - 1), core.getLatestSchema());
if (transformer != null) {
transformer.transform(doc, -1); // unknown docID
}
docList.add(doc);
break;
case UpdateLog.DELETE:
break;
default:
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Unknown Operation! " + oper);
}
continue;
}
}
// didn't find it in the update log, so it should be in the newest searcher opened
if (searcher == null) {
searcherHolder = core.getRealtimeSearcher();
searcher = searcherHolder.get();
}
// SolrCore.verbose("RealTimeGet using searcher ", searcher);
int docid = searcher.getFirstMatch(new Term(idField.getName(), idBytes));
if (docid < 0) continue;
StoredDocument luceneDocument = searcher.doc(docid);
SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
if (transformer != null) {
transformer.transform(doc, docid);
}
docList.add(doc);
}
} finally {
if (searcherHolder != null) {
searcherHolder.decref();
}
}
// if the client specified a single id=foo, then use "doc":{
// otherwise use a standard doclist
if (ids == null && allIds.length <= 1) {
// if the doc was not found, then use a value of null.
rsp.add("doc", docList.size() > 0 ? docList.get(0) : null);
} else {
docList.setNumFound(docList.size());
rsp.add("response", docList);
}
}