private void handleGET() {
if (parts.size() == 1) {
resp.add("solrConfig", req.getCore().getSolrConfig().toMap());
} else {
if (ConfigOverlay.NAME.equals(parts.get(1))) {
resp.add(ConfigOverlay.NAME, req.getCore().getSolrConfig().getOverlay().toMap());
} else if (RequestParams.NAME.equals(parts.get(1))) {
if (parts.size() == 3) {
RequestParams params = req.getCore().getSolrConfig().getRequestParams();
MapSolrParams p = params.getParams(parts.get(2));
Map m = new LinkedHashMap<>();
m.put(ConfigOverlay.ZNODEVER, params.getZnodeVersion());
if (p != null) {
m.put(RequestParams.NAME, ZkNodeProps.makeMap(parts.get(2), p.getMap()));
}
resp.add(SolrQueryResponse.NAME, m);
} else {
resp.add(
SolrQueryResponse.NAME, req.getCore().getSolrConfig().getRequestParams().toMap());
}
} else {
Map<String, Object> m = req.getCore().getSolrConfig().toMap();
resp.add("solrConfig", ZkNodeProps.makeMap(parts.get(1), m.get(parts.get(1))));
}
}
}
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
// TODO: check for id field?
int hash = 0;
if (zkEnabled) {
zkCheck();
hash = hash(cmd);
nodes = setupRequest(hash);
} else {
isLeader = getNonZkLeaderAssumption(req);
}
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionAdd(cmd);
}
if (dropCmd) {
// TODO: do we need to add anything to the response?
return;
}
ModifiableSolrParams params = null;
if (nodes != null) {
params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(
DISTRIB_UPDATE_PARAM,
(isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
if (isLeader) {
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
}
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
cmdDistrib.distribAdd(cmd, nodes, params);
}
// TODO: what to do when no idField?
if (returnVersions && rsp != null && idField != null) {
if (addsResponse == null) {
addsResponse = new NamedList<String>();
rsp.add("adds", addsResponse);
}
if (scratch == null) scratch = new CharsRef();
idField.getType().indexedToReadable(cmd.getIndexedId(), scratch);
addsResponse.add(scratch.toString(), cmd.getVersion());
}
// TODO: keep track of errors? needs to be done at a higher level though since
// an id may fail before it gets to this processor.
// Given that, it may also make sense to move the version reporting out of this
// processor too.
}
@Override
public void processCommit(CommitUpdateCommand cmd) throws IOException {
if (zkEnabled) {
zkCheck();
}
if (vinfo != null) {
vinfo.lockForUpdate();
}
try {
if (ulog == null
|| ulog.getState() == UpdateLog.State.ACTIVE
|| (cmd.getFlags() & UpdateCommand.REPLAY) != 0) {
super.processCommit(cmd);
} else {
log.info(
"Ignoring commit while not ACTIVE - state: "
+ ulog.getState()
+ " replay:"
+ (cmd.getFlags() & UpdateCommand.REPLAY));
}
} finally {
if (vinfo != null) {
vinfo.unlockForUpdate();
}
}
// TODO: we should consider this? commit everyone in the current collection
if (zkEnabled) {
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
if (!req.getParams().getBool(COMMIT_END_POINT, false)) {
params.set(COMMIT_END_POINT, true);
String nodeName =
req.getCore().getCoreDescriptor().getCoreContainer().getZkController().getNodeName();
String shardZkNodeName = nodeName + "_" + req.getCore().getName();
List<Node> nodes =
getCollectionUrls(
req,
req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(),
shardZkNodeName);
if (nodes != null) {
cmdDistrib.distribCommit(cmd, nodes, params);
finish();
}
}
}
}
private List<Node> getCollectionUrls(
SolrQueryRequest req, String collection, String shardZkNodeName) {
ClusterState clusterState =
req.getCore().getCoreDescriptor().getCoreContainer().getZkController().getClusterState();
List<Node> urls = new ArrayList<Node>();
Map<String, Slice> slices = clusterState.getSlices(collection);
if (slices == null) {
throw new ZooKeeperException(
ErrorCode.BAD_REQUEST, "Could not find collection in zk: " + clusterState);
}
for (Map.Entry<String, Slice> sliceEntry : slices.entrySet()) {
Slice replicas = slices.get(sliceEntry.getKey());
Map<String, Replica> shardMap = replicas.getReplicasMap();
for (Entry<String, Replica> entry : shardMap.entrySet()) {
ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(entry.getValue());
if (clusterState.liveNodesContain(nodeProps.getNodeName())
&& !entry.getKey().equals(shardZkNodeName)) {
urls.add(new StdNode(nodeProps));
}
}
}
if (urls.size() == 0) {
return null;
}
return urls;
}
protected void execute(SolrQueryResponse rsp) {
// a custom filter could add more stuff to the request before passing it on.
// for example: sreq.getContext().put( "HttpServletRequest", req );
// used for logging query stats in SolrCore.execute()
solrReq.getContext().put("webapp", req.getContextPath());
solrReq.getCore().execute(handler, solrReq, rsp);
}
// used for deleteByQuery to get the list of nodes this leader should forward to
private List<Node> setupRequest() {
List<Node> nodes = null;
String shardId = cloudDesc.getShardId();
try {
ZkCoreNodeProps leaderProps =
new ZkCoreNodeProps(zkController.getZkStateReader().getLeaderProps(collection, shardId));
String leaderNodeName = leaderProps.getCoreNodeName();
String coreName = req.getCore().getName();
String coreNodeName = zkController.getNodeName() + "_" + coreName;
isLeader = coreNodeName.equals(leaderNodeName);
// TODO: what if we are no longer the leader?
forwardToLeader = false;
List<ZkCoreNodeProps> replicaProps =
zkController
.getZkStateReader()
.getReplicaProps(collection, shardId, zkController.getNodeName(), coreName);
if (replicaProps != null) {
nodes = new ArrayList<Node>(replicaProps.size());
for (ZkCoreNodeProps props : replicaProps) {
nodes.add(new StdNode(props));
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
}
return nodes;
}
public void processGetVersions(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
int nVersions = params.getInt("getVersions", -1);
if (nVersions == -1) return;
String sync = params.get("sync");
if (sync != null) {
processSync(rb, nVersions, sync);
return;
}
UpdateLog ulog = req.getCore().getUpdateHandler().getUpdateLog();
if (ulog == null) return;
UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates();
try {
rb.rsp.add("versions", recentUpdates.getVersions(nVersions));
} finally {
recentUpdates.close(); // cache this somehow?
}
}
private void handleCommands(List<CommandOperation> ops, ConfigOverlay overlay)
throws IOException {
for (CommandOperation op : ops) {
switch (op.name) {
case SET_PROPERTY:
overlay = applySetProp(op, overlay);
break;
case UNSET_PROPERTY:
overlay = applyUnset(op, overlay);
break;
case SET_USER_PROPERTY:
overlay = applySetUserProp(op, overlay);
break;
case UNSET_USER_PROPERTY:
overlay = applyUnsetUserProp(op, overlay);
break;
case UPDATE_REQHANDLER:
case CREATE_REQHANDLER:
overlay = applyRequestHandler(op, overlay);
break;
case DELETE_REQHANDLER:
overlay = applyDeleteHandler(op, overlay);
break;
}
}
List errs = CommandOperation.captureErrors(ops);
if (!errs.isEmpty()) {
resp.add(CommandOperation.ERR_MSGS, errs);
return;
}
SolrResourceLoader loader = req.getCore().getResourceLoader();
if (loader instanceof ZkSolrResourceLoader) {
ZkController.persistConfigResourceToZooKeeper(
loader,
overlay.getZnodeVersion(),
ConfigOverlay.RESOURCE_NAME,
overlay.toByteArray(),
true);
} else {
SolrResourceLoader.persistConfLocally(
loader, ConfigOverlay.RESOURCE_NAME, overlay.toByteArray());
req.getCore().getCoreDescriptor().getCoreContainer().reload(req.getCore().getName());
}
}
@Override
public void processDelete(DeleteUpdateCommand cmd) throws IOException {
if (!cmd.isDeleteById()) {
doDeleteByQuery(cmd);
return;
}
int hash = 0;
if (zkEnabled) {
zkCheck();
hash = hash(cmd);
nodes = setupRequest(hash);
} else {
isLeader = getNonZkLeaderAssumption(req);
}
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionDelete(cmd);
}
if (dropCmd) {
// TODO: do we need to add anything to the response?
return;
}
ModifiableSolrParams params = null;
if (nodes != null) {
params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(
DISTRIB_UPDATE_PARAM,
(isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
if (isLeader) {
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
}
cmdDistrib.distribDelete(cmd, nodes, params);
}
// cmd.getIndexId == null when delete by query
// TODO: what to do when no idField?
if (returnVersions && rsp != null && cmd.getIndexedId() != null && idField != null) {
if (deleteResponse == null) {
deleteResponse = new NamedList<String>();
rsp.add("deletes", deleteResponse);
}
if (scratch == null) scratch = new CharsRef();
idField.getType().indexedToReadable(cmd.getIndexedId(), scratch);
deleteResponse.add(
scratch.toString(),
cmd
.getVersion()); // we're returning the version of the delete.. not the version of the
// doc we deleted.
}
}
// make sure that log isn't needlessly replayed after a clean close
@Test
public void testCleanShutdown() throws Exception {
DirectUpdateHandler2.commitOnClose = true;
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
UpdateLog.testing_logReplayHook =
new Runnable() {
@Override
public void run() {
try {
assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
UpdateLog.testing_logReplayFinishHook =
new Runnable() {
@Override
public void run() {
logReplayFinish.release();
}
};
SolrQueryRequest req = req();
UpdateHandler uhandler = req.getCore().getUpdateHandler();
UpdateLog ulog = uhandler.getUpdateLog();
try {
clearIndex();
assertU(commit());
assertU(adoc("id", "E1", "val_i", "1"));
assertU(adoc("id", "E2", "val_i", "1"));
// set to a high enough number so this test won't hang on a bug
logReplay.release(10);
h.close();
createCore();
// make sure the docs got committed
assertJQ(req("q", "*:*"), "/response/numFound==2");
// make sure no replay happened
assertEquals(10, logReplay.availablePermits());
} finally {
DirectUpdateHandler2.commitOnClose = true;
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
req().close();
}
}
@Override
public QParser createParser(
String qStr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
Log.info("createParser");
ModifiableSolrParams modparams = new ModifiableSolrParams(params);
String modQ = filter(qStr);
modparams.set("q", modQ);
return req.getCore().getQueryPlugin(parserImpl).createParser(modQ, localParams, modparams, req);
}
// Skip encoding for updating the index
void createIndex2(int nDocs, String... fields) throws IOException {
Set<String> fieldSet = new HashSet<String>(Arrays.asList(fields));
SolrQueryRequest req = lrf.makeRequest();
SolrQueryResponse rsp = new SolrQueryResponse();
UpdateRequestProcessorChain processorChain = req.getCore().getUpdateProcessingChain(null);
UpdateRequestProcessor processor = processorChain.createProcessor(req, rsp);
boolean foomany_s = fieldSet.contains("foomany_s");
boolean foo1_s = fieldSet.contains("foo1_s");
boolean foo2_s = fieldSet.contains("foo2_s");
boolean foo4_s = fieldSet.contains("foo4_s");
boolean foo8_s = fieldSet.contains("foo8_s");
boolean t10_100_ws = fieldSet.contains("t10_100_ws");
for (int i = 0; i < nDocs; i++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", Float.toString(i));
if (foomany_s) {
doc.addField("foomany_s", t(r.nextInt(nDocs * 10)));
}
if (foo1_s) {
doc.addField("foo1_s", t(0));
}
if (foo2_s) {
doc.addField("foo2_s", r.nextInt(2));
}
if (foo4_s) {
doc.addField("foo4_s", r.nextInt(4));
}
if (foo8_s) {
doc.addField("foo8_s", r.nextInt(8));
}
if (t10_100_ws) {
StringBuilder sb = new StringBuilder(9 * 100);
for (int j = 0; j < 100; j++) {
sb.append(' ');
sb.append(t(r.nextInt(10)));
}
doc.addField("t10_100_ws", sb.toString());
}
AddUpdateCommand cmd = new AddUpdateCommand();
cmd.solrDoc = doc;
processor.processAdd(cmd);
}
processor.finish();
req.close();
assertU(commit());
req = lrf.makeRequest();
assertEquals(nDocs, req.getSearcher().maxDoc());
req.close();
}
private void handlePOST() throws IOException {
Iterable<ContentStream> streams = req.getContentStreams();
if (streams == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing content stream");
}
ArrayList<CommandOperation> ops = new ArrayList<>();
for (ContentStream stream : streams) ops.addAll(CommandOperation.parse(stream.getReader()));
List<Map> errList = CommandOperation.captureErrors(ops);
if (!errList.isEmpty()) {
resp.add(CommandOperation.ERR_MSGS, errList);
return;
}
try {
for (; ; ) {
ArrayList<CommandOperation> opsCopy = new ArrayList<>(ops.size());
for (CommandOperation op : ops) opsCopy.add(op.getCopy());
try {
if (parts.size() > 1 && RequestParams.NAME.equals(parts.get(1))) {
RequestParams params =
RequestParams.getFreshRequestParams(
req.getCore().getResourceLoader(),
req.getCore().getSolrConfig().getRequestParams());
handleParams(opsCopy, params);
} else {
ConfigOverlay overlay =
SolrConfig.getConfigOverlay(req.getCore().getResourceLoader());
handleCommands(opsCopy, overlay);
}
break; // succeeded . so no need to go over the loop again
} catch (ZkController.ResourceModifiedInZkException e) {
// retry
log.info(
"Race condition, the node is modified in ZK by someone else " + e.getMessage());
}
}
} catch (Exception e) {
resp.setException(e);
resp.add(CommandOperation.ERR_MSGS, singletonList(SchemaManager.getErrorStr(e)));
}
}
public void processGetUpdates(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
String versionsStr = params.get("getUpdates");
if (versionsStr == null) return;
UpdateLog ulog = req.getCore().getUpdateHandler().getUpdateLog();
if (ulog == null) return;
List<String> versions = StrUtils.splitSmart(versionsStr, ",", true);
List<Object> updates = new ArrayList<Object>(versions.size());
long minVersion = Long.MAX_VALUE;
// TODO: get this from cache instead of rebuilding?
UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates();
try {
for (String versionStr : versions) {
long version = Long.parseLong(versionStr);
try {
Object o = recentUpdates.lookup(version);
if (o == null) continue;
if (version > 0) {
minVersion = Math.min(minVersion, version);
}
// TODO: do any kind of validation here?
updates.add(o);
} catch (SolrException e) {
log.warn("Exception reading log for updates", e);
} catch (ClassCastException e) {
log.warn("Exception reading log for updates", e);
}
}
// Must return all delete-by-query commands that occur after the first add requested
// since they may apply.
updates.addAll(recentUpdates.getDeleteByQuery(minVersion));
rb.rsp.add("updates", updates);
} finally {
recentUpdates.close(); // cache this somehow?
}
}
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
SolrParams params = req.getParams();
params = adjustParams(params);
req.setParams(params);
if (params.get("action") != null) {
handleAdmin(req, rsp, params);
return;
}
TupleStream tupleStream;
try {
tupleStream = this.streamFactory.constructStream(params.get("expr"));
} catch (Exception e) {
// Catch exceptions that occur while the stream is being created. This will include streaming
// expression parse rules.
SolrException.log(logger, e);
rsp.add("result-set", new DummyErrorStream(e));
return;
}
int worker = params.getInt("workerID", 0);
int numWorkers = params.getInt("numWorkers", 1);
StreamContext context = new StreamContext();
context.workerID = worker;
context.numWorkers = numWorkers;
context.setSolrClientCache(clientCache);
context.setModelCache(modelCache);
context.put("core", this.coreName);
context.put("solr-core", req.getCore());
tupleStream.setStreamContext(context);
// if asking for explanation then go get it
if (params.getBool("explain", false)) {
rsp.add("explanation", tupleStream.toExplanation(this.streamFactory));
}
if (tupleStream instanceof DaemonStream) {
DaemonStream daemonStream = (DaemonStream) tupleStream;
if (daemons.containsKey(daemonStream.getId())) {
daemons.remove(daemonStream.getId()).close();
}
daemonStream.setDaemons(daemons);
daemonStream.open(); // This will start the deamonStream
daemons.put(daemonStream.getId(), daemonStream);
rsp.add(
"result-set",
new DaemonResponseStream("Deamon:" + daemonStream.getId() + " started on " + coreName));
} else {
rsp.add("result-set", new TimerStream(new ExceptionStream(tupleStream)));
}
}
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
FileFloatSource.resetCache();
log.debug("readerCache has been reset.");
UpdateRequestProcessor processor =
req.getCore().getUpdateProcessingChain(null).createProcessor(req, rsp);
try {
RequestHandlerUtils.handleCommit(processor, req.getParams(), true);
} finally {
processor.finish();
}
}
public DistributedUpdateProcessor(
SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
super(next);
this.rsp = rsp;
this.next = next;
this.idField = req.getSchema().getUniqueKeyField();
// version init
this.updateHandler = req.getCore().getUpdateHandler();
this.ulog = updateHandler.getUpdateLog();
this.vinfo = ulog == null ? null : ulog.getVersionInfo();
versionsStored = this.vinfo != null && this.vinfo.getVersionField() != null;
returnVersions = req.getParams().getBool(UpdateParams.VERSIONS, false);
// TODO: better way to get the response, or pass back info to it?
SolrRequestInfo reqInfo = returnVersions ? SolrRequestInfo.getRequestInfo() : null;
this.req = req;
CoreDescriptor coreDesc = req.getCore().getCoreDescriptor();
this.zkEnabled = coreDesc.getCoreContainer().isZooKeeperAware();
zkController = req.getCore().getCoreDescriptor().getCoreContainer().getZkController();
if (zkEnabled) {
numNodes = zkController.getZkStateReader().getClusterState().getLiveNodes().size();
cmdDistrib =
new SolrCmdDistributor(
numNodes, coreDesc.getCoreContainer().getZkController().getCmdDistribExecutor());
}
// this.rsp = reqInfo != null ? reqInfo.getRsp() : null;
cloudDesc = coreDesc.getCloudDescriptor();
if (cloudDesc != null) {
collection = cloudDesc.getCollectionName();
}
}
/**
* Get Transformer from request context, or from TransformerProvider. This allows either
* getContentType(...) or write(...) to instantiate the Transformer, depending on which one is
* called first, then the other one reuses the same Transformer
*/
Transformer getTransformer(String xslt, SolrQueryRequest request) throws IOException {
// not the cleanest way to achieve this
// no need to synchronize access to context, right?
// Nothing else happens with it at the same time
final Map<Object, Object> ctx = request.getContext();
Transformer result = (Transformer) ctx.get(CONTEXT_TRANSFORMER_KEY);
if (result == null) {
SolrConfig solrConfig = request.getCore().getSolrConfig();
result =
TransformerProvider.instance.getTransformer(solrConfig, xslt, xsltCacheLifetimeSeconds);
result.setErrorListener(xmllog);
ctx.put(CONTEXT_TRANSFORMER_KEY, result);
}
return result;
}
RegexpBoostProcessor(
SolrParams parameters,
SolrQueryRequest request,
SolrQueryResponse response,
UpdateRequestProcessor nextProcessor,
final Map<Object, Object> sharedObjectCache) {
super(nextProcessor);
this.initParameters(parameters);
if (this.boostFilename == null) {
log.warn("Null boost filename. Disabling processor.");
setEnabled(false);
}
if (!isEnabled()) {
return;
}
try {
synchronized (sharedObjectCache) {
List<BoostEntry> cachedBoostEntries =
(List<BoostEntry>) sharedObjectCache.get(BOOST_ENTRIES_CACHE_KEY);
if (cachedBoostEntries == null) {
log.debug("No pre-cached boost entry list found, initializing new");
InputStream is = request.getCore().getResourceLoader().openResource(boostFilename);
cachedBoostEntries = initBoostEntries(is);
sharedObjectCache.put(BOOST_ENTRIES_CACHE_KEY, cachedBoostEntries);
} else {
if (log.isDebugEnabled()) {
log.debug(
"Using cached boost entry list with " + cachedBoostEntries.size() + " elements.");
}
}
this.boostEntries = cachedBoostEntries;
}
} catch (IOException ioe) {
log.warn("IOException while initializing boost entries from file " + this.boostFilename, ioe);
}
}
public ExtractingDocumentLoader(
SolrQueryRequest req,
UpdateRequestProcessor processor,
TikaConfig config,
ParseContextConfig parseContextConfig,
SolrContentHandlerFactory factory) {
this.params = req.getParams();
this.core = req.getCore();
this.config = config;
this.parseContextConfig = parseContextConfig;
this.processor = processor;
templateAdd = new AddUpdateCommand(req);
templateAdd.overwrite = params.getBool(UpdateParams.OVERWRITE, true);
templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
// this is lightweight
autoDetectParser = new AutoDetectParser(config);
this.factory = factory;
ignoreTikaException = params.getBool(ExtractingParams.IGNORE_TIKA_EXCEPTION, false);
}
private void doDefensiveChecks(String shardId, DistribPhase phase) {
String from = req.getParams().get("distrib.from");
boolean logReplay = req.getParams().getBool(LOG_REPLAY, false);
boolean localIsLeader = req.getCore().getCoreDescriptor().getCloudDescriptor().isLeader();
if (!logReplay
&& DistribPhase.FROMLEADER == phase
&& localIsLeader
&& from != null) { // from will be null on log replay
log.error(
"Request says it is coming from leader, but we are the leader: " + req.getParamString());
throw new SolrException(
ErrorCode.SERVICE_UNAVAILABLE,
"Request says it is coming from leader, but we are the leader");
}
if (isLeader && !localIsLeader) {
log.error("ClusterState says we are the leader, but locally we don't think so");
throw new SolrException(
ErrorCode.SERVICE_UNAVAILABLE,
"ClusterState says we are the leader, but locally we don't think so");
}
}
@Override
public void load(
SolrQueryRequest req,
SolrQueryResponse rsp,
ContentStream stream,
UpdateRequestProcessor processor)
throws Exception {
Parser parser = null;
String streamType = req.getParams().get(ExtractingParams.STREAM_TYPE, null);
if (streamType != null) {
// Cache? Parsers are lightweight to construct and thread-safe, so I'm told
MediaType mt = MediaType.parse(streamType.trim().toLowerCase(Locale.ROOT));
parser = new DefaultParser(config.getMediaTypeRegistry()).getParsers().get(mt);
} else {
parser = autoDetectParser;
}
if (parser != null) {
Metadata metadata = new Metadata();
// If you specify the resource name (the filename, roughly) with this parameter,
// then Tika can make use of it in guessing the appropriate MIME type:
String resourceName = req.getParams().get(ExtractingParams.RESOURCE_NAME, null);
if (resourceName != null) {
metadata.add(TikaMetadataKeys.RESOURCE_NAME_KEY, resourceName);
}
// Provide stream's content type as hint for auto detection
if (stream.getContentType() != null) {
metadata.add(HttpHeaders.CONTENT_TYPE, stream.getContentType());
}
InputStream inputStream = null;
try {
inputStream = stream.getStream();
metadata.add(ExtractingMetadataConstants.STREAM_NAME, stream.getName());
metadata.add(ExtractingMetadataConstants.STREAM_SOURCE_INFO, stream.getSourceInfo());
metadata.add(ExtractingMetadataConstants.STREAM_SIZE, String.valueOf(stream.getSize()));
metadata.add(ExtractingMetadataConstants.STREAM_CONTENT_TYPE, stream.getContentType());
// HtmlParser and TXTParser regard Metadata.CONTENT_ENCODING in metadata
String charset = ContentStreamBase.getCharsetFromContentType(stream.getContentType());
if (charset != null) {
metadata.add(HttpHeaders.CONTENT_ENCODING, charset);
}
String xpathExpr = params.get(ExtractingParams.XPATH_EXPRESSION);
boolean extractOnly = params.getBool(ExtractingParams.EXTRACT_ONLY, false);
SolrContentHandler handler =
factory.createSolrContentHandler(metadata, params, req.getSchema());
ContentHandler parsingHandler = handler;
StringWriter writer = null;
BaseMarkupSerializer serializer = null;
if (extractOnly == true) {
String extractFormat = params.get(ExtractingParams.EXTRACT_FORMAT, "xml");
writer = new StringWriter();
if (extractFormat.equals(TEXT_FORMAT)) {
serializer = new TextSerializer();
serializer.setOutputCharStream(writer);
serializer.setOutputFormat(new OutputFormat("Text", "UTF-8", true));
} else {
serializer = new XMLSerializer(writer, new OutputFormat("XML", "UTF-8", true));
}
if (xpathExpr != null) {
Matcher matcher = PARSER.parse(xpathExpr);
serializer
.startDocument(); // The MatchingContentHandler does not invoke startDocument. See
// http://tika.markmail.org/message/kknu3hw7argwiqin
parsingHandler = new MatchingContentHandler(serializer, matcher);
} else {
parsingHandler = serializer;
}
} else if (xpathExpr != null) {
Matcher matcher = PARSER.parse(xpathExpr);
parsingHandler = new MatchingContentHandler(handler, matcher);
} // else leave it as is
try {
// potentially use a wrapper handler for parsing, but we still need the SolrContentHandler
// for getting the document.
ParseContext context = parseContextConfig.create();
context.set(Parser.class, parser);
context.set(HtmlMapper.class, MostlyPassthroughHtmlMapper.INSTANCE);
// Password handling
RegexRulesPasswordProvider epp = new RegexRulesPasswordProvider();
String pwMapFile = params.get(ExtractingParams.PASSWORD_MAP_FILE);
if (pwMapFile != null && pwMapFile.length() > 0) {
InputStream is = req.getCore().getResourceLoader().openResource(pwMapFile);
if (is != null) {
log.debug("Password file supplied: " + pwMapFile);
epp.parse(is);
}
}
context.set(PasswordProvider.class, epp);
String resourcePassword = params.get(ExtractingParams.RESOURCE_PASSWORD);
if (resourcePassword != null) {
epp.setExplicitPassword(resourcePassword);
log.debug("Literal password supplied for file " + resourceName);
}
parser.parse(inputStream, parsingHandler, metadata, context);
} catch (TikaException e) {
if (ignoreTikaException)
log.warn(
new StringBuilder("skip extracting text due to ")
.append(e.getLocalizedMessage())
.append(". metadata=")
.append(metadata.toString())
.toString());
else throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
if (extractOnly == false) {
addDoc(handler);
} else {
// serializer is not null, so we need to call endDoc on it if using xpath
if (xpathExpr != null) {
serializer.endDocument();
}
rsp.add(stream.getName(), writer.toString());
writer.close();
String[] names = metadata.names();
NamedList metadataNL = new NamedList();
for (int i = 0; i < names.length; i++) {
String[] vals = metadata.getValues(names[i]);
metadataNL.add(names[i], vals);
}
rsp.add(stream.getName() + "_metadata", metadataNL);
}
} catch (SAXException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} finally {
IOUtils.closeQuietly(inputStream);
}
} else {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Stream type of "
+ streamType
+ " didn't match any known parsers. Please supply the "
+ ExtractingParams.STREAM_TYPE
+ " parameter.");
}
}
@Override
public void prepare(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrParams params = req.getParams();
// A runtime param can skip
if (!params.getBool(QueryElevationParams.ENABLE, true)) {
return;
}
boolean exclusive = params.getBool(QueryElevationParams.EXCLUSIVE, false);
// A runtime parameter can alter the config value for forceElevation
boolean force = params.getBool(QueryElevationParams.FORCE_ELEVATION, forceElevation);
boolean markExcludes = params.getBool(QueryElevationParams.MARK_EXCLUDES, false);
String boostStr = params.get(QueryElevationParams.IDS);
String exStr = params.get(QueryElevationParams.EXCLUDE);
Query query = rb.getQuery();
String qstr = rb.getQueryString();
if (query == null || qstr == null) {
return;
}
ElevationObj booster = null;
try {
if (boostStr != null || exStr != null) {
List<String> boosts =
(boostStr != null)
? StrUtils.splitSmart(boostStr, ",", true)
: new ArrayList<String>(0);
List<String> excludes =
(exStr != null) ? StrUtils.splitSmart(exStr, ",", true) : new ArrayList<String>(0);
booster = new ElevationObj(qstr, boosts, excludes);
} else {
IndexReader reader = req.getSearcher().getIndexReader();
qstr = getAnalyzedQuery(qstr);
booster = getElevationMap(reader, req.getCore()).get(qstr);
}
} catch (Exception ex) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error loading elevation", ex);
}
if (booster != null) {
rb.req.getContext().put(BOOSTED, booster.ids);
// Change the query to insert forced documents
if (exclusive == true) {
// we only want these results
rb.setQuery(booster.include);
} else {
BooleanQuery newq = new BooleanQuery(true);
newq.add(query, BooleanClause.Occur.SHOULD);
newq.add(booster.include, BooleanClause.Occur.SHOULD);
if (booster.exclude != null) {
if (markExcludes == false) {
for (TermQuery tq : booster.exclude) {
newq.add(new BooleanClause(tq, BooleanClause.Occur.MUST_NOT));
}
} else {
// we are only going to mark items as excluded, not actually exclude them. This works
// with the EditorialMarkerFactory
rb.req.getContext().put(EXCLUDED, booster.excludeIds);
}
}
rb.setQuery(newq);
}
ElevationComparatorSource comparator = new ElevationComparatorSource(booster);
// if the sort is 'score desc' use a custom sorting method to
// insert documents in their proper place
SortSpec sortSpec = rb.getSortSpec();
if (sortSpec.getSort() == null) {
sortSpec.setSortAndFields(
new Sort(
new SortField[] {
new SortField("_elevate_", comparator, true),
new SortField(null, SortField.Type.SCORE, false)
}),
Arrays.asList(new SchemaField[2]));
} else {
// Check if the sort is based on score
SortSpec modSortSpec = this.modifySortSpec(sortSpec, force, comparator);
if (null != modSortSpec) {
rb.setSortSpec(modSortSpec);
}
}
// alter the sorting in the grouping specification if there is one
GroupingSpecification groupingSpec = rb.getGroupingSpec();
if (groupingSpec != null) {
SortField[] groupSort = groupingSpec.getGroupSort().getSort();
Sort modGroupSort = this.modifySort(groupSort, force, comparator);
if (modGroupSort != null) {
groupingSpec.setGroupSort(modGroupSort);
}
SortField[] withinGroupSort = groupingSpec.getSortWithinGroup().getSort();
Sort modWithinGroupSort = this.modifySort(withinGroupSort, force, comparator);
if (modWithinGroupSort != null) {
groupingSpec.setSortWithinGroup(modWithinGroupSort);
}
}
}
// Add debugging information
if (rb.isDebug()) {
List<String> match = null;
if (booster != null) {
// Extract the elevated terms into a list
match = new ArrayList<String>(booster.priority.size());
for (Object o : booster.include.clauses()) {
TermQuery tq = (TermQuery) ((BooleanClause) o).getQuery();
match.add(tq.getTerm().text());
}
}
SimpleOrderedMap<Object> dbg = new SimpleOrderedMap<Object>();
dbg.add("q", qstr);
dbg.add("match", match);
if (rb.isDebugQuery()) {
rb.addDebugInfo("queryBoosting", dbg);
}
}
}
@Test
public void testBufferingFlags() throws Exception {
DirectUpdateHandler2.commitOnClose = false;
final Semaphore logReplayFinish = new Semaphore(0);
UpdateLog.testing_logReplayFinishHook =
new Runnable() {
@Override
public void run() {
logReplayFinish.release();
}
};
SolrQueryRequest req = req();
UpdateHandler uhandler = req.getCore().getUpdateHandler();
UpdateLog ulog = uhandler.getUpdateLog();
try {
clearIndex();
assertU(commit());
assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
ulog.bufferUpdates();
// simulate updates from a leader
updateJ(
jsonAdd(sdoc("id", "Q1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "Q2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "Q3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
req.close();
h.close();
createCore();
req = req();
uhandler = req.getCore().getUpdateHandler();
ulog = uhandler.getUpdateLog();
logReplayFinish.acquire(); // wait for replay to finish
assertTrue(
(ulog.getStartingOperation() & UpdateLog.FLAG_GAP)
!= 0); // since we died while buffering, we should see this last
//
// Try again to ensure that the previous log replay didn't wipe out our flags
//
req.close();
h.close();
createCore();
req = req();
uhandler = req.getCore().getUpdateHandler();
ulog = uhandler.getUpdateLog();
assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) != 0);
// now do some normal non-buffered adds
updateJ(
jsonAdd(sdoc("id", "Q4", "_version_", "114")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "Q5", "_version_", "115")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "Q6", "_version_", "116")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertU(commit());
req.close();
h.close();
createCore();
req = req();
uhandler = req.getCore().getUpdateHandler();
ulog = uhandler.getUpdateLog();
assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0);
ulog.bufferUpdates();
// simulate receiving no updates
ulog.applyBufferedUpdates();
updateJ(
jsonAdd(sdoc("id", "Q7", "_version_", "117")),
params(
DISTRIB_UPDATE_PARAM,
FROM_LEADER)); // do another add to make sure flags are back to normal
req.close();
h.close();
createCore();
req = req();
uhandler = req.getCore().getUpdateHandler();
ulog = uhandler.getUpdateLog();
assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0); // check flags on Q7
logReplayFinish.acquire();
assertEquals(
UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
} finally {
DirectUpdateHandler2.commitOnClose = true;
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
req().close();
}
}
@Test
@Ignore("HDFS-3107: no truncate support yet")
public void testDropBuffered() throws Exception {
DirectUpdateHandler2.commitOnClose = false;
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
UpdateLog.testing_logReplayHook =
new Runnable() {
@Override
public void run() {
try {
assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
UpdateLog.testing_logReplayFinishHook =
new Runnable() {
@Override
public void run() {
logReplayFinish.release();
}
};
SolrQueryRequest req = req();
UpdateHandler uhandler = req.getCore().getUpdateHandler();
UpdateLog ulog = uhandler.getUpdateLog();
try {
clearIndex();
assertU(commit());
assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
ulog.bufferUpdates();
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
assertTrue(rinfoFuture == null);
assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
ulog.bufferUpdates();
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
// simulate updates from a leader
updateJ(
jsonAdd(sdoc("id", "C1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "C2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "C3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertTrue(ulog.dropBufferedUpdates());
ulog.bufferUpdates();
updateJ(
jsonAdd(sdoc("id", "C4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "C5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
logReplay.release(1000);
rinfoFuture = ulog.applyBufferedUpdates();
UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
assertEquals(2, rinfo.adds);
assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[105,104]}");
// this time add some docs first before buffering starts (so tlog won't be at pos 0)
updateJ(
jsonAdd(sdoc("id", "C100", "_version_", "200")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "C101", "_version_", "201")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
ulog.bufferUpdates();
updateJ(
jsonAdd(sdoc("id", "C103", "_version_", "203")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "C104", "_version_", "204")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertTrue(ulog.dropBufferedUpdates());
ulog.bufferUpdates();
updateJ(
jsonAdd(sdoc("id", "C105", "_version_", "205")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "C106", "_version_", "206")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
rinfoFuture = ulog.applyBufferedUpdates();
rinfo = rinfoFuture.get();
assertEquals(2, rinfo.adds);
assertJQ(
req("q", "*:*", "sort", "_version_ asc", "fl", "id,_version_"),
"/response/docs==["
+ "{'id':'C4','_version_':104}"
+ ",{'id':'C5','_version_':105}"
+ ",{'id':'C100','_version_':200}"
+ ",{'id':'C101','_version_':201}"
+ ",{'id':'C105','_version_':205}"
+ ",{'id':'C106','_version_':206}"
+ "]");
assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[206,205,201,200,105,104]}");
ulog.bufferUpdates();
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
updateJ(
jsonAdd(sdoc("id", "C301", "_version_", "998")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "C302", "_version_", "999")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertTrue(ulog.dropBufferedUpdates());
// make sure we can overwrite with a lower version
// TODO: is this functionality needed?
updateJ(
jsonAdd(sdoc("id", "C301", "_version_", "301")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "C302", "_version_", "302")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertU(commit());
assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[302,301]}");
assertJQ(
req("q", "*:*", "sort", "_version_ desc", "fl", "id,_version_", "rows", "2"),
"/response/docs==["
+ "{'id':'C302','_version_':302}"
+ ",{'id':'C301','_version_':301}"
+ "]");
updateJ(
jsonAdd(sdoc("id", "C2", "_version_", "302")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
assertEquals(
UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
} finally {
DirectUpdateHandler2.commitOnClose = true;
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
req().close();
}
}
@Test
public void testBuffering() throws Exception {
DirectUpdateHandler2.commitOnClose = false;
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
UpdateLog.testing_logReplayHook =
new Runnable() {
@Override
public void run() {
try {
assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
UpdateLog.testing_logReplayFinishHook =
new Runnable() {
@Override
public void run() {
logReplayFinish.release();
}
};
SolrQueryRequest req = req();
UpdateHandler uhandler = req.getCore().getUpdateHandler();
UpdateLog ulog = uhandler.getUpdateLog();
try {
clearIndex();
assertU(commit());
assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
ulog.bufferUpdates();
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
assertTrue(rinfoFuture == null);
assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
ulog.bufferUpdates();
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
// simulate updates from a leader
updateJ(
jsonAdd(sdoc("id", "B1", "_version_", "1010")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "B11", "_version_", "1015")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonDelQ("id:B1 id:B11 id:B2 id:B3"),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-1017"));
updateJ(
jsonAdd(sdoc("id", "B2", "_version_", "1020")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "B3", "_version_", "1030")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
deleteAndGetVersion("B1", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2010"));
assertJQ(
req("qt", "/get", "getVersions", "6"),
"=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
assertU(commit());
assertJQ(
req("qt", "/get", "getVersions", "6"),
"=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
// updates should be buffered, so we should not see any results yet.
assertJQ(req("q", "*:*"), "/response/numFound==0");
// real-time get should also not show anything (this could change in the future,
// but it's currently used for validating version numbers too, so it would
// be bad for updates to be visible if we're just buffering.
assertJQ(req("qt", "/get", "id", "B3"), "=={'doc':null}");
rinfoFuture = ulog.applyBufferedUpdates();
assertTrue(rinfoFuture != null);
assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
logReplay.release(1000);
UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
assertJQ(
req("qt", "/get", "getVersions", "6"),
"=={'versions':[-2010,1030,1020,-1017,1015,1010]}");
assertJQ(req("q", "*:*"), "/response/numFound==2");
// move back to recovering
ulog.bufferUpdates();
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
Long ver = getVer(req("qt", "/get", "id", "B3"));
assertEquals(1030L, ver.longValue());
// add a reordered doc that shouldn't overwrite one in the index
updateJ(
jsonAdd(sdoc("id", "B3", "_version_", "3")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
// reorder two buffered updates
updateJ(
jsonAdd(sdoc("id", "B4", "_version_", "1040")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
deleteAndGetVersion(
"B4",
params(
DISTRIB_UPDATE_PARAM,
FROM_LEADER,
"_version_",
"-940")); // this update should not take affect
updateJ(
jsonAdd(sdoc("id", "B6", "_version_", "1060")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "B5", "_version_", "1050")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
updateJ(
jsonAdd(sdoc("id", "B8", "_version_", "1080")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
// test that delete by query is at least buffered along with everything else so it will delete
// the
// currently buffered id:8 (even if it doesn't currently support versioning)
updateJ(
"{\"delete\": { \"query\":\"id:B2 OR id:B8\" }}",
params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-3000"));
assertJQ(
req("qt", "/get", "getVersions", "13"),
"=={'versions':[-3000,1080,1050,1060,-940,1040,3,-2010,1030,1020,-1017,1015,1010]}" // the
// "3"
// appears because versions aren't checked while buffering
);
logReplay.drainPermits();
rinfoFuture = ulog.applyBufferedUpdates();
assertTrue(rinfoFuture != null);
assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
// apply a single update
logReplay.release(1);
// now add another update
updateJ(
jsonAdd(sdoc("id", "B7", "_version_", "1070")),
params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
// a reordered update that should be dropped
deleteAndGetVersion("B5", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-950"));
deleteAndGetVersion("B6", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2060"));
logReplay.release(1000);
UpdateLog.RecoveryInfo recInfo = rinfoFuture.get();
assertJQ(
req("q", "*:*", "sort", "id asc", "fl", "id,_version_"),
"/response/docs==["
+ "{'id':'B3','_version_':1030}"
+ ",{'id':'B4','_version_':1040}"
+ ",{'id':'B5','_version_':1050}"
+ ",{'id':'B7','_version_':1070}"
+ "]");
assertEquals(1, recInfo.deleteByQuery);
assertEquals(
UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
} finally {
DirectUpdateHandler2.commitOnClose = true;
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
req().close();
}
}
public void doDeleteByQuery(DeleteUpdateCommand cmd) throws IOException {
// even in non zk mode, tests simulate updates from a leader
if (!zkEnabled) {
isLeader = getNonZkLeaderAssumption(req);
} else {
zkCheck();
}
// NONE: we are the first to receive this deleteByQuery
// - it must be forwarded to the leader of every shard
// TO: we are a leader receiving a forwarded deleteByQuery... we must:
// - block all updates (use VersionInfo)
// - flush *all* updates going to our replicas
// - forward the DBQ to our replicas and wait for the response
// - log + execute the local DBQ
// FROM: we are a replica receiving a DBQ from our leader
// - log + execute the local DBQ
DistribPhase phase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
if (zkEnabled && DistribPhase.NONE == phase) {
boolean leaderForAnyShard = false; // start off by assuming we are not a leader for any shard
Map<String, Slice> slices = zkController.getClusterState().getSlices(collection);
if (slices == null) {
throw new SolrException(
ErrorCode.BAD_REQUEST,
"Cannot find collection:"
+ collection
+ " in "
+ zkController.getClusterState().getCollections());
}
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString());
List<Node> leaders = new ArrayList<Node>(slices.size());
for (Map.Entry<String, Slice> sliceEntry : slices.entrySet()) {
String sliceName = sliceEntry.getKey();
ZkNodeProps leaderProps;
try {
leaderProps = zkController.getZkStateReader().getLeaderProps(collection, sliceName);
} catch (InterruptedException e) {
throw new SolrException(
ErrorCode.SERVICE_UNAVAILABLE, "Exception finding leader for shard " + sliceName, e);
}
// TODO: What if leaders changed in the meantime?
// should we send out slice-at-a-time and if a node returns "hey, I'm not a leader" (or we
// get an error because it went down) then look up the new leader?
// Am I the leader for this slice?
ZkCoreNodeProps coreLeaderProps = new ZkCoreNodeProps(leaderProps);
String leaderNodeName = coreLeaderProps.getCoreNodeName();
String coreName = req.getCore().getName();
String coreNodeName = zkController.getNodeName() + "_" + coreName;
isLeader = coreNodeName.equals(leaderNodeName);
if (isLeader) {
// don't forward to ourself
leaderForAnyShard = true;
} else {
leaders.add(new StdNode(coreLeaderProps));
}
}
params.remove("commit"); // this will be distributed from the local commit
cmdDistrib.distribDelete(cmd, leaders, params);
if (!leaderForAnyShard) {
return;
}
// change the phase to TOLEADER so we look up and forward to our own replicas (if any)
phase = DistribPhase.TOLEADER;
}
List<Node> replicas = null;
if (zkEnabled && DistribPhase.TOLEADER == phase) {
// This core should be a leader
isLeader = true;
replicas = setupRequest();
} else if (DistribPhase.FROMLEADER == phase) {
isLeader = false;
}
if (vinfo == null) {
super.processDelete(cmd);
return;
}
// at this point, there is an update we need to try and apply.
// we may or may not be the leader.
// Find the version
long versionOnUpdate = cmd.getVersion();
if (versionOnUpdate == 0) {
String versionOnUpdateS = req.getParams().get(VERSION_FIELD);
versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS);
}
versionOnUpdate = Math.abs(versionOnUpdate); // normalize to positive version
boolean isReplay = (cmd.getFlags() & UpdateCommand.REPLAY) != 0;
boolean leaderLogic = isLeader && !isReplay;
if (!leaderLogic && versionOnUpdate == 0) {
throw new SolrException(ErrorCode.BAD_REQUEST, "missing _version_ on update from leader");
}
vinfo.blockUpdates();
try {
if (versionsStored) {
if (leaderLogic) {
long version = vinfo.getNewClock();
cmd.setVersion(-version);
// TODO update versions in all buckets
doLocalDelete(cmd);
} else {
cmd.setVersion(-versionOnUpdate);
if (ulog.getState() != UpdateLog.State.ACTIVE
&& (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
// we're not in an active state, and this update isn't from a replay, so buffer it.
cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
ulog.deleteByQuery(cmd);
return;
}
doLocalDelete(cmd);
}
}
// since we don't know which documents were deleted, the easiest thing to do is to invalidate
// all real-time caches (i.e. UpdateLog) which involves also getting a new version of the
// IndexReader
// (so cache misses will see up-to-date data)
} finally {
vinfo.unblockUpdates();
}
// forward to all replicas
if (leaderLogic && replicas != null) {
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(VERSION_FIELD, Long.toString(cmd.getVersion()));
params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
params.set(
"update.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
cmdDistrib.distribDelete(cmd, replicas, params);
cmdDistrib.finish();
}
if (returnVersions && rsp != null) {
if (deleteByQueryResponse == null) {
deleteByQueryResponse = new NamedList<String>();
rsp.add("deleteByQuery", deleteByQueryResponse);
}
deleteByQueryResponse.add(cmd.getQuery(), cmd.getVersion());
}
}
private void handleParams(ArrayList<CommandOperation> ops, RequestParams params) {
for (CommandOperation op : ops) {
switch (op.name) {
case CREATE:
case UPDATE:
{
Map<String, Object> map = op.getDataMap();
if (op.hasError()) break;
for (Map.Entry<String, Object> entry : map.entrySet()) {
Map val = null;
String key = entry.getKey();
if (key == null || key.trim().isEmpty()) {
op.addError("null key ");
continue;
}
key = key.trim();
if (!validName(key)) {
op.addError(
MessageFormat.format(
"''{0}'' name should only have chars [a-zA-Z_-.0-9] ", key));
continue;
}
try {
val = (Map) entry.getValue();
} catch (Exception e1) {
op.addError("invalid params for key : " + key);
continue;
}
if (val.containsKey("")) {
op.addError("Empty keys are not allowed in params");
continue;
}
MapSolrParams old = params.getParams(key);
if (op.name.equals(UPDATE)) {
LinkedHashMap m = new LinkedHashMap(old.getMap());
m.putAll(val);
val = m;
}
params = params.setParams(key, val);
}
break;
}
case "delete":
{
List<String> name = op.getStrs(CommandOperation.ROOT_OBJ);
if (op.hasError()) break;
for (String s : name) {
if (params.getParams(s) == null) {
op.addError(
MessageFormat.format("can't delete . No such params ''{0}'' exist", s));
}
params = params.setParams(s, null);
}
}
}
}
List errs = CommandOperation.captureErrors(ops);
if (!errs.isEmpty()) {
resp.add(CommandOperation.ERR_MSGS, errs);
return;
}
SolrResourceLoader loader = req.getCore().getResourceLoader();
if (loader instanceof ZkSolrResourceLoader) {
ZkController.persistConfigResourceToZooKeeper(
loader, params.getZnodeVersion(), RequestParams.RESOURCE, params.toByteArray(), true);
} else {
SolrResourceLoader.persistConfLocally(
loader, ConfigOverlay.RESOURCE_NAME, params.toByteArray());
req.getCore().getSolrConfig().refreshRequestParams();
}
}
@Test
public void testSorting() throws Exception {
try {
init("schema12.xml");
assertU(adoc("id", "a", "title", "ipod trash trash", "str_s1", "a"));
assertU(adoc("id", "b", "title", "ipod ipod trash", "str_s1", "b"));
assertU(adoc("id", "c", "title", "ipod ipod ipod ", "str_s1", "c"));
assertU(adoc("id", "x", "title", "boosted", "str_s1", "x"));
assertU(adoc("id", "y", "title", "boosted boosted", "str_s1", "y"));
assertU(adoc("id", "z", "title", "boosted boosted boosted", "str_s1", "z"));
assertU(commit());
String query = "title:ipod";
Map<String, String> args =
new HashMap<>(); // reusing args & requests this way is a solr-test-antipattern. PLEASE
// DO NOT COPY THIS CODE
args.put(CommonParams.Q, query);
args.put(CommonParams.QT, "/elevate");
args.put(CommonParams.FL, "id,score");
args.put("indent", "true");
// args.put( CommonParams.FL, "id,title,score" );
SolrQueryRequest req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
IndexReader reader = req.getSearcher().getIndexReader();
QueryElevationComponent booster =
(QueryElevationComponent) req.getCore().getSearchComponent("elevate");
assertQ(
"Make sure standard sort works as expected",
req,
"//*[@numFound='3']",
"//result/doc[1]/str[@name='id'][.='c']",
"//result/doc[2]/str[@name='id'][.='b']",
"//result/doc[3]/str[@name='id'][.='a']");
// Explicitly set what gets boosted
booster.elevationCache.clear();
booster.setTopQueryResults(reader, query, new String[] {"x", "y", "z"}, null);
req.close();
req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
assertQ(
"All six should make it",
req,
"//*[@numFound='6']",
"//result/doc[1]/str[@name='id'][.='x']",
"//result/doc[2]/str[@name='id'][.='y']",
"//result/doc[3]/str[@name='id'][.='z']",
"//result/doc[4]/str[@name='id'][.='c']",
"//result/doc[5]/str[@name='id'][.='b']",
"//result/doc[6]/str[@name='id'][.='a']");
booster.elevationCache.clear();
// now switch the order:
req.close();
req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
booster.setTopQueryResults(reader, query, new String[] {"a", "x"}, null);
assertQ(
"All four should make it",
req,
"//*[@numFound='4']",
"//result/doc[1]/str[@name='id'][.='a']",
"//result/doc[2]/str[@name='id'][.='x']",
"//result/doc[3]/str[@name='id'][.='c']",
"//result/doc[4]/str[@name='id'][.='b']");
// Test reverse sort
args.put(CommonParams.SORT, "score asc");
req.close();
req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
assertQ(
"All four should make it",
req,
"//*[@numFound='4']"
// NOTE REVERSED doc[X] indices
,
"//result/doc[4]/str[@name='id'][.='a']",
"//result/doc[3]/str[@name='id'][.='x']",
"//result/doc[2]/str[@name='id'][.='c']",
"//result/doc[1]/str[@name='id'][.='b']");
// Try normal sort by 'id'
// default 'forceBoost' should be false
assertEquals(false, booster.forceElevation);
args.put(CommonParams.SORT, "str_s1 asc");
req.close();
req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
assertQ(
null,
req,
"//*[@numFound='4']",
"//result/doc[1]/str[@name='id'][.='a']",
"//result/doc[2]/str[@name='id'][.='b']",
"//result/doc[3]/str[@name='id'][.='c']",
"//result/doc[4]/str[@name='id'][.='x']");
args.put(CommonParams.SORT, "id asc");
req.close();
req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
assertQ(
null,
req,
"//*[@numFound='4']",
"//result/doc[1]/str[@name='id'][.='a']",
"//result/doc[2]/str[@name='id'][.='b']",
"//result/doc[3]/str[@name='id'][.='c']",
"//result/doc[4]/str[@name='id'][.='x']");
booster.forceElevation = true;
args.put(CommonParams.SORT, "id asc");
req.close();
req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
assertQ(
null,
req,
"//*[@numFound='4']",
"//result/doc[1]/str[@name='id'][.='a']",
"//result/doc[2]/str[@name='id'][.='x']",
"//result/doc[3]/str[@name='id'][.='b']",
"//result/doc[4]/str[@name='id'][.='c']");
// Test exclusive (not to be confused with exclusion)
args.put(QueryElevationParams.EXCLUSIVE, "true");
req.close();
req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
booster.setTopQueryResults(reader, query, new String[] {"x", "a"}, new String[] {});
assertQ(
null,
req,
"//*[@numFound='2']",
"//result/doc[1]/str[@name='id'][.='x']",
"//result/doc[2]/str[@name='id'][.='a']");
// Test exclusion
booster.elevationCache.clear();
args.remove(CommonParams.SORT);
args.remove(QueryElevationParams.EXCLUSIVE);
req.close();
req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
booster.setTopQueryResults(reader, query, new String[] {"x"}, new String[] {"a"});
assertQ(
null,
req,
"//*[@numFound='3']",
"//result/doc[1]/str[@name='id'][.='x']",
"//result/doc[2]/str[@name='id'][.='c']",
"//result/doc[3]/str[@name='id'][.='b']");
// Test setting ids and excludes from http parameters
booster.elevationCache.clear();
args.put(QueryElevationParams.IDS, "x,y,z");
args.put(QueryElevationParams.EXCLUDE, "b");
req.close();
req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
assertQ(
"All five should make it",
req,
"//*[@numFound='5']",
"//result/doc[1]/str[@name='id'][.='x']",
"//result/doc[2]/str[@name='id'][.='y']",
"//result/doc[3]/str[@name='id'][.='z']",
"//result/doc[4]/str[@name='id'][.='c']",
"//result/doc[5]/str[@name='id'][.='a']");
args.put(QueryElevationParams.IDS, "x,z,y");
args.put(QueryElevationParams.EXCLUDE, "b,c");
req.close();
req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args));
assertQ(
"All four should make it",
req,
"//*[@numFound='4']",
"//result/doc[1]/str[@name='id'][.='x']",
"//result/doc[2]/str[@name='id'][.='z']",
"//result/doc[3]/str[@name='id'][.='y']",
"//result/doc[4]/str[@name='id'][.='a']");
req.close();
} finally {
delete();
}
}
private List<Node> setupRequest(int hash) {
List<Node> nodes = null;
// if we are in zk mode...
if (zkEnabled) {
// set num nodes
numNodes = zkController.getClusterState().getLiveNodes().size();
String shardId =
getShard(
hash,
collection,
zkController.getClusterState()); // get the right shard based on the hash...
try {
ZkCoreNodeProps leaderProps =
new ZkCoreNodeProps(
zkController.getZkStateReader().getLeaderProps(collection, shardId));
String leaderNodeName = leaderProps.getCoreNodeName();
String coreName = req.getCore().getName();
String coreNodeName = zkController.getNodeName() + "_" + coreName;
isLeader = coreNodeName.equals(leaderNodeName);
DistribPhase phase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
doDefensiveChecks(shardId, phase);
if (DistribPhase.FROMLEADER == phase) {
// we are coming from the leader, just go local - add no urls
forwardToLeader = false;
} else if (isLeader) {
// that means I want to forward onto my replicas...
// so get the replicas...
forwardToLeader = false;
List<ZkCoreNodeProps> replicaProps =
zkController
.getZkStateReader()
.getReplicaProps(
collection,
shardId,
zkController.getNodeName(),
coreName,
null,
ZkStateReader.DOWN);
if (replicaProps != null) {
nodes = new ArrayList<Node>(replicaProps.size());
// check for test param that lets us miss replicas
String[] skipList = req.getParams().getParams("test.distrib.skip.servers");
Set<String> skipListSet = null;
if (skipList != null) {
skipListSet = new HashSet<String>(skipList.length);
skipListSet.addAll(Arrays.asList(skipList));
}
for (ZkCoreNodeProps props : replicaProps) {
if (skipList != null) {
if (!skipListSet.contains(props.getCoreUrl())) {
nodes.add(new StdNode(props));
}
} else {
nodes.add(new StdNode(props));
}
}
}
} else {
// I need to forward onto the leader...
nodes = new ArrayList<Node>(1);
nodes.add(
new RetryNode(leaderProps, zkController.getZkStateReader(), collection, shardId));
forwardToLeader = true;
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
}
}
return nodes;
}
