@Override public void processDelete(DeleteUpdateCommand cmd) throws IOException { if (!cmd.isDeleteById()) { doDeleteByQuery(cmd); return; } int hash = 0; if (zkEnabled) { zkCheck(); hash = hash(cmd); nodes = setupRequest(hash); } else { isLeader = getNonZkLeaderAssumption(req); } boolean dropCmd = false; if (!forwardToLeader) { dropCmd = versionDelete(cmd); } if (dropCmd) { // TODO: do we need to add anything to the response? return; } ModifiableSolrParams params = null; if (nodes != null) { params = new ModifiableSolrParams(filterParams(req.getParams())); params.set( DISTRIB_UPDATE_PARAM, (isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString())); if (isLeader) { params.set( "distrib.from", ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName())); } cmdDistrib.distribDelete(cmd, nodes, params); } // cmd.getIndexId == null when delete by query // TODO: what to do when no idField? if (returnVersions && rsp != null && cmd.getIndexedId() != null && idField != null) { if (deleteResponse == null) { deleteResponse = new NamedList<String>(); rsp.add("deletes", deleteResponse); } if (scratch == null) scratch = new CharsRef(); idField.getType().indexedToReadable(cmd.getIndexedId(), scratch); deleteResponse.add( scratch.toString(), cmd .getVersion()); // we're returning the version of the delete.. not the version of the // doc we deleted. } }
@Override public void processAdd(AddUpdateCommand cmd) throws IOException { // TODO: check for id field? int hash = 0; if (zkEnabled) { zkCheck(); hash = hash(cmd); nodes = setupRequest(hash); } else { isLeader = getNonZkLeaderAssumption(req); } boolean dropCmd = false; if (!forwardToLeader) { dropCmd = versionAdd(cmd); } if (dropCmd) { // TODO: do we need to add anything to the response? return; } ModifiableSolrParams params = null; if (nodes != null) { params = new ModifiableSolrParams(filterParams(req.getParams())); params.set( DISTRIB_UPDATE_PARAM, (isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString())); if (isLeader) { params.set( "distrib.from", ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName())); } params.set( "distrib.from", ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName())); cmdDistrib.distribAdd(cmd, nodes, params); } // TODO: what to do when no idField? if (returnVersions && rsp != null && idField != null) { if (addsResponse == null) { addsResponse = new NamedList<String>(); rsp.add("adds", addsResponse); } if (scratch == null) scratch = new CharsRef(); idField.getType().indexedToReadable(cmd.getIndexedId(), scratch); addsResponse.add(scratch.toString(), cmd.getVersion()); } // TODO: keep track of errors? needs to be done at a higher level though since // an id may fail before it gets to this processor. // Given that, it may also make sense to move the version reporting out of this // processor too. }
public void doDeleteByQuery(DeleteUpdateCommand cmd) throws IOException { // even in non zk mode, tests simulate updates from a leader if (!zkEnabled) { isLeader = getNonZkLeaderAssumption(req); } else { zkCheck(); } // NONE: we are the first to receive this deleteByQuery // - it must be forwarded to the leader of every shard // TO: we are a leader receiving a forwarded deleteByQuery... we must: // - block all updates (use VersionInfo) // - flush *all* updates going to our replicas // - forward the DBQ to our replicas and wait for the response // - log + execute the local DBQ // FROM: we are a replica receiving a DBQ from our leader // - log + execute the local DBQ DistribPhase phase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM)); if (zkEnabled && DistribPhase.NONE == phase) { boolean leaderForAnyShard = false; // start off by assuming we are not a leader for any shard Map<String, Slice> slices = zkController.getClusterState().getSlices(collection); if (slices == null) { throw new SolrException( ErrorCode.BAD_REQUEST, "Cannot find collection:" + collection + " in " + zkController.getClusterState().getCollections()); } ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams())); params.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString()); List<Node> leaders = new ArrayList<Node>(slices.size()); for (Map.Entry<String, Slice> sliceEntry : slices.entrySet()) { String sliceName = sliceEntry.getKey(); ZkNodeProps leaderProps; try { leaderProps = zkController.getZkStateReader().getLeaderProps(collection, sliceName); } catch (InterruptedException e) { throw new SolrException( ErrorCode.SERVICE_UNAVAILABLE, "Exception finding leader for shard " + sliceName, e); } // TODO: What if leaders changed in the meantime? // should we send out slice-at-a-time and if a node returns "hey, I'm not a leader" (or we // get an error because it went down) then look up the new leader? // Am I the leader for this slice? ZkCoreNodeProps coreLeaderProps = new ZkCoreNodeProps(leaderProps); String leaderNodeName = coreLeaderProps.getCoreNodeName(); String coreName = req.getCore().getName(); String coreNodeName = zkController.getNodeName() + "_" + coreName; isLeader = coreNodeName.equals(leaderNodeName); if (isLeader) { // don't forward to ourself leaderForAnyShard = true; } else { leaders.add(new StdNode(coreLeaderProps)); } } params.remove("commit"); // this will be distributed from the local commit cmdDistrib.distribDelete(cmd, leaders, params); if (!leaderForAnyShard) { return; } // change the phase to TOLEADER so we look up and forward to our own replicas (if any) phase = DistribPhase.TOLEADER; } List<Node> replicas = null; if (zkEnabled && DistribPhase.TOLEADER == phase) { // This core should be a leader isLeader = true; replicas = setupRequest(); } else if (DistribPhase.FROMLEADER == phase) { isLeader = false; } if (vinfo == null) { super.processDelete(cmd); return; } // at this point, there is an update we need to try and apply. // we may or may not be the leader. // Find the version long versionOnUpdate = cmd.getVersion(); if (versionOnUpdate == 0) { String versionOnUpdateS = req.getParams().get(VERSION_FIELD); versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS); } versionOnUpdate = Math.abs(versionOnUpdate); // normalize to positive version boolean isReplay = (cmd.getFlags() & UpdateCommand.REPLAY) != 0; boolean leaderLogic = isLeader && !isReplay; if (!leaderLogic && versionOnUpdate == 0) { throw new SolrException(ErrorCode.BAD_REQUEST, "missing _version_ on update from leader"); } vinfo.blockUpdates(); try { if (versionsStored) { if (leaderLogic) { long version = vinfo.getNewClock(); cmd.setVersion(-version); // TODO update versions in all buckets doLocalDelete(cmd); } else { cmd.setVersion(-versionOnUpdate); if (ulog.getState() != UpdateLog.State.ACTIVE && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) { // we're not in an active state, and this update isn't from a replay, so buffer it. cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); ulog.deleteByQuery(cmd); return; } doLocalDelete(cmd); } } // since we don't know which documents were deleted, the easiest thing to do is to invalidate // all real-time caches (i.e. UpdateLog) which involves also getting a new version of the // IndexReader // (so cache misses will see up-to-date data) } finally { vinfo.unblockUpdates(); } // forward to all replicas if (leaderLogic && replicas != null) { ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams())); params.set(VERSION_FIELD, Long.toString(cmd.getVersion())); params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString()); params.set( "update.from", ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName())); cmdDistrib.distribDelete(cmd, replicas, params); cmdDistrib.finish(); } if (returnVersions && rsp != null) { if (deleteByQueryResponse == null) { deleteByQueryResponse = new NamedList<String>(); rsp.add("deleteByQuery", deleteByQueryResponse); } deleteByQueryResponse.add(cmd.getQuery(), cmd.getVersion()); } }