@Override
protected int sliceHash(String id, SolrInputDocument doc, SolrParams params) {
int idx = id.indexOf(separator);
if (idx < 0) {
return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
}
int m1 = mask1;
int m2 = mask2;
String part1 = id.substring(0, idx);
int commaIdx = part1.indexOf(bitsSepartor);
if (commaIdx > 0) {
int firstBits = getBits(part1, commaIdx);
if (firstBits >= 0) {
m1 = firstBits == 0 ? 0 : (-1 << (32 - firstBits));
m2 = firstBits == 32 ? 0 : (-1 >>> firstBits);
part1 = part1.substring(0, commaIdx);
}
}
String part2 = id.substring(idx + 1);
int hash1 = Hash.murmurhash3_x86_32(part1, 0, part1.length(), 0);
int hash2 = Hash.murmurhash3_x86_32(part2, 0, part2.length(), 0);
return (hash1 & m1) | (hash2 & m2);
}
@Override
public Collection<Slice> getSearchSlicesSingle(
String shardKey, SolrParams params, DocCollection collection) {
if (shardKey == null) {
// search across whole collection
// TODO: this may need modification in the future when shard splitting could cause an overlap
return collection.getSlices();
}
String id = shardKey;
int idx = shardKey.indexOf(separator);
if (idx < 0) {
// shardKey is a simple id, so don't do a range
return Collections.singletonList(
hashToSlice(Hash.murmurhash3_x86_32(id, 0, id.length(), 0), collection));
}
int m1 = mask1;
int m2 = mask2;
String part1 = id.substring(0, idx);
int bitsSepIdx = part1.indexOf(bitsSepartor);
if (bitsSepIdx > 0) {
int firstBits = getBits(part1, bitsSepIdx);
if (firstBits >= 0) {
m1 = firstBits == 0 ? 0 : (-1 << (32 - firstBits));
m2 = firstBits == 32 ? 0 : (-1 >>> firstBits);
part1 = part1.substring(0, bitsSepIdx);
}
}
// If the upper bits are 0xF0000000, the range we want to cover is
// 0xF0000000 0xFfffffff
int hash1 = Hash.murmurhash3_x86_32(part1, 0, part1.length(), 0);
int upperBits = hash1 & m1;
int lowerBound = upperBits;
int upperBound = upperBits | m2;
if (m1 == 0) {
// no bits used from first part of key.. the code above will produce 0x000000000->0xffffffff
// which only works on unsigned space, but we're using signed space.
lowerBound = Integer.MIN_VALUE;
upperBound = Integer.MAX_VALUE;
}
Range completeRange = new Range(lowerBound, upperBound);
List<Slice> targetSlices = new ArrayList<Slice>(1);
for (Slice slice : collection.getSlices()) {
Range range = slice.getRange();
if (range != null && range.overlaps(completeRange)) {
targetSlices.add(slice);
}
}
return targetSlices;
}
private boolean versionDelete(DeleteUpdateCommand cmd) throws IOException {
BytesRef idBytes = cmd.getIndexedId();
if (vinfo == null || idBytes == null) {
super.processDelete(cmd);
return false;
}
// This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not
// use a pluggable hash here)
int bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0);
// at this point, there is an update we need to try and apply.
// we may or may not be the leader.
// Find the version
long versionOnUpdate = cmd.getVersion();
if (versionOnUpdate == 0) {
String versionOnUpdateS = req.getParams().get(VERSION_FIELD);
versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS);
}
long signedVersionOnUpdate = versionOnUpdate;
versionOnUpdate = Math.abs(versionOnUpdate); // normalize to positive version
boolean isReplay = (cmd.getFlags() & UpdateCommand.REPLAY) != 0;
boolean leaderLogic = isLeader && !isReplay;
if (!leaderLogic && versionOnUpdate == 0) {
throw new SolrException(ErrorCode.BAD_REQUEST, "missing _version_ on update from leader");
}
VersionBucket bucket = vinfo.bucket(bucketHash);
vinfo.lockForUpdate();
try {
synchronized (bucket) {
if (versionsStored) {
long bucketVersion = bucket.highest;
if (leaderLogic) {
if (signedVersionOnUpdate != 0) {
Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
long foundVersion = lastVersion == null ? -1 : lastVersion;
if ((signedVersionOnUpdate == foundVersion)
|| (signedVersionOnUpdate < 0 && foundVersion < 0)
|| (signedVersionOnUpdate == 1 && foundVersion > 0)) {
// we're ok if versions match, or if both are negative (all missing docs are equal),
// or if cmd
// specified it must exist (versionOnUpdate==1) and it does.
} else {
throw new SolrException(
ErrorCode.CONFLICT,
"version conflict for "
+ cmd.getId()
+ " expected="
+ signedVersionOnUpdate
+ " actual="
+ foundVersion);
}
}
long version = vinfo.getNewClock();
cmd.setVersion(-version);
bucket.updateHighest(version);
} else {
cmd.setVersion(-versionOnUpdate);
if (ulog.getState() != UpdateLog.State.ACTIVE
&& (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
// we're not in an active state, and this update isn't from a replay, so buffer it.
cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
ulog.delete(cmd);
return true;
}
// if we aren't the leader, then we need to check that updates were not re-ordered
if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
// we're OK... this update has a version higher than anything we've seen
// in this bucket so far, so we know that no reordering has yet occured.
bucket.updateHighest(versionOnUpdate);
} else {
// there have been updates higher than the current update. we need to check
// the specific version for this id.
Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) {
// This update is a repeat, or was reordered. We need to drop this update.
return true;
}
}
}
}
doLocalDelete(cmd);
return false;
} // end synchronized (bucket)
} finally {
vinfo.unlockForUpdate();
}
}
/**
* @return whether or not to drop this cmd
* @throws IOException If there is a low-level I/O error.
*/
private boolean versionAdd(AddUpdateCommand cmd) throws IOException {
BytesRef idBytes = cmd.getIndexedId();
if (vinfo == null || idBytes == null) {
super.processAdd(cmd);
return false;
}
// This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not
// use a pluggable hash here)
int bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0);
// at this point, there is an update we need to try and apply.
// we may or may not be the leader.
// Find any existing version in the document
// TODO: don't reuse update commands any more!
long versionOnUpdate = cmd.getVersion();
if (versionOnUpdate == 0) {
SolrInputField versionField = cmd.getSolrInputDocument().getField(VersionInfo.VERSION_FIELD);
if (versionField != null) {
Object o = versionField.getValue();
versionOnUpdate =
o instanceof Number ? ((Number) o).longValue() : Long.parseLong(o.toString());
} else {
// Find the version
String versionOnUpdateS = req.getParams().get(VERSION_FIELD);
versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS);
}
}
boolean isReplay = (cmd.getFlags() & UpdateCommand.REPLAY) != 0;
boolean leaderLogic = isLeader && !isReplay;
VersionBucket bucket = vinfo.bucket(bucketHash);
vinfo.lockForUpdate();
try {
synchronized (bucket) {
// we obtain the version when synchronized and then do the add so we can ensure that
// if version1 < version2 then version1 is actually added before version2.
// even if we don't store the version field, synchronizing on the bucket
// will enable us to know what version happened first, and thus enable
// realtime-get to work reliably.
// TODO: if versions aren't stored, do we need to set on the cmd anyway for some reason?
// there may be other reasons in the future for a version on the commands
boolean checkDeleteByQueries = false;
if (versionsStored) {
long bucketVersion = bucket.highest;
if (leaderLogic) {
boolean updated = getUpdatedDocument(cmd, versionOnUpdate);
if (versionOnUpdate != 0) {
Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
long foundVersion = lastVersion == null ? -1 : lastVersion;
if (versionOnUpdate == foundVersion
|| (versionOnUpdate < 0 && foundVersion < 0)
|| (versionOnUpdate == 1 && foundVersion > 0)) {
// we're ok if versions match, or if both are negative (all missing docs are equal),
// or if cmd
// specified it must exist (versionOnUpdate==1) and it does.
} else {
throw new SolrException(
ErrorCode.CONFLICT,
"version conflict for "
+ cmd.getPrintableId()
+ " expected="
+ versionOnUpdate
+ " actual="
+ foundVersion);
}
}
long version = vinfo.getNewClock();
cmd.setVersion(version);
cmd.getSolrInputDocument().setField(VersionInfo.VERSION_FIELD, version);
bucket.updateHighest(version);
} else {
// The leader forwarded us this update.
cmd.setVersion(versionOnUpdate);
if (ulog.getState() != UpdateLog.State.ACTIVE
&& (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
// we're not in an active state, and this update isn't from a replay, so buffer it.
cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
ulog.add(cmd);
return true;
}
// if we aren't the leader, then we need to check that updates were not re-ordered
if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
// we're OK... this update has a version higher than anything we've seen
// in this bucket so far, so we know that no reordering has yet occurred.
bucket.updateHighest(versionOnUpdate);
} else {
// there have been updates higher than the current update. we need to check
// the specific version for this id.
Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) {
// This update is a repeat, or was reordered. We need to drop this update.
return true;
}
// also need to re-apply newer deleteByQuery commands
checkDeleteByQueries = true;
}
}
}
boolean willDistrib = isLeader && nodes != null && nodes.size() > 0;
SolrInputDocument clonedDoc = null;
if (willDistrib) {
clonedDoc = cmd.solrDoc.deepCopy();
}
// TODO: possibly set checkDeleteByQueries as a flag on the command?
doLocalAdd(cmd);
if (willDistrib) {
cmd.solrDoc = clonedDoc;
}
} // end synchronized (bucket)
} finally {
vinfo.unlockForUpdate();
}
return false;
}
private int hash(DeleteUpdateCommand cmd) {
return Hash.murmurhash3_x86_32(cmd.getId(), 0, cmd.getId().length(), 0);
}
// TODO: move this to AddUpdateCommand/DeleteUpdateCommand and cache it? And
// make the hash pluggable of course.
// The hash also needs to be pluggable
private int hash(AddUpdateCommand cmd) {
String hashableId = cmd.getHashableId();
return Hash.murmurhash3_x86_32(hashableId, 0, hashableId.length(), 0);
}