@Override protected int sliceHash(String id, SolrInputDocument doc, SolrParams params) { int idx = id.indexOf(separator); if (idx < 0) { return Hash.murmurhash3_x86_32(id, 0, id.length(), 0); } int m1 = mask1; int m2 = mask2; String part1 = id.substring(0, idx); int commaIdx = part1.indexOf(bitsSepartor); if (commaIdx > 0) { int firstBits = getBits(part1, commaIdx); if (firstBits >= 0) { m1 = firstBits == 0 ? 0 : (-1 << (32 - firstBits)); m2 = firstBits == 32 ? 0 : (-1 >>> firstBits); part1 = part1.substring(0, commaIdx); } } String part2 = id.substring(idx + 1); int hash1 = Hash.murmurhash3_x86_32(part1, 0, part1.length(), 0); int hash2 = Hash.murmurhash3_x86_32(part2, 0, part2.length(), 0); return (hash1 & m1) | (hash2 & m2); }
@Override public Collection<Slice> getSearchSlicesSingle( String shardKey, SolrParams params, DocCollection collection) { if (shardKey == null) { // search across whole collection // TODO: this may need modification in the future when shard splitting could cause an overlap return collection.getSlices(); } String id = shardKey; int idx = shardKey.indexOf(separator); if (idx < 0) { // shardKey is a simple id, so don't do a range return Collections.singletonList( hashToSlice(Hash.murmurhash3_x86_32(id, 0, id.length(), 0), collection)); } int m1 = mask1; int m2 = mask2; String part1 = id.substring(0, idx); int bitsSepIdx = part1.indexOf(bitsSepartor); if (bitsSepIdx > 0) { int firstBits = getBits(part1, bitsSepIdx); if (firstBits >= 0) { m1 = firstBits == 0 ? 0 : (-1 << (32 - firstBits)); m2 = firstBits == 32 ? 0 : (-1 >>> firstBits); part1 = part1.substring(0, bitsSepIdx); } } // If the upper bits are 0xF0000000, the range we want to cover is // 0xF0000000 0xFfffffff int hash1 = Hash.murmurhash3_x86_32(part1, 0, part1.length(), 0); int upperBits = hash1 & m1; int lowerBound = upperBits; int upperBound = upperBits | m2; if (m1 == 0) { // no bits used from first part of key.. the code above will produce 0x000000000->0xffffffff // which only works on unsigned space, but we're using signed space. lowerBound = Integer.MIN_VALUE; upperBound = Integer.MAX_VALUE; } Range completeRange = new Range(lowerBound, upperBound); List<Slice> targetSlices = new ArrayList<Slice>(1); for (Slice slice : collection.getSlices()) { Range range = slice.getRange(); if (range != null && range.overlaps(completeRange)) { targetSlices.add(slice); } } return targetSlices; }
private boolean versionDelete(DeleteUpdateCommand cmd) throws IOException { BytesRef idBytes = cmd.getIndexedId(); if (vinfo == null || idBytes == null) { super.processDelete(cmd); return false; } // This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not // use a pluggable hash here) int bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0); // at this point, there is an update we need to try and apply. // we may or may not be the leader. // Find the version long versionOnUpdate = cmd.getVersion(); if (versionOnUpdate == 0) { String versionOnUpdateS = req.getParams().get(VERSION_FIELD); versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS); } long signedVersionOnUpdate = versionOnUpdate; versionOnUpdate = Math.abs(versionOnUpdate); // normalize to positive version boolean isReplay = (cmd.getFlags() & UpdateCommand.REPLAY) != 0; boolean leaderLogic = isLeader && !isReplay; if (!leaderLogic && versionOnUpdate == 0) { throw new SolrException(ErrorCode.BAD_REQUEST, "missing _version_ on update from leader"); } VersionBucket bucket = vinfo.bucket(bucketHash); vinfo.lockForUpdate(); try { synchronized (bucket) { if (versionsStored) { long bucketVersion = bucket.highest; if (leaderLogic) { if (signedVersionOnUpdate != 0) { Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); long foundVersion = lastVersion == null ? -1 : lastVersion; if ((signedVersionOnUpdate == foundVersion) || (signedVersionOnUpdate < 0 && foundVersion < 0) || (signedVersionOnUpdate == 1 && foundVersion > 0)) { // we're ok if versions match, or if both are negative (all missing docs are equal), // or if cmd // specified it must exist (versionOnUpdate==1) and it does. } else { throw new SolrException( ErrorCode.CONFLICT, "version conflict for " + cmd.getId() + " expected=" + signedVersionOnUpdate + " actual=" + foundVersion); } } long version = vinfo.getNewClock(); cmd.setVersion(-version); bucket.updateHighest(version); } else { cmd.setVersion(-versionOnUpdate); if (ulog.getState() != UpdateLog.State.ACTIVE && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) { // we're not in an active state, and this update isn't from a replay, so buffer it. cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); ulog.delete(cmd); return true; } // if we aren't the leader, then we need to check that updates were not re-ordered if (bucketVersion != 0 && bucketVersion < versionOnUpdate) { // we're OK... this update has a version higher than anything we've seen // in this bucket so far, so we know that no reordering has yet occured. bucket.updateHighest(versionOnUpdate); } else { // there have been updates higher than the current update. we need to check // the specific version for this id. Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) { // This update is a repeat, or was reordered. We need to drop this update. return true; } } } } doLocalDelete(cmd); return false; } // end synchronized (bucket) } finally { vinfo.unlockForUpdate(); } }
/** * @return whether or not to drop this cmd * @throws IOException If there is a low-level I/O error. */ private boolean versionAdd(AddUpdateCommand cmd) throws IOException { BytesRef idBytes = cmd.getIndexedId(); if (vinfo == null || idBytes == null) { super.processAdd(cmd); return false; } // This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not // use a pluggable hash here) int bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0); // at this point, there is an update we need to try and apply. // we may or may not be the leader. // Find any existing version in the document // TODO: don't reuse update commands any more! long versionOnUpdate = cmd.getVersion(); if (versionOnUpdate == 0) { SolrInputField versionField = cmd.getSolrInputDocument().getField(VersionInfo.VERSION_FIELD); if (versionField != null) { Object o = versionField.getValue(); versionOnUpdate = o instanceof Number ? ((Number) o).longValue() : Long.parseLong(o.toString()); } else { // Find the version String versionOnUpdateS = req.getParams().get(VERSION_FIELD); versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS); } } boolean isReplay = (cmd.getFlags() & UpdateCommand.REPLAY) != 0; boolean leaderLogic = isLeader && !isReplay; VersionBucket bucket = vinfo.bucket(bucketHash); vinfo.lockForUpdate(); try { synchronized (bucket) { // we obtain the version when synchronized and then do the add so we can ensure that // if version1 < version2 then version1 is actually added before version2. // even if we don't store the version field, synchronizing on the bucket // will enable us to know what version happened first, and thus enable // realtime-get to work reliably. // TODO: if versions aren't stored, do we need to set on the cmd anyway for some reason? // there may be other reasons in the future for a version on the commands boolean checkDeleteByQueries = false; if (versionsStored) { long bucketVersion = bucket.highest; if (leaderLogic) { boolean updated = getUpdatedDocument(cmd, versionOnUpdate); if (versionOnUpdate != 0) { Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); long foundVersion = lastVersion == null ? -1 : lastVersion; if (versionOnUpdate == foundVersion || (versionOnUpdate < 0 && foundVersion < 0) || (versionOnUpdate == 1 && foundVersion > 0)) { // we're ok if versions match, or if both are negative (all missing docs are equal), // or if cmd // specified it must exist (versionOnUpdate==1) and it does. } else { throw new SolrException( ErrorCode.CONFLICT, "version conflict for " + cmd.getPrintableId() + " expected=" + versionOnUpdate + " actual=" + foundVersion); } } long version = vinfo.getNewClock(); cmd.setVersion(version); cmd.getSolrInputDocument().setField(VersionInfo.VERSION_FIELD, version); bucket.updateHighest(version); } else { // The leader forwarded us this update. cmd.setVersion(versionOnUpdate); if (ulog.getState() != UpdateLog.State.ACTIVE && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) { // we're not in an active state, and this update isn't from a replay, so buffer it. cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); ulog.add(cmd); return true; } // if we aren't the leader, then we need to check that updates were not re-ordered if (bucketVersion != 0 && bucketVersion < versionOnUpdate) { // we're OK... this update has a version higher than anything we've seen // in this bucket so far, so we know that no reordering has yet occurred. bucket.updateHighest(versionOnUpdate); } else { // there have been updates higher than the current update. we need to check // the specific version for this id. Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) { // This update is a repeat, or was reordered. We need to drop this update. return true; } // also need to re-apply newer deleteByQuery commands checkDeleteByQueries = true; } } } boolean willDistrib = isLeader && nodes != null && nodes.size() > 0; SolrInputDocument clonedDoc = null; if (willDistrib) { clonedDoc = cmd.solrDoc.deepCopy(); } // TODO: possibly set checkDeleteByQueries as a flag on the command? doLocalAdd(cmd); if (willDistrib) { cmd.solrDoc = clonedDoc; } } // end synchronized (bucket) } finally { vinfo.unlockForUpdate(); } return false; }
private int hash(DeleteUpdateCommand cmd) { return Hash.murmurhash3_x86_32(cmd.getId(), 0, cmd.getId().length(), 0); }
// TODO: move this to AddUpdateCommand/DeleteUpdateCommand and cache it? And // make the hash pluggable of course. // The hash also needs to be pluggable private int hash(AddUpdateCommand cmd) { String hashableId = cmd.getHashableId(); return Hash.murmurhash3_x86_32(hashableId, 0, hashableId.length(), 0); }