@Override
public Set<Object> loadAllKeys(Set<Object> keysToExclude) throws CacheLoaderException {
Cassandra.Client cassandraClient = null;
try {
cassandraClient = dataSource.getConnection();
Set<Object> s = new HashSet<Object>();
SlicePredicate slicePredicate = new SlicePredicate();
slicePredicate.setSlice_range(
new SliceRange(
ByteBuffer.wrap(entryColumnPath.getColumn()),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
false,
1));
String startKey = "";
boolean complete = false;
// Get the keys in SLICE_SIZE blocks
while (!complete) {
KeyRange keyRange = new KeyRange(SLICE_SIZE);
keyRange.setStart_token(startKey);
keyRange.setEnd_token("");
List<KeySlice> keySlices =
cassandraClient.get_range_slices(
entryColumnParent, slicePredicate, keyRange, readConsistencyLevel);
if (keySlices.size() < SLICE_SIZE) {
complete = true;
} else {
startKey = new String(keySlices.get(keySlices.size() - 1).getKey(), UTF8Charset);
}
for (KeySlice keySlice : keySlices) {
if (keySlice.getColumnsSize() > 0) {
Object key = unhashKey(keySlice.getKey());
if (key != null && (keysToExclude == null || !keysToExclude.contains(key))) s.add(key);
}
}
}
return s;
} catch (Exception e) {
throw new CacheLoaderException(e);
} finally {
dataSource.releaseConnection(cassandraClient);
}
}
@Override
public void clear() throws CacheLoaderException {
Cassandra.Client cassandraClient = null;
try {
cassandraClient = dataSource.getConnection();
SlicePredicate slicePredicate = new SlicePredicate();
slicePredicate.setSlice_range(
new SliceRange(
ByteBuffer.wrap(entryColumnPath.getColumn()),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
false,
1));
String startKey = "";
boolean complete = false;
// Get the keys in SLICE_SIZE blocks
while (!complete) {
KeyRange keyRange = new KeyRange(SLICE_SIZE);
keyRange.setStart_token(startKey);
keyRange.setEnd_token("");
List<KeySlice> keySlices =
cassandraClient.get_range_slices(
entryColumnParent, slicePredicate, keyRange, readConsistencyLevel);
if (keySlices.size() < SLICE_SIZE) {
complete = true;
} else {
startKey = new String(keySlices.get(keySlices.size() - 1).getKey(), UTF8Charset);
}
Map<ByteBuffer, Map<String, List<Mutation>>> mutationMap =
new HashMap<ByteBuffer, Map<String, List<Mutation>>>();
for (KeySlice keySlice : keySlices) {
remove0(ByteBuffer.wrap(keySlice.getKey()), mutationMap);
}
cassandraClient.batch_mutate(mutationMap, ConsistencyLevel.ALL);
}
} catch (Exception e) {
throw new CacheLoaderException(e);
} finally {
dataSource.releaseConnection(cassandraClient);
}
}
private void loadTerms(Term skipTo) {
if (initTerm == null) initTerm = skipTo;
// chose starting term
String startTerm =
CassandraUtils.hashKey(
indexName + CassandraUtils.delimeter + CassandraUtils.createColumnName(skipTo));
// ending term. the initial query we don't care since
// we only pull 2 terms, also we don't
String endTerm = "";
// The boundary condition for this search. currently the field.
String boundryTerm =
CassandraUtils.hashKey(
indexName
+ CassandraUtils.delimeter
+ CassandraUtils.createColumnName(skipTo.field(), CassandraUtils.finalToken));
if ((!skipTo.equals(chunkBoundryTerm) || termPosition == 0) && termCache != null) {
termDocFreqBuffer = termCache.subMap(skipTo, termCache.lastKey());
} else {
termDocFreqBuffer = null;
}
if (termDocFreqBuffer != null) {
termBuffer = termDocFreqBuffer.keySet().toArray(new Term[] {});
termPosition = 0;
logger.debug("Found " + startTerm + " in cache");
return;
} else if (chunkCount > 1 && actualInitSize < maxChunkSize) {
// include last term
if (skipTo.equals(chunkBoundryTerm) && termCache.containsKey(skipTo)) {
termBuffer = new Term[] {skipTo};
termDocFreqBuffer = termCache.subMap(skipTo, termCache.lastKey());
} else {
termBuffer = new Term[] {};
}
termPosition = 0;
return; // done!
}
chunkCount++;
// The first time we grab just a few keys
int count = maxInitSize;
// otherwise we grab all the rest of the keys
if (chunkBoundryTerm != null) {
count = maxChunkSize;
startTerm =
CassandraUtils.hashKey(
indexName
+ CassandraUtils.delimeter
+ CassandraUtils.createColumnName(chunkBoundryTerm));
// After first pass use the boundary term, since we know on pass 2 we are using the OPP
endTerm = boundryTerm;
}
long start = System.currentTimeMillis();
termDocFreqBuffer = new TreeMap<Term, List<ColumnOrSuperColumn>>();
ColumnParent columnParent = new ColumnParent(CassandraUtils.termVecColumnFamily);
SlicePredicate slicePredicate = new SlicePredicate();
// Get all columns
SliceRange sliceRange = new SliceRange(new byte[] {}, new byte[] {}, true, Integer.MAX_VALUE);
slicePredicate.setSlice_range(sliceRange);
List<KeySlice> columns;
try {
columns =
client.get_range_slice(
CassandraUtils.keySpace,
columnParent,
slicePredicate,
startTerm,
endTerm,
count,
ConsistencyLevel.ONE);
} catch (InvalidRequestException e) {
throw new RuntimeException(e);
} catch (TException e) {
throw new RuntimeException(e);
} catch (UnavailableException e) {
throw new RuntimeException(e);
} catch (TimedOutException e) {
throw new RuntimeException(e);
}
// term to start with next time
actualInitSize = columns.size();
logger.debug(
"Found "
+ columns.size()
+ " keys in range:"
+ startTerm
+ " to "
+ endTerm
+ " in "
+ (System.currentTimeMillis() - start)
+ "ms");
if (actualInitSize > 0) {
for (KeySlice entry : columns) {
// term keys look like wikipedia/body/wiki
String termStr =
entry
.getKey()
.substring(
entry.getKey().indexOf(CassandraUtils.delimeter)
+ CassandraUtils.delimeter.length());
Term term = CassandraUtils.parseTerm(termStr);
logger.debug(termStr + " has " + entry.getColumns().size());
// check for tombstone keys or incorrect keys (from RP)
if (entry.getColumns().size() > 0
&& term.field().equals(skipTo.field())
&&
// from this index
entry
.getKey()
.equals(
CassandraUtils.hashKey(
indexName
+ CassandraUtils.delimeter
+ term.field()
+ CassandraUtils.delimeter
+ term.text()))) termDocFreqBuffer.put(term, entry.getColumns());
}
if (!termDocFreqBuffer.isEmpty()) {
chunkBoundryTerm = termDocFreqBuffer.lastKey();
}
}
// add a final key (excluded in submap below)
termDocFreqBuffer.put(finalTerm, null);
// put in cache
for (Term termKey : termDocFreqBuffer.keySet()) {
if (termCache == null) {
termCache = termDocFreqBuffer;
} else {
termCache.putAll(termDocFreqBuffer);
}
indexReader.addTermEnumCache(termKey, this);
}
// cache the initial term too
indexReader.addTermEnumCache(skipTo, this);
termBuffer = termDocFreqBuffer.keySet().toArray(new Term[] {});
termPosition = 0;
long end = System.currentTimeMillis();
logger.debug(
"loadTerms: " + startTerm + "(" + termBuffer.length + ") took " + (end - start) + "ms");
}
@Override
public Set<InternalCacheEntry> load(int numEntries) throws CacheLoaderException {
Cassandra.Client cassandraClient = null;
try {
cassandraClient = dataSource.getConnection();
Set<InternalCacheEntry> s = new HashSet<InternalCacheEntry>();
SlicePredicate slicePredicate = new SlicePredicate();
slicePredicate.setSlice_range(
new SliceRange(
ByteBuffer.wrap(entryColumnPath.getColumn()),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
false,
1));
String startKey = "";
// Get the keys in SLICE_SIZE blocks
int sliceSize = Math.min(SLICE_SIZE, numEntries);
for (boolean complete = false; !complete; ) {
KeyRange keyRange = new KeyRange(sliceSize);
keyRange.setStart_token(startKey);
keyRange.setEnd_token("");
List<KeySlice> keySlices =
cassandraClient.get_range_slices(
entryColumnParent, slicePredicate, keyRange, readConsistencyLevel);
// Cycle through all the keys
for (KeySlice keySlice : keySlices) {
Object key = unhashKey(keySlice.getKey());
if (key == null) // Skip invalid keys
continue;
List<ColumnOrSuperColumn> columns = keySlice.getColumns();
if (columns.size() > 0) {
if (log.isDebugEnabled()) {
log.debugf("Loading %s", key);
}
byte[] value = columns.get(0).getColumn().getValue();
InternalCacheEntry ice = unmarshall(value, key);
s.add(ice);
} else if (log.isDebugEnabled()) {
log.debugf("Skipping empty key %s", key);
}
}
if (keySlices.size() < sliceSize) {
// Cassandra has returned less keys than what we asked for.
// Assume we have finished
complete = true;
} else {
// Cassandra has returned exactly the amount of keys we
// asked for. If we haven't reached the required quota yet,
// assume we need to cycle again starting from
// the last returned key (excluded)
sliceSize = Math.min(SLICE_SIZE, numEntries - s.size());
if (sliceSize == 0) {
complete = true;
} else {
startKey = new String(keySlices.get(keySlices.size() - 1).getKey(), UTF8Charset);
}
}
}
return s;
} catch (Exception e) {
throw new CacheLoaderException(e);
} finally {
dataSource.releaseConnection(cassandraClient);
}
}