@Override public Set<Object> loadAllKeys(Set<Object> keysToExclude) throws CacheLoaderException { Cassandra.Client cassandraClient = null; try { cassandraClient = dataSource.getConnection(); Set<Object> s = new HashSet<Object>(); SlicePredicate slicePredicate = new SlicePredicate(); slicePredicate.setSlice_range( new SliceRange( ByteBuffer.wrap(entryColumnPath.getColumn()), ByteBufferUtil.EMPTY_BYTE_BUFFER, false, 1)); String startKey = ""; boolean complete = false; // Get the keys in SLICE_SIZE blocks while (!complete) { KeyRange keyRange = new KeyRange(SLICE_SIZE); keyRange.setStart_token(startKey); keyRange.setEnd_token(""); List<KeySlice> keySlices = cassandraClient.get_range_slices( entryColumnParent, slicePredicate, keyRange, readConsistencyLevel); if (keySlices.size() < SLICE_SIZE) { complete = true; } else { startKey = new String(keySlices.get(keySlices.size() - 1).getKey(), UTF8Charset); } for (KeySlice keySlice : keySlices) { if (keySlice.getColumnsSize() > 0) { Object key = unhashKey(keySlice.getKey()); if (key != null && (keysToExclude == null || !keysToExclude.contains(key))) s.add(key); } } } return s; } catch (Exception e) { throw new CacheLoaderException(e); } finally { dataSource.releaseConnection(cassandraClient); } }
@Override public void clear() throws CacheLoaderException { Cassandra.Client cassandraClient = null; try { cassandraClient = dataSource.getConnection(); SlicePredicate slicePredicate = new SlicePredicate(); slicePredicate.setSlice_range( new SliceRange( ByteBuffer.wrap(entryColumnPath.getColumn()), ByteBufferUtil.EMPTY_BYTE_BUFFER, false, 1)); String startKey = ""; boolean complete = false; // Get the keys in SLICE_SIZE blocks while (!complete) { KeyRange keyRange = new KeyRange(SLICE_SIZE); keyRange.setStart_token(startKey); keyRange.setEnd_token(""); List<KeySlice> keySlices = cassandraClient.get_range_slices( entryColumnParent, slicePredicate, keyRange, readConsistencyLevel); if (keySlices.size() < SLICE_SIZE) { complete = true; } else { startKey = new String(keySlices.get(keySlices.size() - 1).getKey(), UTF8Charset); } Map<ByteBuffer, Map<String, List<Mutation>>> mutationMap = new HashMap<ByteBuffer, Map<String, List<Mutation>>>(); for (KeySlice keySlice : keySlices) { remove0(ByteBuffer.wrap(keySlice.getKey()), mutationMap); } cassandraClient.batch_mutate(mutationMap, ConsistencyLevel.ALL); } } catch (Exception e) { throw new CacheLoaderException(e); } finally { dataSource.releaseConnection(cassandraClient); } }
private void loadTerms(Term skipTo) { if (initTerm == null) initTerm = skipTo; // chose starting term String startTerm = CassandraUtils.hashKey( indexName + CassandraUtils.delimeter + CassandraUtils.createColumnName(skipTo)); // ending term. the initial query we don't care since // we only pull 2 terms, also we don't String endTerm = ""; // The boundary condition for this search. currently the field. String boundryTerm = CassandraUtils.hashKey( indexName + CassandraUtils.delimeter + CassandraUtils.createColumnName(skipTo.field(), CassandraUtils.finalToken)); if ((!skipTo.equals(chunkBoundryTerm) || termPosition == 0) && termCache != null) { termDocFreqBuffer = termCache.subMap(skipTo, termCache.lastKey()); } else { termDocFreqBuffer = null; } if (termDocFreqBuffer != null) { termBuffer = termDocFreqBuffer.keySet().toArray(new Term[] {}); termPosition = 0; logger.debug("Found " + startTerm + " in cache"); return; } else if (chunkCount > 1 && actualInitSize < maxChunkSize) { // include last term if (skipTo.equals(chunkBoundryTerm) && termCache.containsKey(skipTo)) { termBuffer = new Term[] {skipTo}; termDocFreqBuffer = termCache.subMap(skipTo, termCache.lastKey()); } else { termBuffer = new Term[] {}; } termPosition = 0; return; // done! } chunkCount++; // The first time we grab just a few keys int count = maxInitSize; // otherwise we grab all the rest of the keys if (chunkBoundryTerm != null) { count = maxChunkSize; startTerm = CassandraUtils.hashKey( indexName + CassandraUtils.delimeter + CassandraUtils.createColumnName(chunkBoundryTerm)); // After first pass use the boundary term, since we know on pass 2 we are using the OPP endTerm = boundryTerm; } long start = System.currentTimeMillis(); termDocFreqBuffer = new TreeMap<Term, List<ColumnOrSuperColumn>>(); ColumnParent columnParent = new ColumnParent(CassandraUtils.termVecColumnFamily); SlicePredicate slicePredicate = new SlicePredicate(); // Get all columns SliceRange sliceRange = new SliceRange(new byte[] {}, new byte[] {}, true, Integer.MAX_VALUE); slicePredicate.setSlice_range(sliceRange); List<KeySlice> columns; try { columns = client.get_range_slice( CassandraUtils.keySpace, columnParent, slicePredicate, startTerm, endTerm, count, ConsistencyLevel.ONE); } catch (InvalidRequestException e) { throw new RuntimeException(e); } catch (TException e) { throw new RuntimeException(e); } catch (UnavailableException e) { throw new RuntimeException(e); } catch (TimedOutException e) { throw new RuntimeException(e); } // term to start with next time actualInitSize = columns.size(); logger.debug( "Found " + columns.size() + " keys in range:" + startTerm + " to " + endTerm + " in " + (System.currentTimeMillis() - start) + "ms"); if (actualInitSize > 0) { for (KeySlice entry : columns) { // term keys look like wikipedia/body/wiki String termStr = entry .getKey() .substring( entry.getKey().indexOf(CassandraUtils.delimeter) + CassandraUtils.delimeter.length()); Term term = CassandraUtils.parseTerm(termStr); logger.debug(termStr + " has " + entry.getColumns().size()); // check for tombstone keys or incorrect keys (from RP) if (entry.getColumns().size() > 0 && term.field().equals(skipTo.field()) && // from this index entry .getKey() .equals( CassandraUtils.hashKey( indexName + CassandraUtils.delimeter + term.field() + CassandraUtils.delimeter + term.text()))) termDocFreqBuffer.put(term, entry.getColumns()); } if (!termDocFreqBuffer.isEmpty()) { chunkBoundryTerm = termDocFreqBuffer.lastKey(); } } // add a final key (excluded in submap below) termDocFreqBuffer.put(finalTerm, null); // put in cache for (Term termKey : termDocFreqBuffer.keySet()) { if (termCache == null) { termCache = termDocFreqBuffer; } else { termCache.putAll(termDocFreqBuffer); } indexReader.addTermEnumCache(termKey, this); } // cache the initial term too indexReader.addTermEnumCache(skipTo, this); termBuffer = termDocFreqBuffer.keySet().toArray(new Term[] {}); termPosition = 0; long end = System.currentTimeMillis(); logger.debug( "loadTerms: " + startTerm + "(" + termBuffer.length + ") took " + (end - start) + "ms"); }
@Override public Set<InternalCacheEntry> load(int numEntries) throws CacheLoaderException { Cassandra.Client cassandraClient = null; try { cassandraClient = dataSource.getConnection(); Set<InternalCacheEntry> s = new HashSet<InternalCacheEntry>(); SlicePredicate slicePredicate = new SlicePredicate(); slicePredicate.setSlice_range( new SliceRange( ByteBuffer.wrap(entryColumnPath.getColumn()), ByteBufferUtil.EMPTY_BYTE_BUFFER, false, 1)); String startKey = ""; // Get the keys in SLICE_SIZE blocks int sliceSize = Math.min(SLICE_SIZE, numEntries); for (boolean complete = false; !complete; ) { KeyRange keyRange = new KeyRange(sliceSize); keyRange.setStart_token(startKey); keyRange.setEnd_token(""); List<KeySlice> keySlices = cassandraClient.get_range_slices( entryColumnParent, slicePredicate, keyRange, readConsistencyLevel); // Cycle through all the keys for (KeySlice keySlice : keySlices) { Object key = unhashKey(keySlice.getKey()); if (key == null) // Skip invalid keys continue; List<ColumnOrSuperColumn> columns = keySlice.getColumns(); if (columns.size() > 0) { if (log.isDebugEnabled()) { log.debugf("Loading %s", key); } byte[] value = columns.get(0).getColumn().getValue(); InternalCacheEntry ice = unmarshall(value, key); s.add(ice); } else if (log.isDebugEnabled()) { log.debugf("Skipping empty key %s", key); } } if (keySlices.size() < sliceSize) { // Cassandra has returned less keys than what we asked for. // Assume we have finished complete = true; } else { // Cassandra has returned exactly the amount of keys we // asked for. If we haven't reached the required quota yet, // assume we need to cycle again starting from // the last returned key (excluded) sliceSize = Math.min(SLICE_SIZE, numEntries - s.size()); if (sliceSize == 0) { complete = true; } else { startKey = new String(keySlices.get(keySlices.size() - 1).getKey(), UTF8Charset); } } } return s; } catch (Exception e) { throw new CacheLoaderException(e); } finally { dataSource.releaseConnection(cassandraClient); } }