예제 #1
0
  @Override
  public Set<Object> loadAllKeys(Set<Object> keysToExclude) throws CacheLoaderException {
    Cassandra.Client cassandraClient = null;
    try {
      cassandraClient = dataSource.getConnection();
      Set<Object> s = new HashSet<Object>();
      SlicePredicate slicePredicate = new SlicePredicate();
      slicePredicate.setSlice_range(
          new SliceRange(
              ByteBuffer.wrap(entryColumnPath.getColumn()),
              ByteBufferUtil.EMPTY_BYTE_BUFFER,
              false,
              1));
      String startKey = "";
      boolean complete = false;
      // Get the keys in SLICE_SIZE blocks
      while (!complete) {
        KeyRange keyRange = new KeyRange(SLICE_SIZE);
        keyRange.setStart_token(startKey);
        keyRange.setEnd_token("");
        List<KeySlice> keySlices =
            cassandraClient.get_range_slices(
                entryColumnParent, slicePredicate, keyRange, readConsistencyLevel);
        if (keySlices.size() < SLICE_SIZE) {
          complete = true;
        } else {
          startKey = new String(keySlices.get(keySlices.size() - 1).getKey(), UTF8Charset);
        }

        for (KeySlice keySlice : keySlices) {
          if (keySlice.getColumnsSize() > 0) {
            Object key = unhashKey(keySlice.getKey());
            if (key != null && (keysToExclude == null || !keysToExclude.contains(key))) s.add(key);
          }
        }
      }
      return s;
    } catch (Exception e) {
      throw new CacheLoaderException(e);
    } finally {
      dataSource.releaseConnection(cassandraClient);
    }
  }
예제 #2
0
  @Override
  public void clear() throws CacheLoaderException {
    Cassandra.Client cassandraClient = null;
    try {
      cassandraClient = dataSource.getConnection();
      SlicePredicate slicePredicate = new SlicePredicate();
      slicePredicate.setSlice_range(
          new SliceRange(
              ByteBuffer.wrap(entryColumnPath.getColumn()),
              ByteBufferUtil.EMPTY_BYTE_BUFFER,
              false,
              1));
      String startKey = "";
      boolean complete = false;
      // Get the keys in SLICE_SIZE blocks
      while (!complete) {
        KeyRange keyRange = new KeyRange(SLICE_SIZE);
        keyRange.setStart_token(startKey);
        keyRange.setEnd_token("");
        List<KeySlice> keySlices =
            cassandraClient.get_range_slices(
                entryColumnParent, slicePredicate, keyRange, readConsistencyLevel);
        if (keySlices.size() < SLICE_SIZE) {
          complete = true;
        } else {
          startKey = new String(keySlices.get(keySlices.size() - 1).getKey(), UTF8Charset);
        }
        Map<ByteBuffer, Map<String, List<Mutation>>> mutationMap =
            new HashMap<ByteBuffer, Map<String, List<Mutation>>>();

        for (KeySlice keySlice : keySlices) {
          remove0(ByteBuffer.wrap(keySlice.getKey()), mutationMap);
        }
        cassandraClient.batch_mutate(mutationMap, ConsistencyLevel.ALL);
      }
    } catch (Exception e) {
      throw new CacheLoaderException(e);
    } finally {
      dataSource.releaseConnection(cassandraClient);
    }
  }
예제 #3
0
  private void loadTerms(Term skipTo) {

    if (initTerm == null) initTerm = skipTo;

    // chose starting term
    String startTerm =
        CassandraUtils.hashKey(
            indexName + CassandraUtils.delimeter + CassandraUtils.createColumnName(skipTo));

    // ending term. the initial query we don't care since
    // we only pull 2 terms, also we don't
    String endTerm = "";

    // The boundary condition for this search. currently the field.
    String boundryTerm =
        CassandraUtils.hashKey(
            indexName
                + CassandraUtils.delimeter
                + CassandraUtils.createColumnName(skipTo.field(), CassandraUtils.finalToken));

    if ((!skipTo.equals(chunkBoundryTerm) || termPosition == 0) && termCache != null) {
      termDocFreqBuffer = termCache.subMap(skipTo, termCache.lastKey());
    } else {
      termDocFreqBuffer = null;
    }

    if (termDocFreqBuffer != null) {

      termBuffer = termDocFreqBuffer.keySet().toArray(new Term[] {});
      termPosition = 0;

      logger.debug("Found " + startTerm + " in cache");
      return;
    } else if (chunkCount > 1 && actualInitSize < maxChunkSize) {

      // include last term
      if (skipTo.equals(chunkBoundryTerm) && termCache.containsKey(skipTo)) {
        termBuffer = new Term[] {skipTo};
        termDocFreqBuffer = termCache.subMap(skipTo, termCache.lastKey());
      } else {
        termBuffer = new Term[] {};
      }

      termPosition = 0;
      return; // done!
    }

    chunkCount++;

    // The first time we grab just a few keys
    int count = maxInitSize;

    // otherwise we grab all the rest of the keys
    if (chunkBoundryTerm != null) {
      count = maxChunkSize;
      startTerm =
          CassandraUtils.hashKey(
              indexName
                  + CassandraUtils.delimeter
                  + CassandraUtils.createColumnName(chunkBoundryTerm));

      // After first pass use the boundary term, since we know on pass 2 we are using the OPP
      endTerm = boundryTerm;
    }

    long start = System.currentTimeMillis();

    termDocFreqBuffer = new TreeMap<Term, List<ColumnOrSuperColumn>>();

    ColumnParent columnParent = new ColumnParent(CassandraUtils.termVecColumnFamily);
    SlicePredicate slicePredicate = new SlicePredicate();

    // Get all columns
    SliceRange sliceRange = new SliceRange(new byte[] {}, new byte[] {}, true, Integer.MAX_VALUE);
    slicePredicate.setSlice_range(sliceRange);

    List<KeySlice> columns;
    try {
      columns =
          client.get_range_slice(
              CassandraUtils.keySpace,
              columnParent,
              slicePredicate,
              startTerm,
              endTerm,
              count,
              ConsistencyLevel.ONE);
    } catch (InvalidRequestException e) {
      throw new RuntimeException(e);
    } catch (TException e) {
      throw new RuntimeException(e);
    } catch (UnavailableException e) {
      throw new RuntimeException(e);
    } catch (TimedOutException e) {
      throw new RuntimeException(e);
    }

    // term to start with next time
    actualInitSize = columns.size();
    logger.debug(
        "Found "
            + columns.size()
            + " keys in range:"
            + startTerm
            + " to "
            + endTerm
            + " in "
            + (System.currentTimeMillis() - start)
            + "ms");

    if (actualInitSize > 0) {
      for (KeySlice entry : columns) {

        // term keys look like wikipedia/body/wiki
        String termStr =
            entry
                .getKey()
                .substring(
                    entry.getKey().indexOf(CassandraUtils.delimeter)
                        + CassandraUtils.delimeter.length());
        Term term = CassandraUtils.parseTerm(termStr);

        logger.debug(termStr + " has " + entry.getColumns().size());

        // check for tombstone keys or incorrect keys (from RP)
        if (entry.getColumns().size() > 0
            && term.field().equals(skipTo.field())
            &&
            // from this index
            entry
                .getKey()
                .equals(
                    CassandraUtils.hashKey(
                        indexName
                            + CassandraUtils.delimeter
                            + term.field()
                            + CassandraUtils.delimeter
                            + term.text()))) termDocFreqBuffer.put(term, entry.getColumns());
      }

      if (!termDocFreqBuffer.isEmpty()) {
        chunkBoundryTerm = termDocFreqBuffer.lastKey();
      }
    }

    // add a final key (excluded in submap below)
    termDocFreqBuffer.put(finalTerm, null);

    // put in cache
    for (Term termKey : termDocFreqBuffer.keySet()) {

      if (termCache == null) {
        termCache = termDocFreqBuffer;
      } else {
        termCache.putAll(termDocFreqBuffer);
      }

      indexReader.addTermEnumCache(termKey, this);
    }

    // cache the initial term too
    indexReader.addTermEnumCache(skipTo, this);

    termBuffer = termDocFreqBuffer.keySet().toArray(new Term[] {});

    termPosition = 0;

    long end = System.currentTimeMillis();

    logger.debug(
        "loadTerms: " + startTerm + "(" + termBuffer.length + ") took " + (end - start) + "ms");
  }
예제 #4
0
  @Override
  public Set<InternalCacheEntry> load(int numEntries) throws CacheLoaderException {
    Cassandra.Client cassandraClient = null;
    try {
      cassandraClient = dataSource.getConnection();
      Set<InternalCacheEntry> s = new HashSet<InternalCacheEntry>();
      SlicePredicate slicePredicate = new SlicePredicate();
      slicePredicate.setSlice_range(
          new SliceRange(
              ByteBuffer.wrap(entryColumnPath.getColumn()),
              ByteBufferUtil.EMPTY_BYTE_BUFFER,
              false,
              1));
      String startKey = "";

      // Get the keys in SLICE_SIZE blocks
      int sliceSize = Math.min(SLICE_SIZE, numEntries);
      for (boolean complete = false; !complete; ) {
        KeyRange keyRange = new KeyRange(sliceSize);
        keyRange.setStart_token(startKey);
        keyRange.setEnd_token("");
        List<KeySlice> keySlices =
            cassandraClient.get_range_slices(
                entryColumnParent, slicePredicate, keyRange, readConsistencyLevel);

        // Cycle through all the keys
        for (KeySlice keySlice : keySlices) {
          Object key = unhashKey(keySlice.getKey());
          if (key == null) // Skip invalid keys
          continue;
          List<ColumnOrSuperColumn> columns = keySlice.getColumns();
          if (columns.size() > 0) {
            if (log.isDebugEnabled()) {
              log.debugf("Loading %s", key);
            }
            byte[] value = columns.get(0).getColumn().getValue();
            InternalCacheEntry ice = unmarshall(value, key);
            s.add(ice);
          } else if (log.isDebugEnabled()) {
            log.debugf("Skipping empty key %s", key);
          }
        }
        if (keySlices.size() < sliceSize) {
          // Cassandra has returned less keys than what we asked for.
          // Assume we have finished
          complete = true;
        } else {
          // Cassandra has returned exactly the amount of keys we
          // asked for. If we haven't reached the required quota yet,
          // assume we need to cycle again starting from
          // the last returned key (excluded)
          sliceSize = Math.min(SLICE_SIZE, numEntries - s.size());
          if (sliceSize == 0) {
            complete = true;
          } else {
            startKey = new String(keySlices.get(keySlices.size() - 1).getKey(), UTF8Charset);
          }
        }
      }
      return s;
    } catch (Exception e) {
      throw new CacheLoaderException(e);
    } finally {
      dataSource.releaseConnection(cassandraClient);
    }
  }