@Test
  public void testStream_bytesRefArray() throws Exception {
    final BytesRefArray bArr = new BytesRefArray(Counter.newCounter(false));
    bArr.append(new BytesRef("foo"));
    bArr.append(new BytesRef("bar"));
    bArr.append(new BytesRef("baz"));

    Assert.assertEquals("Not all items streamed.", 3L, StreamUtils.stream(bArr).count());

    Assert.assertEquals(
        "Term not found.",
        1L,
        StreamUtils.stream(bArr).filter(br -> br.bytesEquals(new BytesRef("foo"))).count());
    Assert.assertEquals(
        "Term not found.",
        1L,
        StreamUtils.stream(bArr).filter(br -> br.bytesEquals(new BytesRef("bar"))).count());
    Assert.assertEquals(
        "Term not found.",
        1L,
        StreamUtils.stream(bArr).filter(br -> br.bytesEquals(new BytesRef("baz"))).count());

    Assert.assertEquals(
        "Unknown term found.",
        0L,
        StreamUtils.stream(bArr)
            .filter(
                t ->
                    !t.bytesEquals(new BytesRef("foo"))
                        && !t.bytesEquals(new BytesRef("bar"))
                        && !t.bytesEquals(new BytesRef("baz")))
            .count());
  }
  @Override
  public void readFrom(StreamInput in) throws IOException {
    this.setIsPruned(in.readBoolean());
    int size = in.readInt();

    bytesUsed = Counter.newCounter();
    pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
    set = new BytesRefHash(pool);

    for (long i = 0; i < size; i++) {
      set.add(in.readBytesRef());
    }
  }
  private void readFromBytes(BytesRef bytes) {
    // Read pruned flag
    this.setIsPruned(bytes.bytes[bytes.offset++] == 1 ? true : false);

    // Read size fo the set
    int size = Bytes.readInt(bytes);

    // Read terms
    bytesUsed = Counter.newCounter();
    pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
    set = new BytesRefHash(pool);

    BytesRef reusable = new BytesRef();
    for (int i = 0; i < size; i++) {
      Bytes.readBytesRef(bytes, reusable);
      set.add(reusable);
    }
  }
Exemple #4
0
  private PossiblyLimitedTopDocs getTopDocs(Query query, Sort sort) throws IOException {
    final TopFieldCollector topCollector =
        TopFieldCollector.create(sort, maxHits, true, false, false, false);
    final Counter clock = Counter.newCounter(true);
    final int waitMillis = 1000;
    // TODO: if we interrupt the whole thread anyway, do we still need the TimeLimitingCollector?
    final TimeLimitingCollector collector =
        new TimeLimitingCollector(topCollector, clock, maxSearchTimeMillis / waitMillis);
    collector.setBaseline(0);
    final Thread counterThread =
        new Thread() {
          @Override
          public void run() {
            final long startTime = System.currentTimeMillis();
            while (true) {
              final long runTimeMillis = System.currentTimeMillis() - startTime;
              if (runTimeMillis > maxSearchTimeMillis) {
                // make sure there's no lingering thread for too long
                return;
              }
              clock.addAndGet(1);
              try {
                Thread.sleep(waitMillis);
              } catch (InterruptedException e) {
                throw new RuntimeException(e);
              }
            }
          }
        };
    counterThread.setName("LuceneSearchTimeoutThread");
    counterThread.start();

    boolean timeLimitActivated = false;
    try {
      indexSearcher.search(query, collector);
    } catch (TimeLimitingCollector.TimeExceededException e) {
      timeLimitActivated = true;
    }
    return new PossiblyLimitedTopDocs(topCollector.topDocs(), timeLimitActivated);
  }
Exemple #5
0
 /**
  * Expert: This constructor accepts an upper limit for the number of bytes that should be reused
  * if this instance is {@link #reset()}. The payload storage, if used, is unaffected by
  * maxReusuedBytes, however.
  *
  * @param storeOffsets <code>true</code> if offsets should be stored
  * @param storePayloads <code>true</code> if payloads should be stored
  * @param maxReusedBytes the number of bytes that should remain in the internal memory pools after
  *     {@link #reset()} is called
  */
 MemoryIndex(boolean storeOffsets, boolean storePayloads, long maxReusedBytes) {
   this.storeOffsets = storeOffsets;
   this.storePayloads = storePayloads;
   this.bytesUsed = Counter.newCounter();
   final int maxBufferedByteBlocks = (int) ((maxReusedBytes / 2) / ByteBlockPool.BYTE_BLOCK_SIZE);
   final int maxBufferedIntBlocks =
       (int)
           ((maxReusedBytes - (maxBufferedByteBlocks * ByteBlockPool.BYTE_BLOCK_SIZE))
               / (IntBlockPool.INT_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT));
   assert (maxBufferedByteBlocks * ByteBlockPool.BYTE_BLOCK_SIZE)
           + (maxBufferedIntBlocks * IntBlockPool.INT_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT)
       <= maxReusedBytes;
   byteBlockPool =
       new ByteBlockPool(
           new RecyclingByteBlockAllocator(
               ByteBlockPool.BYTE_BLOCK_SIZE, maxBufferedByteBlocks, bytesUsed));
   intBlockPool =
       new IntBlockPool(
           new RecyclingIntBlockAllocator(
               IntBlockPool.INT_BLOCK_SIZE, maxBufferedIntBlocks, bytesUsed));
   postingsWriter = new SliceWriter(intBlockPool);
   // TODO refactor BytesRefArray to allow us to apply maxReusedBytes option
   payloadsBytesRefs = storePayloads ? new BytesRefArray(bytesUsed) : null;
 }
 public BytesRefTermsSet(final CircuitBreaker breaker) {
   super(breaker);
   this.bytesUsed = Counter.newCounter();
   this.pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
   this.set = new BytesRefHash(pool);
 }
 @Test
 public void testStream_bytesRefArray_empty() throws Exception {
   final BytesRefArray bArr = new BytesRefArray(Counter.newCounter(false));
   Assert.assertEquals("Too much items streamed.", 0L, StreamUtils.stream(bArr).count());
 }
/**
 * This wrapper buffers incoming elements.
 *
 * @lucene.experimental
 */
public class BufferedInputIterator implements InputIterator {
  // TODO keep this for now
  /** buffered term entries */
  protected BytesRefArray entries = new BytesRefArray(Counter.newCounter());
  /** buffered payload entries */
  protected BytesRefArray payloads = new BytesRefArray(Counter.newCounter());
  /** buffered context set entries */
  protected List<Set<BytesRef>> contextSets = new ArrayList<>();
  /** current buffer position */
  protected int curPos = -1;
  /** buffered weights, parallel with {@link #entries} */
  protected long[] freqs = new long[1];

  private final BytesRefBuilder spare = new BytesRefBuilder();
  private final BytesRefBuilder payloadSpare = new BytesRefBuilder();
  private final boolean hasPayloads;
  private final boolean hasContexts;

  /** Creates a new iterator, buffering entries from the specified iterator */
  public BufferedInputIterator(InputIterator source) throws IOException {
    BytesRef spare;
    int freqIndex = 0;
    hasPayloads = source.hasPayloads();
    hasContexts = source.hasContexts();
    while ((spare = source.next()) != null) {
      entries.append(spare);
      if (hasPayloads) {
        payloads.append(source.payload());
      }
      if (hasContexts) {
        contextSets.add(source.contexts());
      }
      if (freqIndex >= freqs.length) {
        freqs = ArrayUtil.grow(freqs, freqs.length + 1);
      }
      freqs[freqIndex++] = source.weight();
    }
  }

  @Override
  public long weight() {
    return freqs[curPos];
  }

  @Override
  public BytesRef next() throws IOException {
    if (++curPos < entries.size()) {
      entries.get(spare, curPos);
      return spare.get();
    }
    return null;
  }

  @Override
  public BytesRef payload() {
    if (hasPayloads && curPos < payloads.size()) {
      return payloads.get(payloadSpare, curPos);
    }
    return null;
  }

  @Override
  public boolean hasPayloads() {
    return hasPayloads;
  }

  @Override
  public Set<BytesRef> contexts() {
    if (hasContexts && curPos < contextSets.size()) {
      return contextSets.get(curPos);
    }
    return null;
  }

  @Override
  public boolean hasContexts() {
    return hasContexts;
  }
}