Ejemplo n.º 1
0
  /* (non-Javadoc)
   * @see com.linkedin.zoie.impl.indexing.internal.BatchedIndexDataLoader#consume(java.util.Collection)
   */
  @Override
  public void consume(Collection<DataEvent<D>> events) throws ZoieException {
    if (events != null) {
      ArrayList<DataEvent<ZoieIndexable>> indexableList =
          new ArrayList<DataEvent<ZoieIndexable>>(events.size());
      Iterator<DataEvent<D>> iter = events.iterator();
      while (iter.hasNext()) {
        try {
          DataEvent<D> event = iter.next();
          ZoieIndexable indexable =
              ((ZoieIndexableInterpreter<D>) _interpreter).convertAndInterpret(event.getData());

          DataEvent<ZoieIndexable> newEvent =
              new DataEvent<ZoieIndexable>(indexable, event.getVersion());
          indexableList.add(newEvent);
        } catch (Exception e) {
          ZoieHealth.setFatal();
          log.error(e.getMessage(), e);
        }
      }

      synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
      {
        int size = indexableList.size();
        _ramConsumer.consume(indexableList); // consumer clear the list!
        _currentBatchSize += size;
        _eventCount += size;

        while (_currentBatchSize > _maxBatchSize) {
          // check if load manager thread is alive
          if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
            ZoieHealth.setFatal();
            throw new ZoieException("fatal: indexing thread loader manager has stopped");
          }

          this.notifyAll(); // wake up load manager thread

          try {
            this.wait(60000); // 1 min
          } catch (InterruptedException e) {
            continue;
          }
        }
        this.notifyAll();
      }
    }
  }
Ejemplo n.º 2
0
  @Override
  protected synchronized void processBatch() {
    RAMSearchIndex<R> readOnlyMemIndex = null;
    long now = System.currentTimeMillis();
    long duration = now - _lastFlushTime;
    int eventCount = 0;
    while (_currentBatchSize < _batchSize && !_stop && !_flush && duration < _delay) {
      try {
        wait(_delay - duration);
      } catch (InterruptedException e) {
        log.warn(e.getMessage());
      }
      now = System.currentTimeMillis();
      duration = now - _lastFlushTime;
    }
    _flush = false;
    _lastFlushTime = now;

    if (_currentBatchSize > 0) {
      // change the status and get the read only memory index
      // this has to be done in the block synchronized on CopyingBatchIndexDataLoader
      _idxMgr.setDiskIndexerStatus(SearchIndexManager.Status.Working);
      readOnlyMemIndex = _idxMgr.getCurrentReadOnlyMemoryIndex();
      eventCount = _currentBatchSize;
      _currentBatchSize = 0;
    }

    if (eventCount > 0) {
      long t1 = System.currentTimeMillis();
      try {
        if (readOnlyMemIndex != null) {
          _luceneDataLoader.loadFromIndex(readOnlyMemIndex);
        }
      } catch (ZoieException e) {
        ZoieHealth.setFatal();
        log.error(e.getMessage(), e);
      } finally {
        long t2 = System.currentTimeMillis();
        _eventCount -= eventCount;
        int segmentCount = -1;
        String segmentInfo = "";
        try {
          segmentCount = _idxMgr.getDiskSegmentCount();
          segmentInfo = _idxMgr.getDiskSegmentInfo();

          IndexUpdatedEvent evt = new IndexUpdatedEvent(eventCount, t1, t2, _eventCount);
          fireIndexingEvent(evt);
          fireNewVersionEvent(readOnlyMemIndex.getVersion());
        } catch (IOException e) {
          log.error("error getting disk information after disk flush", e);
        }
        if (log.isInfoEnabled()) {
          log.info(
              "flushed batch of "
                  + eventCount
                  + " events to disk indexer, took: "
                  + (t2 - t1)
                  + " current event count: "
                  + _eventCount
                  + ", current disk segment count: "
                  + segmentCount);
          log.info("post-flush segment info: " + segmentInfo);
        }
        notifyAll();
      }
    } else {
      if (log.isDebugEnabled()) {
        log.debug("batch size is 0");
      }
    }
  }