public void consume(Collection<proj.zoie.api.DataConsumer.DataEvent<V>> events)
      throws ZoieException {
    if (_writer == null) {
      throw new ZoieException("Internal IndexWriter null, perhaps not started?");
    }

    if (events.size() > 0) {
      for (DataEvent<V> event : events) {
        ZoieIndexable indexable = _interpreter.convertAndInterpret(event.getData());
        if (indexable.isSkip()) continue;

        try {
          _writer.deleteDocuments(new Term(DOCUMENT_ID_FIELD, String.valueOf(indexable.getUID())));
        } catch (IOException e) {
          throw new ZoieException(e.getMessage(), e);
        }

        IndexingReq[] reqs = indexable.buildIndexingReqs();
        for (IndexingReq req : reqs) {
          Analyzer localAnalyzer = req.getAnalyzer();
          Document doc = req.getDocument();
          Field uidField =
              new Field(
                  DOCUMENT_ID_FIELD,
                  String.valueOf(indexable.getUID()),
                  Store.NO,
                  Index.NOT_ANALYZED_NO_NORMS);
          uidField.setOmitTermFreqAndPositions(true);
          doc.add(uidField);
          if (localAnalyzer == null) localAnalyzer = _analyzer;
          try {
            _writer.addDocument(doc, localAnalyzer);
          } catch (IOException e) {
            throw new ZoieException(e.getMessage(), e);
          }
        }
      }

      int numdocs;
      try {
        // for realtime commit is not needed per lucene mailing list
        // _writer.commit();
        numdocs = _writer.numDocs();
      } catch (IOException e) {
        throw new ZoieException(e.getMessage(), e);
      }

      logger.info(
          "flushed "
              + events.size()
              + " events to index, index now contains "
              + numdocs
              + " docs.");
    }
  }
예제 #2
0
  /* (non-Javadoc)
   * @see com.linkedin.zoie.impl.indexing.internal.BatchedIndexDataLoader#consume(java.util.Collection)
   */
  @Override
  public void consume(Collection<DataEvent<D>> events) throws ZoieException {
    if (events != null) {
      ArrayList<DataEvent<ZoieIndexable>> indexableList =
          new ArrayList<DataEvent<ZoieIndexable>>(events.size());
      Iterator<DataEvent<D>> iter = events.iterator();
      while (iter.hasNext()) {
        try {
          DataEvent<D> event = iter.next();
          ZoieIndexable indexable =
              ((ZoieIndexableInterpreter<D>) _interpreter).convertAndInterpret(event.getData());

          DataEvent<ZoieIndexable> newEvent =
              new DataEvent<ZoieIndexable>(indexable, event.getVersion());
          indexableList.add(newEvent);
        } catch (Exception e) {
          ZoieHealth.setFatal();
          log.error(e.getMessage(), e);
        }
      }

      synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
      {
        int size = indexableList.size();
        _ramConsumer.consume(indexableList); // consumer clear the list!
        _currentBatchSize += size;
        _eventCount += size;

        while (_currentBatchSize > _maxBatchSize) {
          // check if load manager thread is alive
          if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
            ZoieHealth.setFatal();
            throw new ZoieException("fatal: indexing thread loader manager has stopped");
          }

          this.notifyAll(); // wake up load manager thread

          try {
            this.wait(60000); // 1 min
          } catch (InterruptedException e) {
            continue;
          }
        }
        this.notifyAll();
      }
    }
  }
  /** @see proj.zoie.api.DataConsumer#consume(java.util.Collection) */
  public void consume(Collection<DataEvent<V>> events) throws ZoieException {
    if (events != null) {
      // PriorityQueue<DataEvent<ZoieIndexable>> indexableList = new
      // PriorityQueue<DataEvent<ZoieIndexable>>(data.size(), DataEvent.getComparator());
      ArrayList<DataEvent<ZoieIndexable>> indexableList =
          new ArrayList<DataEvent<ZoieIndexable>>(events.size());
      Iterator<DataEvent<V>> iter = events.iterator();
      while (iter.hasNext()) {
        try {
          DataEvent<V> event = iter.next();
          ZoieIndexable indexable =
              ((ZoieIndexableInterpreter<V>) _interpreter).convertAndInterpret(event.getData());

          DataEvent<ZoieIndexable> newEvent =
              new DataEvent<ZoieIndexable>(event.getVersion(), indexable);
          indexableList.add(newEvent);
        } catch (Exception e) {
          log.error(e.getMessage(), e);
        }
      }

      synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
      {
        while (_batchList.size() > _maxBatchSize) {
          // check if load manager thread is alive
          if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
            throw new ZoieException("load manager has stopped");
          }

          try {
            this.wait(60000); // 1 min
          } catch (InterruptedException e) {
            continue;
          }
        }
        _eventCount += indexableList.size();
        _batchList.addAll(indexableList);
        this.notifyAll();
      }
    }
  }