Esempio n. 1
0
  /* (non-Javadoc)
   * @see com.linkedin.zoie.impl.indexing.internal.BatchedIndexDataLoader#consume(java.util.Collection)
   */
  @Override
  public void consume(Collection<DataEvent<D>> events) throws ZoieException {
    if (events != null) {
      ArrayList<DataEvent<ZoieIndexable>> indexableList =
          new ArrayList<DataEvent<ZoieIndexable>>(events.size());
      Iterator<DataEvent<D>> iter = events.iterator();
      while (iter.hasNext()) {
        try {
          DataEvent<D> event = iter.next();
          ZoieIndexable indexable =
              ((ZoieIndexableInterpreter<D>) _interpreter).convertAndInterpret(event.getData());

          DataEvent<ZoieIndexable> newEvent =
              new DataEvent<ZoieIndexable>(indexable, event.getVersion());
          indexableList.add(newEvent);
        } catch (Exception e) {
          ZoieHealth.setFatal();
          log.error(e.getMessage(), e);
        }
      }

      synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
      {
        int size = indexableList.size();
        _ramConsumer.consume(indexableList); // consumer clear the list!
        _currentBatchSize += size;
        _eventCount += size;

        while (_currentBatchSize > _maxBatchSize) {
          // check if load manager thread is alive
          if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
            ZoieHealth.setFatal();
            throw new ZoieException("fatal: indexing thread loader manager has stopped");
          }

          this.notifyAll(); // wake up load manager thread

          try {
            this.wait(60000); // 1 min
          } catch (InterruptedException e) {
            continue;
          }
        }
        this.notifyAll();
      }
    }
  }
  /** @see proj.zoie.api.DataConsumer#consume(java.util.Collection) */
  public void consume(Collection<DataEvent<V>> events) throws ZoieException {
    if (events != null) {
      // PriorityQueue<DataEvent<ZoieIndexable>> indexableList = new
      // PriorityQueue<DataEvent<ZoieIndexable>>(data.size(), DataEvent.getComparator());
      ArrayList<DataEvent<ZoieIndexable>> indexableList =
          new ArrayList<DataEvent<ZoieIndexable>>(events.size());
      Iterator<DataEvent<V>> iter = events.iterator();
      while (iter.hasNext()) {
        try {
          DataEvent<V> event = iter.next();
          ZoieIndexable indexable =
              ((ZoieIndexableInterpreter<V>) _interpreter).convertAndInterpret(event.getData());

          DataEvent<ZoieIndexable> newEvent =
              new DataEvent<ZoieIndexable>(event.getVersion(), indexable);
          indexableList.add(newEvent);
        } catch (Exception e) {
          log.error(e.getMessage(), e);
        }
      }

      synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
      {
        while (_batchList.size() > _maxBatchSize) {
          // check if load manager thread is alive
          if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
            throw new ZoieException("load manager has stopped");
          }

          try {
            this.wait(60000); // 1 min
          } catch (InterruptedException e) {
            continue;
          }
        }
        _eventCount += indexableList.size();
        _batchList.addAll(indexableList);
        this.notifyAll();
      }
    }
  }