public void consume(Collection<proj.zoie.api.DataConsumer.DataEvent<V>> events)
      throws ZoieException {
    if (_writer == null) {
      throw new ZoieException("Internal IndexWriter null, perhaps not started?");
    }

    if (events.size() > 0) {
      for (DataEvent<V> event : events) {
        ZoieIndexable indexable = _interpreter.convertAndInterpret(event.getData());
        if (indexable.isSkip()) continue;

        try {
          _writer.deleteDocuments(new Term(DOCUMENT_ID_FIELD, String.valueOf(indexable.getUID())));
        } catch (IOException e) {
          throw new ZoieException(e.getMessage(), e);
        }

        IndexingReq[] reqs = indexable.buildIndexingReqs();
        for (IndexingReq req : reqs) {
          Analyzer localAnalyzer = req.getAnalyzer();
          Document doc = req.getDocument();
          Field uidField =
              new Field(
                  DOCUMENT_ID_FIELD,
                  String.valueOf(indexable.getUID()),
                  Store.NO,
                  Index.NOT_ANALYZED_NO_NORMS);
          uidField.setOmitTermFreqAndPositions(true);
          doc.add(uidField);
          if (localAnalyzer == null) localAnalyzer = _analyzer;
          try {
            _writer.addDocument(doc, localAnalyzer);
          } catch (IOException e) {
            throw new ZoieException(e.getMessage(), e);
          }
        }
      }

      int numdocs;
      try {
        // for realtime commit is not needed per lucene mailing list
        // _writer.commit();
        numdocs = _writer.numDocs();
      } catch (IOException e) {
        throw new ZoieException(e.getMessage(), e);
      }

      logger.info(
          "flushed "
              + events.size()
              + " events to index, index now contains "
              + numdocs
              + " docs.");
    }
  }
Ejemplo n.º 2
0
  /* (non-Javadoc)
   * @see com.linkedin.zoie.impl.indexing.internal.BatchedIndexDataLoader#consume(java.util.Collection)
   */
  @Override
  public void consume(Collection<DataEvent<D>> events) throws ZoieException {
    if (events != null) {
      ArrayList<DataEvent<ZoieIndexable>> indexableList =
          new ArrayList<DataEvent<ZoieIndexable>>(events.size());
      Iterator<DataEvent<D>> iter = events.iterator();
      while (iter.hasNext()) {
        try {
          DataEvent<D> event = iter.next();
          ZoieIndexable indexable =
              ((ZoieIndexableInterpreter<D>) _interpreter).convertAndInterpret(event.getData());

          DataEvent<ZoieIndexable> newEvent =
              new DataEvent<ZoieIndexable>(indexable, event.getVersion());
          indexableList.add(newEvent);
        } catch (Exception e) {
          ZoieHealth.setFatal();
          log.error(e.getMessage(), e);
        }
      }

      synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
      {
        int size = indexableList.size();
        _ramConsumer.consume(indexableList); // consumer clear the list!
        _currentBatchSize += size;
        _eventCount += size;

        while (_currentBatchSize > _maxBatchSize) {
          // check if load manager thread is alive
          if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
            ZoieHealth.setFatal();
            throw new ZoieException("fatal: indexing thread loader manager has stopped");
          }

          this.notifyAll(); // wake up load manager thread

          try {
            this.wait(60000); // 1 min
          } catch (InterruptedException e) {
            continue;
          }
        }
        this.notifyAll();
      }
    }
  }
Ejemplo n.º 3
0
  @Override
  public void onDataChanged(DataEventBuffer dataEvents) {
    DataMap dataMap;
    for (DataEvent event : dataEvents) {

      // Check the data type
      if (event.getType() == DataEvent.TYPE_CHANGED) {
        // Check the data path
        dataMap = DataMapItem.fromDataItem(event.getDataItem()).getDataMap();
        String path = event.getDataItem().getUri().getPath();

        if (path.equals(WEARABLE_TO_PHONE_DATA_PATH) && dataMap.containsKey(ACTION_KEY)) {
          doAction(dataMap.getString(ACTION_KEY), dataMap);
        }
      }
    }
  }
Ejemplo n.º 4
0
 /**
  * Inserts a {@link DataElement} at the specified index. After the element has been successfully
  * inserted all {@link DataListener DataListeners} will be informed.
  *
  * @param index Index at which <tt>element</tt> will be inserted. All elements with an index &gt;=
  *     <tt>index</tt> will be shifted.
  * @param element DataElement to be added.
  * @throws IllegalArgumentException if <tt>element</tt> is not of the correct type which will be
  *     checked by the method {@link #isValid}.
  */
 public void insertElementAt(int index, DataElement element) {
   if (isValid(element)) {
     _container.insertElementAt(element, index);
     element.setContainer(this);
     notifyListeners(DataEvent.createInsertEvent(this, index));
   } else {
     throwException(INSERT, element);
   }
 }
Ejemplo n.º 5
0
 /**
  * Adds a {@link DataElement}. After the element has been successfully added all {@link
  * DataListener DataListeners} will be informed.
  *
  * @param element DataElement to be added.
  * @throws IllegalArgumentException if <tt>element</tt> is not of the correct type which will be
  *     checked by the method {@link #isValid}.
  */
 public void addElement(DataElement element) {
   if (isValid(element)) {
     _container.addElement(element);
     element.setContainer(this);
     notifyListeners(DataEvent.createAddEvent(this));
   } else {
     throwException(ADD, element);
   }
 }
Ejemplo n.º 6
0
 /**
  * Replaces the {@link DataElement} at the specified index. After the element has been
  * successfully replaced all {@link DataListener DataListeners} will be informed.
  *
  * @param index Index of the element which will be replaced by <tt>element</tt>.
  * @param element The new <tt>DataElement</tt>.
  * @throws IllegalArgumentException if <tt>element</tt> is not of the correct type which will be
  *     checked by the method {@link #isValid}.
  */
 public void replaceElementAt(int index, DataElement element) {
   if (isValid(element)) {
     DataElement oldElement = (DataElement) _container.elementAt(index);
     oldElement.setContainer(null);
     _container.setElementAt(element, index);
     element.setContainer(this);
     notifyListeners(DataEvent.createReplaceEvent(this, index, oldElement));
   } else {
     throwException(REPLACE, element);
   }
 }
  /** @see proj.zoie.api.DataConsumer#consume(java.util.Collection) */
  public void consume(Collection<DataEvent<V>> events) throws ZoieException {
    if (events != null) {
      // PriorityQueue<DataEvent<ZoieIndexable>> indexableList = new
      // PriorityQueue<DataEvent<ZoieIndexable>>(data.size(), DataEvent.getComparator());
      ArrayList<DataEvent<ZoieIndexable>> indexableList =
          new ArrayList<DataEvent<ZoieIndexable>>(events.size());
      Iterator<DataEvent<V>> iter = events.iterator();
      while (iter.hasNext()) {
        try {
          DataEvent<V> event = iter.next();
          ZoieIndexable indexable =
              ((ZoieIndexableInterpreter<V>) _interpreter).convertAndInterpret(event.getData());

          DataEvent<ZoieIndexable> newEvent =
              new DataEvent<ZoieIndexable>(event.getVersion(), indexable);
          indexableList.add(newEvent);
        } catch (Exception e) {
          log.error(e.getMessage(), e);
        }
      }

      synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
      {
        while (_batchList.size() > _maxBatchSize) {
          // check if load manager thread is alive
          if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
            throw new ZoieException("load manager has stopped");
          }

          try {
            this.wait(60000); // 1 min
          } catch (InterruptedException e) {
            continue;
          }
        }
        _eventCount += indexableList.size();
        _batchList.addAll(indexableList);
        this.notifyAll();
      }
    }
  }
Ejemplo n.º 8
0
 /**
  * Removes a {@link DataElement} at the specified index. After the element has been successfully
  * removed all {@link DataListener DataListeners} will be informed.
  *
  * @param index Index of the element which will be removed. All elements with an index &gt;
  *     <tt>index</tt> will be shifted.
  */
 public void removeElementAt(int index) {
   DataElement element = (DataElement) _container.elementAt(index);
   element.setContainer(null);
   _container.removeElementAt(index);
   notifyListeners(DataEvent.createRemoveEvent(this, index, element));
 }