public void consume(Collection<proj.zoie.api.DataConsumer.DataEvent<V>> events) throws ZoieException { if (_writer == null) { throw new ZoieException("Internal IndexWriter null, perhaps not started?"); } if (events.size() > 0) { for (DataEvent<V> event : events) { ZoieIndexable indexable = _interpreter.convertAndInterpret(event.getData()); if (indexable.isSkip()) continue; try { _writer.deleteDocuments(new Term(DOCUMENT_ID_FIELD, String.valueOf(indexable.getUID()))); } catch (IOException e) { throw new ZoieException(e.getMessage(), e); } IndexingReq[] reqs = indexable.buildIndexingReqs(); for (IndexingReq req : reqs) { Analyzer localAnalyzer = req.getAnalyzer(); Document doc = req.getDocument(); Field uidField = new Field( DOCUMENT_ID_FIELD, String.valueOf(indexable.getUID()), Store.NO, Index.NOT_ANALYZED_NO_NORMS); uidField.setOmitTermFreqAndPositions(true); doc.add(uidField); if (localAnalyzer == null) localAnalyzer = _analyzer; try { _writer.addDocument(doc, localAnalyzer); } catch (IOException e) { throw new ZoieException(e.getMessage(), e); } } } int numdocs; try { // for realtime commit is not needed per lucene mailing list // _writer.commit(); numdocs = _writer.numDocs(); } catch (IOException e) { throw new ZoieException(e.getMessage(), e); } logger.info( "flushed " + events.size() + " events to index, index now contains " + numdocs + " docs."); } }
/* (non-Javadoc) * @see com.linkedin.zoie.impl.indexing.internal.BatchedIndexDataLoader#consume(java.util.Collection) */ @Override public void consume(Collection<DataEvent<D>> events) throws ZoieException { if (events != null) { ArrayList<DataEvent<ZoieIndexable>> indexableList = new ArrayList<DataEvent<ZoieIndexable>>(events.size()); Iterator<DataEvent<D>> iter = events.iterator(); while (iter.hasNext()) { try { DataEvent<D> event = iter.next(); ZoieIndexable indexable = ((ZoieIndexableInterpreter<D>) _interpreter).convertAndInterpret(event.getData()); DataEvent<ZoieIndexable> newEvent = new DataEvent<ZoieIndexable>(indexable, event.getVersion()); indexableList.add(newEvent); } catch (Exception e) { ZoieHealth.setFatal(); log.error(e.getMessage(), e); } } synchronized (this) // this blocks the batch disk loader thread while indexing to RAM { int size = indexableList.size(); _ramConsumer.consume(indexableList); // consumer clear the list! _currentBatchSize += size; _eventCount += size; while (_currentBatchSize > _maxBatchSize) { // check if load manager thread is alive if (_loadMgrThread == null || !_loadMgrThread.isAlive()) { ZoieHealth.setFatal(); throw new ZoieException("fatal: indexing thread loader manager has stopped"); } this.notifyAll(); // wake up load manager thread try { this.wait(60000); // 1 min } catch (InterruptedException e) { continue; } } this.notifyAll(); } } }
@Override public void onDataChanged(DataEventBuffer dataEvents) { DataMap dataMap; for (DataEvent event : dataEvents) { // Check the data type if (event.getType() == DataEvent.TYPE_CHANGED) { // Check the data path dataMap = DataMapItem.fromDataItem(event.getDataItem()).getDataMap(); String path = event.getDataItem().getUri().getPath(); if (path.equals(WEARABLE_TO_PHONE_DATA_PATH) && dataMap.containsKey(ACTION_KEY)) { doAction(dataMap.getString(ACTION_KEY), dataMap); } } } }
/** * Inserts a {@link DataElement} at the specified index. After the element has been successfully * inserted all {@link DataListener DataListeners} will be informed. * * @param index Index at which <tt>element</tt> will be inserted. All elements with an index >= * <tt>index</tt> will be shifted. * @param element DataElement to be added. * @throws IllegalArgumentException if <tt>element</tt> is not of the correct type which will be * checked by the method {@link #isValid}. */ public void insertElementAt(int index, DataElement element) { if (isValid(element)) { _container.insertElementAt(element, index); element.setContainer(this); notifyListeners(DataEvent.createInsertEvent(this, index)); } else { throwException(INSERT, element); } }
/** * Adds a {@link DataElement}. After the element has been successfully added all {@link * DataListener DataListeners} will be informed. * * @param element DataElement to be added. * @throws IllegalArgumentException if <tt>element</tt> is not of the correct type which will be * checked by the method {@link #isValid}. */ public void addElement(DataElement element) { if (isValid(element)) { _container.addElement(element); element.setContainer(this); notifyListeners(DataEvent.createAddEvent(this)); } else { throwException(ADD, element); } }
/** * Replaces the {@link DataElement} at the specified index. After the element has been * successfully replaced all {@link DataListener DataListeners} will be informed. * * @param index Index of the element which will be replaced by <tt>element</tt>. * @param element The new <tt>DataElement</tt>. * @throws IllegalArgumentException if <tt>element</tt> is not of the correct type which will be * checked by the method {@link #isValid}. */ public void replaceElementAt(int index, DataElement element) { if (isValid(element)) { DataElement oldElement = (DataElement) _container.elementAt(index); oldElement.setContainer(null); _container.setElementAt(element, index); element.setContainer(this); notifyListeners(DataEvent.createReplaceEvent(this, index, oldElement)); } else { throwException(REPLACE, element); } }
/** @see proj.zoie.api.DataConsumer#consume(java.util.Collection) */ public void consume(Collection<DataEvent<V>> events) throws ZoieException { if (events != null) { // PriorityQueue<DataEvent<ZoieIndexable>> indexableList = new // PriorityQueue<DataEvent<ZoieIndexable>>(data.size(), DataEvent.getComparator()); ArrayList<DataEvent<ZoieIndexable>> indexableList = new ArrayList<DataEvent<ZoieIndexable>>(events.size()); Iterator<DataEvent<V>> iter = events.iterator(); while (iter.hasNext()) { try { DataEvent<V> event = iter.next(); ZoieIndexable indexable = ((ZoieIndexableInterpreter<V>) _interpreter).convertAndInterpret(event.getData()); DataEvent<ZoieIndexable> newEvent = new DataEvent<ZoieIndexable>(event.getVersion(), indexable); indexableList.add(newEvent); } catch (Exception e) { log.error(e.getMessage(), e); } } synchronized (this) // this blocks the batch disk loader thread while indexing to RAM { while (_batchList.size() > _maxBatchSize) { // check if load manager thread is alive if (_loadMgrThread == null || !_loadMgrThread.isAlive()) { throw new ZoieException("load manager has stopped"); } try { this.wait(60000); // 1 min } catch (InterruptedException e) { continue; } } _eventCount += indexableList.size(); _batchList.addAll(indexableList); this.notifyAll(); } } }
/** * Removes a {@link DataElement} at the specified index. After the element has been successfully * removed all {@link DataListener DataListeners} will be informed. * * @param index Index of the element which will be removed. All elements with an index > * <tt>index</tt> will be shifted. */ public void removeElementAt(int index) { DataElement element = (DataElement) _container.elementAt(index); element.setContainer(null); _container.removeElementAt(index); notifyListeners(DataEvent.createRemoveEvent(this, index, element)); }