public void consume(Collection<proj.zoie.api.DataConsumer.DataEvent<V>> events)
throws ZoieException {
if (_writer == null) {
throw new ZoieException("Internal IndexWriter null, perhaps not started?");
}
if (events.size() > 0) {
for (DataEvent<V> event : events) {
ZoieIndexable indexable = _interpreter.convertAndInterpret(event.getData());
if (indexable.isSkip()) continue;
try {
_writer.deleteDocuments(new Term(DOCUMENT_ID_FIELD, String.valueOf(indexable.getUID())));
} catch (IOException e) {
throw new ZoieException(e.getMessage(), e);
}
IndexingReq[] reqs = indexable.buildIndexingReqs();
for (IndexingReq req : reqs) {
Analyzer localAnalyzer = req.getAnalyzer();
Document doc = req.getDocument();
Field uidField =
new Field(
DOCUMENT_ID_FIELD,
String.valueOf(indexable.getUID()),
Store.NO,
Index.NOT_ANALYZED_NO_NORMS);
uidField.setOmitTermFreqAndPositions(true);
doc.add(uidField);
if (localAnalyzer == null) localAnalyzer = _analyzer;
try {
_writer.addDocument(doc, localAnalyzer);
} catch (IOException e) {
throw new ZoieException(e.getMessage(), e);
}
}
}
int numdocs;
try {
// for realtime commit is not needed per lucene mailing list
// _writer.commit();
numdocs = _writer.numDocs();
} catch (IOException e) {
throw new ZoieException(e.getMessage(), e);
}
logger.info(
"flushed "
+ events.size()
+ " events to index, index now contains "
+ numdocs
+ " docs.");
}
}
/* (non-Javadoc)
* @see com.linkedin.zoie.impl.indexing.internal.BatchedIndexDataLoader#consume(java.util.Collection)
*/
@Override
public void consume(Collection<DataEvent<D>> events) throws ZoieException {
if (events != null) {
ArrayList<DataEvent<ZoieIndexable>> indexableList =
new ArrayList<DataEvent<ZoieIndexable>>(events.size());
Iterator<DataEvent<D>> iter = events.iterator();
while (iter.hasNext()) {
try {
DataEvent<D> event = iter.next();
ZoieIndexable indexable =
((ZoieIndexableInterpreter<D>) _interpreter).convertAndInterpret(event.getData());
DataEvent<ZoieIndexable> newEvent =
new DataEvent<ZoieIndexable>(indexable, event.getVersion());
indexableList.add(newEvent);
} catch (Exception e) {
ZoieHealth.setFatal();
log.error(e.getMessage(), e);
}
}
synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
{
int size = indexableList.size();
_ramConsumer.consume(indexableList); // consumer clear the list!
_currentBatchSize += size;
_eventCount += size;
while (_currentBatchSize > _maxBatchSize) {
// check if load manager thread is alive
if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
ZoieHealth.setFatal();
throw new ZoieException("fatal: indexing thread loader manager has stopped");
}
this.notifyAll(); // wake up load manager thread
try {
this.wait(60000); // 1 min
} catch (InterruptedException e) {
continue;
}
}
this.notifyAll();
}
}
}
@Override
public void onDataChanged(DataEventBuffer dataEvents) {
DataMap dataMap;
for (DataEvent event : dataEvents) {
// Check the data type
if (event.getType() == DataEvent.TYPE_CHANGED) {
// Check the data path
dataMap = DataMapItem.fromDataItem(event.getDataItem()).getDataMap();
String path = event.getDataItem().getUri().getPath();
if (path.equals(WEARABLE_TO_PHONE_DATA_PATH) && dataMap.containsKey(ACTION_KEY)) {
doAction(dataMap.getString(ACTION_KEY), dataMap);
}
}
}
}
/**
* Inserts a {@link DataElement} at the specified index. After the element has been successfully
* inserted all {@link DataListener DataListeners} will be informed.
*
* @param index Index at which <tt>element</tt> will be inserted. All elements with an index >=
* <tt>index</tt> will be shifted.
* @param element DataElement to be added.
* @throws IllegalArgumentException if <tt>element</tt> is not of the correct type which will be
* checked by the method {@link #isValid}.
*/
public void insertElementAt(int index, DataElement element) {
if (isValid(element)) {
_container.insertElementAt(element, index);
element.setContainer(this);
notifyListeners(DataEvent.createInsertEvent(this, index));
} else {
throwException(INSERT, element);
}
}
/**
* Adds a {@link DataElement}. After the element has been successfully added all {@link
* DataListener DataListeners} will be informed.
*
* @param element DataElement to be added.
* @throws IllegalArgumentException if <tt>element</tt> is not of the correct type which will be
* checked by the method {@link #isValid}.
*/
public void addElement(DataElement element) {
if (isValid(element)) {
_container.addElement(element);
element.setContainer(this);
notifyListeners(DataEvent.createAddEvent(this));
} else {
throwException(ADD, element);
}
}
/**
* Replaces the {@link DataElement} at the specified index. After the element has been
* successfully replaced all {@link DataListener DataListeners} will be informed.
*
* @param index Index of the element which will be replaced by <tt>element</tt>.
* @param element The new <tt>DataElement</tt>.
* @throws IllegalArgumentException if <tt>element</tt> is not of the correct type which will be
* checked by the method {@link #isValid}.
*/
public void replaceElementAt(int index, DataElement element) {
if (isValid(element)) {
DataElement oldElement = (DataElement) _container.elementAt(index);
oldElement.setContainer(null);
_container.setElementAt(element, index);
element.setContainer(this);
notifyListeners(DataEvent.createReplaceEvent(this, index, oldElement));
} else {
throwException(REPLACE, element);
}
}
/** @see proj.zoie.api.DataConsumer#consume(java.util.Collection) */
public void consume(Collection<DataEvent<V>> events) throws ZoieException {
if (events != null) {
// PriorityQueue<DataEvent<ZoieIndexable>> indexableList = new
// PriorityQueue<DataEvent<ZoieIndexable>>(data.size(), DataEvent.getComparator());
ArrayList<DataEvent<ZoieIndexable>> indexableList =
new ArrayList<DataEvent<ZoieIndexable>>(events.size());
Iterator<DataEvent<V>> iter = events.iterator();
while (iter.hasNext()) {
try {
DataEvent<V> event = iter.next();
ZoieIndexable indexable =
((ZoieIndexableInterpreter<V>) _interpreter).convertAndInterpret(event.getData());
DataEvent<ZoieIndexable> newEvent =
new DataEvent<ZoieIndexable>(event.getVersion(), indexable);
indexableList.add(newEvent);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
{
while (_batchList.size() > _maxBatchSize) {
// check if load manager thread is alive
if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
throw new ZoieException("load manager has stopped");
}
try {
this.wait(60000); // 1 min
} catch (InterruptedException e) {
continue;
}
}
_eventCount += indexableList.size();
_batchList.addAll(indexableList);
this.notifyAll();
}
}
}
/**
* Removes a {@link DataElement} at the specified index. After the element has been successfully
* removed all {@link DataListener DataListeners} will be informed.
*
* @param index Index of the element which will be removed. All elements with an index >
* <tt>index</tt> will be shifted.
*/
public void removeElementAt(int index) {
DataElement element = (DataElement) _container.elementAt(index);
element.setContainer(null);
_container.removeElementAt(index);
notifyListeners(DataEvent.createRemoveEvent(this, index, element));
}