public void consume(Collection<proj.zoie.api.DataConsumer.DataEvent<V>> events)
throws ZoieException {
if (_writer == null) {
throw new ZoieException("Internal IndexWriter null, perhaps not started?");
}
if (events.size() > 0) {
for (DataEvent<V> event : events) {
ZoieIndexable indexable = _interpreter.convertAndInterpret(event.getData());
if (indexable.isSkip()) continue;
try {
_writer.deleteDocuments(new Term(DOCUMENT_ID_FIELD, String.valueOf(indexable.getUID())));
} catch (IOException e) {
throw new ZoieException(e.getMessage(), e);
}
IndexingReq[] reqs = indexable.buildIndexingReqs();
for (IndexingReq req : reqs) {
Analyzer localAnalyzer = req.getAnalyzer();
Document doc = req.getDocument();
Field uidField =
new Field(
DOCUMENT_ID_FIELD,
String.valueOf(indexable.getUID()),
Store.NO,
Index.NOT_ANALYZED_NO_NORMS);
uidField.setOmitTermFreqAndPositions(true);
doc.add(uidField);
if (localAnalyzer == null) localAnalyzer = _analyzer;
try {
_writer.addDocument(doc, localAnalyzer);
} catch (IOException e) {
throw new ZoieException(e.getMessage(), e);
}
}
}
int numdocs;
try {
// for realtime commit is not needed per lucene mailing list
// _writer.commit();
numdocs = _writer.numDocs();
} catch (IOException e) {
throw new ZoieException(e.getMessage(), e);
}
logger.info(
"flushed "
+ events.size()
+ " events to index, index now contains "
+ numdocs
+ " docs.");
}
}
/* (non-Javadoc)
* @see com.linkedin.zoie.impl.indexing.internal.BatchedIndexDataLoader#consume(java.util.Collection)
*/
@Override
public void consume(Collection<DataEvent<D>> events) throws ZoieException {
if (events != null) {
ArrayList<DataEvent<ZoieIndexable>> indexableList =
new ArrayList<DataEvent<ZoieIndexable>>(events.size());
Iterator<DataEvent<D>> iter = events.iterator();
while (iter.hasNext()) {
try {
DataEvent<D> event = iter.next();
ZoieIndexable indexable =
((ZoieIndexableInterpreter<D>) _interpreter).convertAndInterpret(event.getData());
DataEvent<ZoieIndexable> newEvent =
new DataEvent<ZoieIndexable>(indexable, event.getVersion());
indexableList.add(newEvent);
} catch (Exception e) {
ZoieHealth.setFatal();
log.error(e.getMessage(), e);
}
}
synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
{
int size = indexableList.size();
_ramConsumer.consume(indexableList); // consumer clear the list!
_currentBatchSize += size;
_eventCount += size;
while (_currentBatchSize > _maxBatchSize) {
// check if load manager thread is alive
if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
ZoieHealth.setFatal();
throw new ZoieException("fatal: indexing thread loader manager has stopped");
}
this.notifyAll(); // wake up load manager thread
try {
this.wait(60000); // 1 min
} catch (InterruptedException e) {
continue;
}
}
this.notifyAll();
}
}
}
/** @see proj.zoie.api.DataConsumer#consume(java.util.Collection) */
public void consume(Collection<DataEvent<V>> events) throws ZoieException {
if (events != null) {
// PriorityQueue<DataEvent<ZoieIndexable>> indexableList = new
// PriorityQueue<DataEvent<ZoieIndexable>>(data.size(), DataEvent.getComparator());
ArrayList<DataEvent<ZoieIndexable>> indexableList =
new ArrayList<DataEvent<ZoieIndexable>>(events.size());
Iterator<DataEvent<V>> iter = events.iterator();
while (iter.hasNext()) {
try {
DataEvent<V> event = iter.next();
ZoieIndexable indexable =
((ZoieIndexableInterpreter<V>) _interpreter).convertAndInterpret(event.getData());
DataEvent<ZoieIndexable> newEvent =
new DataEvent<ZoieIndexable>(event.getVersion(), indexable);
indexableList.add(newEvent);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
{
while (_batchList.size() > _maxBatchSize) {
// check if load manager thread is alive
if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
throw new ZoieException("load manager has stopped");
}
try {
this.wait(60000); // 1 min
} catch (InterruptedException e) {
continue;
}
}
_eventCount += indexableList.size();
_batchList.addAll(indexableList);
this.notifyAll();
}
}
}
