/* (non-Javadoc)
* @see com.linkedin.zoie.impl.indexing.internal.BatchedIndexDataLoader#consume(java.util.Collection)
*/
@Override
public void consume(Collection<DataEvent<D>> events) throws ZoieException {
if (events != null) {
ArrayList<DataEvent<ZoieIndexable>> indexableList =
new ArrayList<DataEvent<ZoieIndexable>>(events.size());
Iterator<DataEvent<D>> iter = events.iterator();
while (iter.hasNext()) {
try {
DataEvent<D> event = iter.next();
ZoieIndexable indexable =
((ZoieIndexableInterpreter<D>) _interpreter).convertAndInterpret(event.getData());
DataEvent<ZoieIndexable> newEvent =
new DataEvent<ZoieIndexable>(indexable, event.getVersion());
indexableList.add(newEvent);
} catch (Exception e) {
ZoieHealth.setFatal();
log.error(e.getMessage(), e);
}
}
synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
{
int size = indexableList.size();
_ramConsumer.consume(indexableList); // consumer clear the list!
_currentBatchSize += size;
_eventCount += size;
while (_currentBatchSize > _maxBatchSize) {
// check if load manager thread is alive
if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
ZoieHealth.setFatal();
throw new ZoieException("fatal: indexing thread loader manager has stopped");
}
this.notifyAll(); // wake up load manager thread
try {
this.wait(60000); // 1 min
} catch (InterruptedException e) {
continue;
}
}
this.notifyAll();
}
}
}
/** @see proj.zoie.api.DataConsumer#consume(java.util.Collection) */
public void consume(Collection<DataEvent<V>> events) throws ZoieException {
if (events != null) {
// PriorityQueue<DataEvent<ZoieIndexable>> indexableList = new
// PriorityQueue<DataEvent<ZoieIndexable>>(data.size(), DataEvent.getComparator());
ArrayList<DataEvent<ZoieIndexable>> indexableList =
new ArrayList<DataEvent<ZoieIndexable>>(events.size());
Iterator<DataEvent<V>> iter = events.iterator();
while (iter.hasNext()) {
try {
DataEvent<V> event = iter.next();
ZoieIndexable indexable =
((ZoieIndexableInterpreter<V>) _interpreter).convertAndInterpret(event.getData());
DataEvent<ZoieIndexable> newEvent =
new DataEvent<ZoieIndexable>(event.getVersion(), indexable);
indexableList.add(newEvent);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
synchronized (this) // this blocks the batch disk loader thread while indexing to RAM
{
while (_batchList.size() > _maxBatchSize) {
// check if load manager thread is alive
if (_loadMgrThread == null || !_loadMgrThread.isAlive()) {
throw new ZoieException("load manager has stopped");
}
try {
this.wait(60000); // 1 min
} catch (InterruptedException e) {
continue;
}
}
_eventCount += indexableList.size();
_batchList.addAll(indexableList);
this.notifyAll();
}
}
}
