/**
* Returns an iterator over a set of elements of type E. If you do not exhaust the iterator, you
* should call {@link org.openjena.atlas.iterator.Iter#close(Iterator)} to be sure any open file
* handles are closed.
*
* @return an Iterator
*/
@Override
@SuppressWarnings({"unchecked", "rawtypes"})
public Iterator<E> iterator() {
checkClosed();
int memSize = memory.size();
// Constructing an iterator from this class is not thread-safe (just like all the the other
// methods)
if (!finishedAdding && memSize > 1) {
// Again, some ugliness for speed
Object[] array = memory.toArray();
Arrays.sort(array, (Comparator) comparator);
memory = Arrays.asList((E[]) array);
}
finishedAdding = true;
if (spilled) {
List<Iterator<E>> inputs = new ArrayList<Iterator<E>>();
if (memSize > 0) {
inputs.add(memory.iterator());
}
for (File spillFile : getSpillFiles()) {
try {
Iterator<E> irc = getInputIterator(spillFile);
inputs.add(irc);
} catch (FileNotFoundException e) {
// Close any open streams before we throw an exception
for (Iterator<E> it : inputs) {
Iter.close(it);
}
throw new AtlasException("Cannot find one of the spill files", e);
}
}
SpillSortIterator<E> ssi = new SpillSortIterator<E>(inputs, comparator);
registerCloseableIterator(ssi);
return ssi;
} else {
if (memSize > 0) {
return memory.iterator();
} else {
return Iter.nullIterator();
}
}
}
private void buildNodeTableBPTreeIndex() {
try {
// Node table B+Tree index (i.e. node2id.dat/idn)
log.info(
"Node Table (3/3): building node table B+Tree index (i.e. node2id.dat and node2id.idn files)...");
final ProgressLogger monitor03 =
new ProgressLogger(
log,
"records for node table (3/3) phase",
BulkLoader.DataTickPoint,
BulkLoader.superTick);
monitor03.start();
String path = dsg.getLocation().getDirectoryPath();
new File(path, "node2id.dat").delete();
new File(path, "node2id.idn").delete();
final RecordFactory recordFactory = new RecordFactory(LenNodeHash, SizeOfNodeId);
Transform<Pair<byte[], byte[]>, Record> transformPair2Record =
new Transform<Pair<byte[], byte[]>, Record>() {
@Override
public Record convert(Pair<byte[], byte[]> pair) {
monitor03.tick();
return recordFactory.create(pair.getLeft(), pair.getRight());
}
};
int order = BPlusTreeParams.calcOrder(SystemTDB.BlockSize, recordFactory);
BPlusTreeParams bptParams = new BPlusTreeParams(order, recordFactory);
int readCacheSize = 10;
int writeCacheSize = 100;
FileSet destination = new FileSet(dsg.getLocation(), Names.indexNode2Id);
BlockMgr blkMgrNodes =
BlockMgrFactory.create(
destination, Names.bptExtTree, SystemTDB.BlockSize, readCacheSize, writeCacheSize);
BlockMgr blkMgrRecords =
BlockMgrFactory.create(
destination, Names.bptExtRecords, SystemTDB.BlockSize, readCacheSize, writeCacheSize);
Iterator<Record> iter2 = Iter.iter(sdb03.iterator()).map(transformPair2Record);
BPlusTree bpt2 =
BPlusTreeRewriter.packIntoBPlusTree(
iter2, bptParams, recordFactory, blkMgrNodes, blkMgrRecords);
bpt2.sync();
ProgressLogger.print(log, monitor03);
} finally {
sdb03.close();
sdb03 = null;
}
}
@Override
public void close() {
for (Iterator<T> it : inputs) {
Iter.close(it);
}
}