public synchronized <K> NavigableSet<K> createTreeSet(BTreeSetMaker m) {
checkNameNotExists(m.name);
m.serializer = fillNulls(m.serializer);
m.serializer =
catPut(
m.name + ".keySerializer",
m.serializer,
new BTreeKeySerializer.BasicKeySerializer(getDefaultSerializer()));
m.comparator = catPut(m.name + ".comparator", m.comparator, Utils.COMPARABLE_COMPARATOR);
if (m.pumpPresortBatchSize != -1) {
m.pumpSource =
Pump.sort(
m.pumpSource,
m.pumpPresortBatchSize,
Collections.reverseOrder(m.comparator),
getDefaultSerializer());
}
long counterRecid = !m.keepCounter ? 0L : engine.put(0L, Serializer.LONG);
long rootRecidRef;
if (m.pumpSource == null) {
rootRecidRef = BTreeMap.createRootRef(engine, m.serializer, null, m.comparator);
} else {
rootRecidRef =
Pump.buildTreeMap(
m.pumpSource,
engine,
Fun.noTransformExtractor(),
null,
m.nodeSize,
false,
counterRecid,
m.serializer,
null,
m.comparator);
}
NavigableSet<K> ret =
new BTreeMap<K, Object>(
engine,
catPut(m.name + ".rootRecidRef", rootRecidRef),
catPut(m.name + ".maxNodeSize", m.nodeSize),
false,
catPut(m.name + ".counterRecid", counterRecid),
m.serializer,
null,
m.comparator)
.keySet();
catalog.put(m.name + ".type", "TreeSet");
collections.put(m.name, new WeakReference<Object>(ret));
return ret;
}
public static void main(String[] args) throws IOException {
/** max number of elements to import */
final long max = (int) 1e6;
/** Open database in temporary directory */
File dbFile = File.createTempFile("mapdb", "temp");
DB db =
DBMaker.newFileDB(dbFile)
/** disabling Write Ahead Log makes import much faster */
.transactionDisable()
.make();
long time = System.currentTimeMillis();
/**
* Source of data which randomly generates strings. In real world this would return data from
* file.
*/
Iterator<String> source =
new Iterator<String>() {
long counter = 0;
@Override
public boolean hasNext() {
return counter < max;
}
@Override
public String next() {
counter++;
return randomString(10);
}
@Override
public void remove() {}
};
/**
* BTreeMap Data Pump requires data source to be presorted in reverse order (highest to lowest).
* There is method in Data Pump we can use to sort data. It uses temporarly files and can handle
* fairly large data sets.
*/
source =
Pump.sort(
source,
true,
100000,
Collections.reverseOrder(BTreeMap.COMPARABLE_COMPARATOR), // reverse order comparator
db.getDefaultSerializer());
/**
* Disk space used by serialized keys should be minimised. Keys are sorted, so only difference
* between consequential keys is stored. This method is called delta-packing and typically saves
* 60% of disk space.
*/
BTreeKeySerializer<String> keySerializer = BTreeKeySerializer.STRING;
/** Translates Map Key into Map Value. */
Fun.Function1<Integer, String> valueExtractor =
new Fun.Function1<Integer, String>() {
@Override
public Integer run(String s) {
return s.hashCode();
}
};
/** Create BTreeMap and fill it with data */
Map<String, Integer> map =
db.createTreeMap("map")
.pumpSource(source, valueExtractor)
// .pumpPresort(100000) // for presorting data we could also use this method
.keySerializer(keySerializer)
.make();
System.out.println(
"Finished; total time: "
+ (System.currentTimeMillis() - time) / 1000
+ "s; there are "
+ map.size()
+ " items in map");
db.close();
}
