public synchronized <K> NavigableSet<K> createTreeSet(BTreeSetMaker m) { checkNameNotExists(m.name); m.serializer = fillNulls(m.serializer); m.serializer = catPut( m.name + ".keySerializer", m.serializer, new BTreeKeySerializer.BasicKeySerializer(getDefaultSerializer())); m.comparator = catPut(m.name + ".comparator", m.comparator, Utils.COMPARABLE_COMPARATOR); if (m.pumpPresortBatchSize != -1) { m.pumpSource = Pump.sort( m.pumpSource, m.pumpPresortBatchSize, Collections.reverseOrder(m.comparator), getDefaultSerializer()); } long counterRecid = !m.keepCounter ? 0L : engine.put(0L, Serializer.LONG); long rootRecidRef; if (m.pumpSource == null) { rootRecidRef = BTreeMap.createRootRef(engine, m.serializer, null, m.comparator); } else { rootRecidRef = Pump.buildTreeMap( m.pumpSource, engine, Fun.noTransformExtractor(), null, m.nodeSize, false, counterRecid, m.serializer, null, m.comparator); } NavigableSet<K> ret = new BTreeMap<K, Object>( engine, catPut(m.name + ".rootRecidRef", rootRecidRef), catPut(m.name + ".maxNodeSize", m.nodeSize), false, catPut(m.name + ".counterRecid", counterRecid), m.serializer, null, m.comparator) .keySet(); catalog.put(m.name + ".type", "TreeSet"); collections.put(m.name, new WeakReference<Object>(ret)); return ret; }
public static void main(String[] args) throws IOException { /** max number of elements to import */ final long max = (int) 1e6; /** Open database in temporary directory */ File dbFile = File.createTempFile("mapdb", "temp"); DB db = DBMaker.newFileDB(dbFile) /** disabling Write Ahead Log makes import much faster */ .transactionDisable() .make(); long time = System.currentTimeMillis(); /** * Source of data which randomly generates strings. In real world this would return data from * file. */ Iterator<String> source = new Iterator<String>() { long counter = 0; @Override public boolean hasNext() { return counter < max; } @Override public String next() { counter++; return randomString(10); } @Override public void remove() {} }; /** * BTreeMap Data Pump requires data source to be presorted in reverse order (highest to lowest). * There is method in Data Pump we can use to sort data. It uses temporarly files and can handle * fairly large data sets. */ source = Pump.sort( source, true, 100000, Collections.reverseOrder(BTreeMap.COMPARABLE_COMPARATOR), // reverse order comparator db.getDefaultSerializer()); /** * Disk space used by serialized keys should be minimised. Keys are sorted, so only difference * between consequential keys is stored. This method is called delta-packing and typically saves * 60% of disk space. */ BTreeKeySerializer<String> keySerializer = BTreeKeySerializer.STRING; /** Translates Map Key into Map Value. */ Fun.Function1<Integer, String> valueExtractor = new Fun.Function1<Integer, String>() { @Override public Integer run(String s) { return s.hashCode(); } }; /** Create BTreeMap and fill it with data */ Map<String, Integer> map = db.createTreeMap("map") .pumpSource(source, valueExtractor) // .pumpPresort(100000) // for presorting data we could also use this method .keySerializer(keySerializer) .make(); System.out.println( "Finished; total time: " + (System.currentTimeMillis() - time) / 1000 + "s; there are " + map.size() + " items in map"); db.close(); }