public static void main(String[] args) {
datacontainers.add(new HashMapDataContainer());
datacontainers.add(new HardCodeBasicBeanDataContainer());
datacontainers.add(new HardCodeWrapperBeanDataContainer());
datacontainers.add(new ApacheDynaBeanDataContainer());
datacontainers.add(new CglibBeanMapDataContainer());
datacontainers.add(new DynamicBeanSetObjDataContainer());
for (int i = 0; i < lens.length; i++) {
System.out.println("data rows:" + lens[i]);
for (DataContainer db : datacontainers) {
Object obj = db.createRows(lens[i]);
System.out.println(
MemoryMeasurer.measureBytes(obj)
+ " --DataContainer:"
+ db.getClass().getSimpleName().replace("DataContainer", ""));
// System.out.println(ObjectGraphMeasurer.measure(obj));
}
}
}
/**
* initializes the Object set of this hash table.
*
* @param initialCapacity an <code>int</code> value
* @return an <code>int</code> value
* @throws HashtableFullException
* @throws FileNotFoundException
*/
public long setUp() throws Exception {
File _fs = new File(fileName);
if (!_fs.getParentFile().exists()) {
_fs.getParentFile().mkdirs();
}
SDFSLogger.getLog().info("Folder = " + _fs.getPath());
SDFSLogger.getLog().info("Loading freebits bitset");
long rsz = 0;
long _tbs = maxSz / (256);
int max = Integer.MAX_VALUE / ProgressiveFileByteArrayLongMap.EL;
if (_tbs > max) {
this.hashTblSz = max;
} else if (_tbs > this.hashTblSz) {
this.hashTblSz = (int) _tbs;
}
SDFSLogger.getLog()
.info(
"table setup max="
+ max
+ " maxsz="
+ this.maxSz
+ " _tbs="
+ _tbs
+ " hashTblSz="
+ this.hashTblSz);
this.hashTblSz = NextPrime.getNextPrimeI((int) (this.hashTblSz));
File[] files = _fs.getParentFile().listFiles(new DBFileFilter());
if (files.length > 0) {
CommandLineProgressBar bar =
new CommandLineProgressBar("Loading Existing Hash Tables", files.length, System.out);
this.loadEvent.maxCt = files.length + 128;
for (int i = 0; i < files.length; i++) {
this.loadEvent.curCt = this.loadEvent.curCt + 1;
int sz = NextPrime.getNextPrimeI((int) (this.hashTblSz));
// SDFSLogger.getLog().debug("will create byte array of size "
// + sz + " propsize was " + propsize);
ProgressiveFileByteArrayLongMap m = null;
String pth = files[i].getPath();
String pfx = pth.substring(0, pth.length() - 5);
m = new ProgressiveFileByteArrayLongMap(pfx, sz);
long mep = m.setUp();
if (mep > endPos) endPos = mep;
maps.add(m);
rsz = rsz + m.size();
bar.update(i);
if (!m.isFull() && this.activeWriteMaps.remainingCapacity() > 0) {
m.setActive(true);
this.activeWriteMaps.add(m);
this.loadCacheExecutor.execute(m);
} else {
m.setActive(false);
m.full = true;
}
}
bar.finish();
}
this.loadEvent.shortMsg = "Loading BloomFilters";
if (maps.size() == 0) lbf = new LargeBloomFilter(maxSz, .01);
else {
try {
lbf = new LargeBloomFilter(_fs.getParentFile(), maxSz, true);
} catch (Exception e) {
SDFSLogger.getLog().warn("Recreating BloomFilters...");
this.loadEvent.shortMsg = "Recreating BloomFilters";
lbf = new LargeBloomFilter(maxSz, .01);
executor =
new ThreadPoolExecutor(
Main.writeThreads,
Main.writeThreads,
10,
TimeUnit.SECONDS,
worksQueue,
new ProcessPriorityThreadFactory(Thread.MIN_PRIORITY),
executionHandler);
CommandLineProgressBar bar =
new CommandLineProgressBar("ReCreating BloomFilters", maps.size(), System.out);
Iterator<ProgressiveFileByteArrayLongMap> iter = maps.iterator();
int i = 0;
ArrayList<LBFReconstructThread> al = new ArrayList<LBFReconstructThread>();
while (iter.hasNext()) {
ProgressiveFileByteArrayLongMap m = iter.next();
LBFReconstructThread th = new LBFReconstructThread(lbf, m);
executor.execute(th);
al.add(th);
i++;
bar.update(i);
}
executor.shutdown();
bar.finish();
try {
System.out.print("Waiting for all BloomFilters creation threads to finish");
while (!executor.awaitTermination(10, TimeUnit.SECONDS)) {
SDFSLogger.getLog().debug("Awaiting fdisk completion of threads.");
System.out.print(".");
}
for (LBFReconstructThread th : al) {
if (th.ex != null) throw th.ex;
}
System.out.println(" done");
} catch (Exception e1) {
throw new IOException(e1);
}
}
}
while (this.activeWriteMaps.remainingCapacity() > 0) {
String guid = null;
boolean written = false;
while (!written) {
guid = RandomGUID.getGuid();
File f = new File(fileName + "-" + guid + ".keys");
if (!f.exists()) {
ProgressiveFileByteArrayLongMap activeWMap =
new ProgressiveFileByteArrayLongMap(fileName + "-" + guid, this.hashTblSz);
activeWMap.setUp();
this.maps.add(activeWMap);
written = true;
activeWMap.setActive(true);
this.activeWriteMaps.offer(activeWMap);
}
}
}
if (SDFSLogger.isDebug()) {
long mem = MemoryMeasurer.measureBytes(lbf);
long mmem = MemoryMeasurer.measureBytes(maps);
SDFSLogger.getLog().debug("Large BloomFilter Size=" + StorageUnit.of(mem).format(mem));
SDFSLogger.getLog().debug("Maps Size=" + StorageUnit.of(mmem).format(mmem));
}
this.loadEvent.endEvent("Loaded entries " + rsz);
System.out.println("Loaded entries " + rsz);
SDFSLogger.getLog().info("Active Maps " + this.activeWriteMaps.size());
SDFSLogger.getLog().info("Loaded entries " + rsz);
SDFSLogger.getLog().info("Loading BloomFilters " + rsz);
this.kSz.set(rsz);
this.closed = false;
return size;
}