Esempio n. 1
0
  public static void main(String[] args) {
    datacontainers.add(new HashMapDataContainer());
    datacontainers.add(new HardCodeBasicBeanDataContainer());
    datacontainers.add(new HardCodeWrapperBeanDataContainer());
    datacontainers.add(new ApacheDynaBeanDataContainer());
    datacontainers.add(new CglibBeanMapDataContainer());
    datacontainers.add(new DynamicBeanSetObjDataContainer());

    for (int i = 0; i < lens.length; i++) {
      System.out.println("data rows:" + lens[i]);
      for (DataContainer db : datacontainers) {
        Object obj = db.createRows(lens[i]);
        System.out.println(
            MemoryMeasurer.measureBytes(obj)
                + "  --DataContainer:"
                + db.getClass().getSimpleName().replace("DataContainer", ""));
        // System.out.println(ObjectGraphMeasurer.measure(obj));

      }
    }
  }
  /**
   * initializes the Object set of this hash table.
   *
   * @param initialCapacity an <code>int</code> value
   * @return an <code>int</code> value
   * @throws HashtableFullException
   * @throws FileNotFoundException
   */
  public long setUp() throws Exception {
    File _fs = new File(fileName);
    if (!_fs.getParentFile().exists()) {
      _fs.getParentFile().mkdirs();
    }
    SDFSLogger.getLog().info("Folder = " + _fs.getPath());
    SDFSLogger.getLog().info("Loading freebits bitset");
    long rsz = 0;
    long _tbs = maxSz / (256);
    int max = Integer.MAX_VALUE / ProgressiveFileByteArrayLongMap.EL;
    if (_tbs > max) {
      this.hashTblSz = max;
    } else if (_tbs > this.hashTblSz) {
      this.hashTblSz = (int) _tbs;
    }
    SDFSLogger.getLog()
        .info(
            "table setup max="
                + max
                + " maxsz="
                + this.maxSz
                + " _tbs="
                + _tbs
                + " hashTblSz="
                + this.hashTblSz);
    this.hashTblSz = NextPrime.getNextPrimeI((int) (this.hashTblSz));
    File[] files = _fs.getParentFile().listFiles(new DBFileFilter());
    if (files.length > 0) {
      CommandLineProgressBar bar =
          new CommandLineProgressBar("Loading Existing Hash Tables", files.length, System.out);
      this.loadEvent.maxCt = files.length + 128;

      for (int i = 0; i < files.length; i++) {
        this.loadEvent.curCt = this.loadEvent.curCt + 1;
        int sz = NextPrime.getNextPrimeI((int) (this.hashTblSz));
        // SDFSLogger.getLog().debug("will create byte array of size "
        // + sz + " propsize was " + propsize);
        ProgressiveFileByteArrayLongMap m = null;
        String pth = files[i].getPath();
        String pfx = pth.substring(0, pth.length() - 5);
        m = new ProgressiveFileByteArrayLongMap(pfx, sz);
        long mep = m.setUp();
        if (mep > endPos) endPos = mep;
        maps.add(m);
        rsz = rsz + m.size();
        bar.update(i);
        if (!m.isFull() && this.activeWriteMaps.remainingCapacity() > 0) {
          m.setActive(true);
          this.activeWriteMaps.add(m);
          this.loadCacheExecutor.execute(m);
        } else {
          m.setActive(false);
          m.full = true;
        }
      }
      bar.finish();
    }

    this.loadEvent.shortMsg = "Loading BloomFilters";
    if (maps.size() == 0) lbf = new LargeBloomFilter(maxSz, .01);
    else {
      try {
        lbf = new LargeBloomFilter(_fs.getParentFile(), maxSz, true);
      } catch (Exception e) {
        SDFSLogger.getLog().warn("Recreating BloomFilters...");
        this.loadEvent.shortMsg = "Recreating BloomFilters";
        lbf = new LargeBloomFilter(maxSz, .01);
        executor =
            new ThreadPoolExecutor(
                Main.writeThreads,
                Main.writeThreads,
                10,
                TimeUnit.SECONDS,
                worksQueue,
                new ProcessPriorityThreadFactory(Thread.MIN_PRIORITY),
                executionHandler);
        CommandLineProgressBar bar =
            new CommandLineProgressBar("ReCreating BloomFilters", maps.size(), System.out);
        Iterator<ProgressiveFileByteArrayLongMap> iter = maps.iterator();
        int i = 0;
        ArrayList<LBFReconstructThread> al = new ArrayList<LBFReconstructThread>();
        while (iter.hasNext()) {
          ProgressiveFileByteArrayLongMap m = iter.next();
          LBFReconstructThread th = new LBFReconstructThread(lbf, m);
          executor.execute(th);
          al.add(th);
          i++;
          bar.update(i);
        }
        executor.shutdown();
        bar.finish();
        try {
          System.out.print("Waiting for all BloomFilters creation threads to finish");
          while (!executor.awaitTermination(10, TimeUnit.SECONDS)) {
            SDFSLogger.getLog().debug("Awaiting fdisk completion of threads.");
            System.out.print(".");
          }
          for (LBFReconstructThread th : al) {
            if (th.ex != null) throw th.ex;
          }
          System.out.println(" done");
        } catch (Exception e1) {
          throw new IOException(e1);
        }
      }
    }
    while (this.activeWriteMaps.remainingCapacity() > 0) {
      String guid = null;
      boolean written = false;
      while (!written) {
        guid = RandomGUID.getGuid();

        File f = new File(fileName + "-" + guid + ".keys");
        if (!f.exists()) {
          ProgressiveFileByteArrayLongMap activeWMap =
              new ProgressiveFileByteArrayLongMap(fileName + "-" + guid, this.hashTblSz);
          activeWMap.setUp();
          this.maps.add(activeWMap);
          written = true;
          activeWMap.setActive(true);
          this.activeWriteMaps.offer(activeWMap);
        }
      }
    }
    if (SDFSLogger.isDebug()) {
      long mem = MemoryMeasurer.measureBytes(lbf);
      long mmem = MemoryMeasurer.measureBytes(maps);
      SDFSLogger.getLog().debug("Large BloomFilter Size=" + StorageUnit.of(mem).format(mem));
      SDFSLogger.getLog().debug("Maps Size=" + StorageUnit.of(mmem).format(mmem));
    }
    this.loadEvent.endEvent("Loaded entries " + rsz);
    System.out.println("Loaded entries " + rsz);
    SDFSLogger.getLog().info("Active Maps " + this.activeWriteMaps.size());
    SDFSLogger.getLog().info("Loaded entries " + rsz);
    SDFSLogger.getLog().info("Loading BloomFilters " + rsz);
    this.kSz.set(rsz);
    this.closed = false;
    return size;
  }