/** Opens the database and creates the Map. */
  private void open() throws Exception {

    // use a generic database configuration
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setTransactional(true);
    if (create) {
      dbConfig.setAllowCreate(true);
      dbConfig.setType(DatabaseType.BTREE);
    }

    // catalog is needed for serial bindings (java serialization)
    Database catalogDb = env.openDatabase(null, "catalog", null, dbConfig);
    catalog = new StoredClassCatalog(catalogDb);

    // use Integer tuple binding for key entries
    TupleBinding keyBinding = TupleBinding.getPrimitiveBinding(Integer.class);

    // use String serial binding for data entries
    SerialBinding dataBinding = new SerialBinding(catalog, String.class);

    this.db = env.openDatabase(null, "helloworld", null, dbConfig);

    // create a map view of the database
    this.map = new StoredSortedMap(db, keyBinding, dataBinding, true);
  }
Ejemplo n.º 2
0
  ObjectStore(
      String facet,
      String facetType,
      boolean createDb,
      EvictorI evictor,
      java.util.List<Index> indices,
      boolean populateEmptyIndices) {
    _cache = new Cache(this);

    _facet = facet;

    _evictor = evictor;
    _indices = indices;
    _communicator = evictor.communicator();
    _encoding = evictor.encoding();
    _keepStats = false;

    if (facet.equals("")) {
      _dbName = EvictorI.defaultDb;
    } else {
      _dbName = facet;
    }

    if (facetType != null) {
      //
      // Create a sample servant with this type
      //
      Ice.ValueFactory factory = _communicator.getValueFactoryManager().find(facetType);
      if (factory == null) {
        throw new DatabaseException(
            _evictor.errorPrefix() + "No value factory registered for type-id '" + facetType + "'");
      }

      _sampleServant = factory.create(facetType);
    }

    Connection connection = Util.createConnection(_communicator, evictor.dbEnv().getEnvName());

    try {
      Catalog catalog = new Catalog(connection, Util.catalogName(), true);
      CatalogData catalogData = catalog.get(evictor.filename());

      if (catalogData != null) {
        if (catalogData.evictor) {
          _keepStats = catalogData.value.isEmpty();
        } else {
          DatabaseException ex = new DatabaseException();
          ex.message = _evictor.errorPrefix() + evictor.filename() + " is not an evictor database";
          throw ex;
        }
      }

      com.sleepycat.db.Environment dbEnv = evictor.dbEnv().getEnv();

      //
      // TODO: FREEZE_DB_MODE
      //
      com.sleepycat.db.DatabaseConfig config = new com.sleepycat.db.DatabaseConfig();
      config.setType(com.sleepycat.db.DatabaseType.BTREE);
      config.setAllowCreate(createDb);

      Ice.Properties properties = _evictor.communicator().getProperties();
      String propPrefix = "Freeze.Evictor." + _evictor.filename() + ".";

      int btreeMinKey = properties.getPropertyAsInt(propPrefix + _dbName + ".BtreeMinKey");
      if (btreeMinKey > 2) {
        if (_evictor.trace() >= 1) {
          _evictor
              .communicator()
              .getLogger()
              .trace(
                  "Freeze.Evictor",
                  "Setting \""
                      + _evictor.filename()
                      + "."
                      + _dbName
                      + "\"'s btree minkey to "
                      + btreeMinKey);
        }
        config.setBtreeMinKey(btreeMinKey);
      }

      boolean checksum = properties.getPropertyAsInt(propPrefix + "Checksum") > 0;
      if (checksum) {
        if (_evictor.trace() >= 1) {
          _evictor
              .communicator()
              .getLogger()
              .trace("Freeze.Evictor", "Turning checksum on for \"" + _evictor.filename() + "\"");
        }

        config.setChecksum(true);
      }

      int pageSize = properties.getPropertyAsInt(propPrefix + "PageSize");
      if (pageSize > 0) {
        if (_evictor.trace() >= 1) {
          _evictor
              .communicator()
              .getLogger()
              .trace(
                  "Freeze.Evictor",
                  "Setting \"" + _evictor.filename() + "\"'s pagesize to " + pageSize);
        }
        config.setPageSize(pageSize);
      }

      try {
        Transaction tx = connection.beginTransaction();
        com.sleepycat.db.Transaction txn = Util.getTxn(tx);

        _db = dbEnv.openDatabase(txn, evictor.filename(), _dbName, config);

        for (Index index : _indices) {
          index.associate(this, txn, createDb, populateEmptyIndices);
        }

        if (catalogData == null) {
          catalogData = new CatalogData(true, "::Ice::Identity", "Object");
          catalog.put(evictor.filename(), catalogData);
        }

        tx.commit();
      } catch (java.io.FileNotFoundException dx) {
        throw new NotFoundException(_evictor.errorPrefix() + "Db.open: " + dx.getMessage(), dx);
      } catch (com.sleepycat.db.DatabaseException dx) {
        throw new DatabaseException(_evictor.errorPrefix() + "Db.open: " + dx.getMessage(), dx);
      } finally {
        Transaction tx = connection.currentTransaction();
        if (tx != null) {
          try {
            tx.rollback();
          } catch (DatabaseException de) {
          }
        }
      }
    } finally {
      connection.close();
    }
  }
Ejemplo n.º 3
0
  //
  // Initialize the database to the number of accounts, branches,
  // history records, and tellers given to the constructor.
  //
  public void populate() {
    Database dbp = null;

    int err;
    int balance, idnum;
    int end_anum, end_bnum, end_tnum;
    int start_anum, start_bnum, start_tnum;
    int h_nelem;

    idnum = BEGID;
    balance = 500000;

    h_nelem = accounts;

    try {
      DatabaseConfig config = new DatabaseConfig();
      config.setType(DatabaseType.HASH);
      config.setHashNumElements(h_nelem);
      config.setAllowCreate(true);
      dbp = dbenv.openDatabase(null, "account", null, config);
    } catch (Exception e1) {
      // can be DatabaseException or FileNotFoundException
      errExit(e1, "Open of account file failed");
    }

    start_anum = idnum;
    populateTable(dbp, idnum, balance, h_nelem, "account");
    idnum += h_nelem;
    end_anum = idnum - 1;
    try {
      dbp.close();
    } catch (DatabaseException e2) {
      errExit(e2, "Account file close failed");
    }

    if (verbose)
      System.out.println(
          "Populated accounts: " + String.valueOf(start_anum) + " - " + String.valueOf(end_anum));

    //
    // Since the number of branches is very small, we want to use very
    // small pages and only 1 key per page.  This is the poor-man's way
    // of getting key locking instead of page locking.
    //
    h_nelem = (int) branches;

    try {
      DatabaseConfig config = new DatabaseConfig();
      config.setType(DatabaseType.HASH);
      config.setHashNumElements(h_nelem);
      config.setHashFillFactor(1);
      config.setPageSize(512);
      config.setAllowCreate(true);
      dbp = dbenv.openDatabase(null, "branch", null, config);
    } catch (Exception e3) {
      // can be DatabaseException or FileNotFoundException
      errExit(e3, "Branch file create failed");
    }

    start_bnum = idnum;
    populateTable(dbp, idnum, balance, h_nelem, "branch");
    idnum += h_nelem;
    end_bnum = idnum - 1;

    try {
      dbp.close();
    } catch (DatabaseException dbe4) {
      errExit(dbe4, "Close of branch file failed");
    }

    if (verbose)
      System.out.println(
          "Populated branches: " + String.valueOf(start_bnum) + " - " + String.valueOf(end_bnum));

    //
    // In the case of tellers, we also want small pages, but we'll let
    // the fill factor dynamically adjust itself.
    //
    h_nelem = (int) tellers;

    try {
      DatabaseConfig config = new DatabaseConfig();
      config.setType(DatabaseType.HASH);
      config.setHashNumElements(h_nelem);
      config.setHashFillFactor(0);
      config.setPageSize(512);
      config.setAllowCreate(true);
      dbp = dbenv.openDatabase(null, "teller", null, config);
    } catch (Exception e5) {
      // can be DatabaseException or FileNotFoundException
      errExit(e5, "Teller file create failed");
    }

    start_tnum = idnum;
    populateTable(dbp, idnum, balance, h_nelem, "teller");
    idnum += h_nelem;
    end_tnum = idnum - 1;

    try {
      dbp.close();
    } catch (DatabaseException e6) {
      errExit(e6, "Close of teller file failed");
    }

    if (verbose)
      System.out.println(
          "Populated tellers: " + String.valueOf(start_tnum) + " - " + String.valueOf(end_tnum));

    try {
      DatabaseConfig config = new DatabaseConfig();
      config.setType(DatabaseType.RECNO);
      config.setRecordLength(HISTORY_LEN);
      config.setAllowCreate(true);
      dbp = dbenv.openDatabase(null, "history", null, config);
    } catch (Exception e7) {
      // can be DatabaseException or FileNotFoundException
      errExit(e7, "Create of history file failed");
    }

    populateHistory(dbp);

    try {
      dbp.close();
    } catch (DatabaseException e8) {
      errExit(e8, "Close of history file failed");
    }
  }
Ejemplo n.º 4
0
  /** Open all storage containers, indices, and catalogs. */
  public SampleDatabase(String homeDirectory) throws DatabaseException, FileNotFoundException {

    // Open the Berkeley DB environment in transactional mode.
    //
    System.out.println("Opening environment in: " + homeDirectory);
    EnvironmentConfig envConfig = new EnvironmentConfig();
    envConfig.setTransactional(true);
    envConfig.setAllowCreate(true);
    envConfig.setInitializeCache(true);
    envConfig.setInitializeLocking(true);
    env = new Environment(new File(homeDirectory), envConfig);

    // Set the Berkeley DB config for opening all stores.
    //
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setTransactional(true);
    dbConfig.setAllowCreate(true);
    dbConfig.setType(DatabaseType.BTREE);

    // Create the Serial class catalog.  This holds the serialized class
    // format for all database records of serial format.
    //
    Database catalogDb = env.openDatabase(null, CLASS_CATALOG, null, dbConfig);
    javaCatalog = new StoredClassCatalog(catalogDb);

    // Open the Berkeley DB database for the part, supplier and shipment
    // stores.  The stores are opened with no duplicate keys allowed.
    //
    partDb = env.openDatabase(null, PART_STORE, null, dbConfig);

    supplierDb = env.openDatabase(null, SUPPLIER_STORE, null, dbConfig);

    shipmentDb = env.openDatabase(null, SHIPMENT_STORE, null, dbConfig);

    // Open the SecondaryDatabase for the city index of the supplier store,
    // and for the part and supplier indices of the shipment store.
    // Duplicate keys are allowed since more than one supplier may be in
    // the same city, and more than one shipment may exist for the same
    // supplier or part.  A foreign key constraint is defined for the
    // supplier and part indices to ensure that a shipment only refers to
    // existing part and supplier keys.  The CASCADE delete action means
    // that shipments will be deleted if their associated part or supplier
    // is deleted.
    //
    SecondaryConfig secConfig = new SecondaryConfig();
    secConfig.setTransactional(true);
    secConfig.setAllowCreate(true);
    secConfig.setType(DatabaseType.BTREE);
    secConfig.setSortedDuplicates(true);

    secConfig.setKeyCreator(
        new SupplierByCityKeyCreator(
            javaCatalog, SupplierKey.class, SupplierData.class, String.class));
    supplierByCityDb =
        env.openSecondaryDatabase(null, SUPPLIER_CITY_INDEX, null, supplierDb, secConfig);

    secConfig.setForeignKeyDatabase(partDb);
    secConfig.setForeignKeyDeleteAction(ForeignKeyDeleteAction.CASCADE);
    secConfig.setKeyCreator(
        new ShipmentByPartKeyCreator(
            javaCatalog, ShipmentKey.class, ShipmentData.class, PartKey.class));
    shipmentByPartDb =
        env.openSecondaryDatabase(null, SHIPMENT_PART_INDEX, null, shipmentDb, secConfig);

    secConfig.setForeignKeyDatabase(supplierDb);
    secConfig.setForeignKeyDeleteAction(ForeignKeyDeleteAction.CASCADE);
    secConfig.setKeyCreator(
        new ShipmentBySupplierKeyCreator(
            javaCatalog, ShipmentKey.class, ShipmentData.class, SupplierKey.class));
    shipmentBySupplierDb =
        env.openSecondaryDatabase(null, SHIPMENT_SUPPLIER_INDEX, null, shipmentDb, secConfig);
  }
Ejemplo n.º 5
0
  @Test
  public void testDraining() throws Exception {
    EnvironmentConfig masterConfig = makeBasicConfig();
    masterConfig.setReplicationLimit(100000000);
    ReplicationManagerSiteConfig site = new ReplicationManagerSiteConfig("localhost", masterPort);
    site.setLocalSite(true);
    site.setLegacy(true);
    masterConfig.addReplicationManagerSite(site);

    site = new ReplicationManagerSiteConfig("localhost", clientPort);
    site.setLegacy(true);
    masterConfig.addReplicationManagerSite(site);
    site = new ReplicationManagerSiteConfig("localhost", client2Port);
    site.setLegacy(true);
    masterConfig.addReplicationManagerSite(site);
    site = new ReplicationManagerSiteConfig("localhost", client3Port);
    site.setLegacy(true);
    masterConfig.addReplicationManagerSite(site);

    Environment master = new Environment(mkdir("master"), masterConfig);
    setTimeouts(master);
    // Prevent connection retries, so that all connections
    // originate from clients
    master.setReplicationTimeout(ReplicationTimeoutType.CONNECTION_RETRY, Integer.MAX_VALUE);
    master.replicationManagerStart(2, ReplicationManagerStartPolicy.REP_MASTER);

    DatabaseConfig dc = new DatabaseConfig();
    dc.setTransactional(true);
    dc.setAllowCreate(true);
    dc.setType(DatabaseType.BTREE);
    dc.setPageSize(4096);
    Database db = master.openDatabase(null, "test.db", null, dc);

    DatabaseEntry key = new DatabaseEntry();
    DatabaseEntry value = new DatabaseEntry();
    value.setData(data);

    for (int i = 0; ((BtreeStats) db.getStats(null, null)).getPageCount() < 500; i++) {
      String k = "The record number is: " + i;
      key.setData(k.getBytes());
      db.put(null, key, value);
    }

    // tell fiddler to stop reading once it sees a PAGE message
    Socket s = new Socket("localhost", mgrPort);
    OutputStreamWriter w = new OutputStreamWriter(s.getOutputStream());

    String path1 = "{" + masterPort + "," + clientPort + "}"; // looks like {6000,6001}
    w.write("{init," + path1 + ",page_clog}\r\n");
    w.flush();
    BufferedReader br = new BufferedReader(new InputStreamReader(s.getInputStream()));
    br.readLine();
    assertEquals("ok", br.readLine());
    // create client
    //
    EnvironmentConfig ec = makeBasicConfig();
    site = new ReplicationManagerSiteConfig("localhost", clientPort);
    site.setLocalSite(true);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);
    site = new ReplicationManagerSiteConfig("localhost", masterPort);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);
    site = new ReplicationManagerSiteConfig("localhost", client2Port);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);
    site = new ReplicationManagerSiteConfig("localhost", client3Port);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);
    Environment client = new Environment(mkdir("client"), ec);
    setTimeouts(client);
    client.replicationManagerStart(1, ReplicationManagerStartPolicy.REP_CLIENT);

    // wait til it gets stuck
    Thread.sleep(5000); // FIXME

    // Do the same for another client, because the master has 2
    // msg processing threads.  (It's no longer possible to
    // configure just 1.)
    String path2 = "{" + masterPort + "," + client2Port + "}";
    w.write("{init," + path2 + ",page_clog}\r\n");
    w.flush();
    br = new BufferedReader(new InputStreamReader(s.getInputStream()));
    br.readLine();
    assertEquals("ok", br.readLine());

    ec = makeBasicConfig();
    site = new ReplicationManagerSiteConfig("localhost", client2Port);
    site.setLocalSite(true);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);
    site = new ReplicationManagerSiteConfig("localhost", masterPort);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);
    site = new ReplicationManagerSiteConfig("localhost", clientPort);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);
    site = new ReplicationManagerSiteConfig("localhost", client3Port);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);
    Environment client2 = new Environment(mkdir("client2"), ec);
    setTimeouts(client2);
    client2.replicationManagerStart(1, ReplicationManagerStartPolicy.REP_CLIENT);

    // wait til it gets stuck
    Thread.sleep(5000);

    // With the connection stuck, the master cannot write out log
    // records for new "live" transactions.  Knowing we didn't
    // write the record, we should not bother waiting for an ack
    // that cannot possibly arrive; so we should simply return
    // quickly.  The duration should be very quick, but anything
    // less than the ack timeout indicates correct behavior (in
    // case this test runs on a slow, overloaded system).
    //
    long startTime = System.currentTimeMillis();
    key.setData("one extra record".getBytes());
    db.put(null, key, value);
    long duration = System.currentTimeMillis() - startTime;
    assertTrue("txn duration: " + duration, duration < 29000);
    System.out.println("txn duration: " + duration);
    db.close();

    // Tell fiddler to close the connections.  That should trigger
    // us to abandon the timeout.  Then create another client and
    // see that it can complete its internal init quickly.  Since
    // we have limited threads at the master, this demonstrates
    // that they were abandoned.
    //
    path1 = "{" + clientPort + "," + masterPort + "}"; // looks like {6001,6000}
    w.write("{" + path1 + ",shutdown}\r\n");
    w.flush();
    assertEquals("ok", br.readLine());
    path2 = "{" + client2Port + "," + masterPort + "}"; // looks like {6001,6000}
    w.write("{" + path2 + ",shutdown}\r\n");
    w.flush();
    assertEquals("ok", br.readLine());

    ec = makeBasicConfig();
    site = new ReplicationManagerSiteConfig("localhost", client3Port);
    site.setLocalSite(true);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);
    site = new ReplicationManagerSiteConfig("localhost", masterPort);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);
    site = new ReplicationManagerSiteConfig("localhost", clientPort);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);
    site = new ReplicationManagerSiteConfig("localhost", client2Port);
    site.setLegacy(true);
    ec.addReplicationManagerSite(site);

    EventHandler clientMonitor = new EventHandler();
    ec.setEventHandler(clientMonitor);
    Environment client3 = new Environment(mkdir("client3"), ec);
    setTimeouts(client3);
    startTime = System.currentTimeMillis();
    client3.replicationManagerStart(2, ReplicationManagerStartPolicy.REP_CLIENT);
    clientMonitor.await();
    duration = System.currentTimeMillis() - startTime;
    assertTrue("sync duration: " + duration, duration < 20000); // 20 seconds should be plenty

    client3.close();
    master.close();

    w.write("shutdown\r\n");
    w.flush();
    assertEquals("ok", br.readLine());
    s.close();
  }