Example #1
0
  public AbstractFrontier(String homeDirectory) throws DatabaseException, FileNotFoundException {
    // 打开env
    File file = new File(homeDirectory);
    if (!file.exists()) {
      file.mkdirs();
    }
    System.out.println("Opening environment in: " + homeDirectory);
    EnvironmentConfig envConfig = new EnvironmentConfig();
    envConfig.setTransactional(true);
    envConfig.setAllowCreate(true);

    env = new Environment(new File(homeDirectory), envConfig);
    // 设置DatabaseConfig
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setTransactional(true);
    dbConfig.setAllowCreate(true);
    dbConfig.setSortedDuplicates(false);
    // 打开
    catalogdatabase = env.openDatabase(null, CLASS_CATALOG, dbConfig);
    javaCatalog = new StoredClassCatalog(catalogdatabase);
    // 设置DatabaseConfig
    DatabaseConfig dbConfig0 = new DatabaseConfig();
    dbConfig0.setTransactional(true);
    dbConfig0.setAllowCreate(true);
    // 打开
    database = env.openDatabase(null, "URL", dbConfig);
  }
Example #2
0
  public void merge() throws Exception {
    LOG.info("start merge");
    Database crawldbDatabase = env.openDatabase(null, "crawldb", BerkeleyDBUtils.defaultDBConfig);
    /*合并fetch库*/
    LOG.info("merge fetch database");
    Database fetchDatabase = env.openDatabase(null, "fetch", BerkeleyDBUtils.defaultDBConfig);
    Cursor fetchCursor = fetchDatabase.openCursor(null, null);
    DatabaseEntry key = new DatabaseEntry();
    DatabaseEntry value = new DatabaseEntry();
    while (fetchCursor.getNext(key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
      crawldbDatabase.put(null, key, value);
    }
    fetchCursor.close();
    fetchDatabase.close();
    /*合并link库*/
    LOG.info("merge link database");
    Database linkDatabase = env.openDatabase(null, "link", BerkeleyDBUtils.defaultDBConfig);
    Cursor linkCursor = linkDatabase.openCursor(null, null);
    while (linkCursor.getNext(key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
      if (!(crawldbDatabase.get(null, key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS)) {
        crawldbDatabase.put(null, key, value);
      }
    }
    linkCursor.close();
    linkDatabase.close();
    LOG.info("end merge");
    crawldbDatabase.sync();
    crawldbDatabase.close();

    env.removeDatabase(null, "fetch");
    LOG.debug("remove fetch database");
    env.removeDatabase(null, "link");
    LOG.debug("remove link database");
  }
  private Environment getEnvironment(String storeName) throws DatabaseException {
    synchronized (lock) {
      if (useOneEnvPerStore) {

        // if we have already created this environment return a
        // reference
        if (environments.containsKey(storeName)) return environments.get(storeName);

        // otherwise create a new environment
        File bdbDir = new File(bdbMasterDir, storeName);
        createBdbDirIfNecessary(bdbDir);

        Environment environment = new Environment(bdbDir, environmentConfig);
        logger.info("Creating environment for " + storeName + ": ");
        logEnvironmentConfig(environment.getConfig());
        environments.put(storeName, environment);
        return environment;
      } else {
        if (!environments.isEmpty()) return environments.get(SHARED_ENV_KEY);

        File bdbDir = new File(bdbMasterDir);
        createBdbDirIfNecessary(bdbDir);

        Environment environment = new Environment(bdbDir, environmentConfig);
        logger.info("Creating shared BDB environment: ");
        logEnvironmentConfig(environment.getConfig());
        environments.put(SHARED_ENV_KEY, environment);
        return environment;
      }
    }
  }
  public BDB_GIDLookupImpl(String aim) {

    EnvironmentConfig environmentConfig = new EnvironmentConfig();

    // will be created if not existing -> will crash if folder not found
    environmentConfig.setAllowCreate(true);

    // no transaction yet -> might be needed later
    // not sure if needed to be set in environment and database
    environmentConfig.setTransactional(false);

    File file = new File(aim);

    if (!file.exists()) {
      file.mkdirs();
    }

    environment = new Environment(file, environmentConfig);
    DatabaseConfig databaseConfig = new DatabaseConfig();

    // will be created if not existing -> will crash if folder not found
    databaseConfig.setAllowCreate(true);

    // no transaction yet -> might be needed later
    // not sure if needed to be set in environment and database
    databaseConfig.setTransactional(false);

    // create 2 "tables" one for relations-gid one for node-gid
    RelationBDB = environment.openDatabase(null, "Relation", databaseConfig);
    NodeBDB = environment.openDatabase(null, "Node", databaseConfig);
  }
  /**
   * Close the database.
   *
   * @param cleanLog if true then wait for the BerkeleyDB clean thread to finish.
   */
  @Override
  public void close(boolean cleanLog) {
    log.info("Closing db & env for: " + dir.getAbsolutePath());
    if (openIterators.size() > 0) {
      log.warn("closing " + openIterators.size() + " iterators on close");
      for (Object e : openIterators.toArray(new Object[openIterators.size()])) {
        ((ClosableIterator<Map.Entry<byte[], byte[]>>) e).close();
      }
    }
    log.info("pages:gets=" + gets + " puts=" + puts + " in=" + bytesIn + " out=" + bytesOut);

    bdb.close();
    if (cleanLog) {
      bdb_env.getConfig().setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, "false");
      int totalLogFiles = 0;
      int logFiles;
      do {
        logFiles = bdb_env.cleanLog();
        totalLogFiles += logFiles;
      } while (logFiles > 0);

      log.info("Total of " + totalLogFiles + " log files cleaned.");

      if (totalLogFiles > 0) {
        CheckpointConfig force = new CheckpointConfig();
        force.setForce(true);
        bdb_env.checkpoint(force);
      }
    }
    bdb_env.close();
  }
  /*
   * Check that the notReplicate attribute is properly immutable and
   * persistent.
   */
  private void validate(DatabaseConfig config, boolean replicated) throws DatabaseException {

    /* Create the database -- is its config what we expect? */
    db = env.openDatabase(null, TEST_DB, config);
    DatabaseConfig inUseConfig = db.getConfig();
    assertEquals(replicated, DbInternal.getReplicated(inUseConfig));

    /* Close, re-open. */
    db.close();
    db = null;
    db = env.openDatabase(null, TEST_DB, inUseConfig);
    assertEquals(replicated, DbInternal.getReplicated(db.getConfig()));

    /*
     * Close, re-open w/inappropriate value for the replicated bit. This is
     * only checked for replicated environments.
     */
    db.close();
    db = null;
    if (DbInternal.getEnvironmentImpl(env).isReplicated()) {
      DbInternal.setReplicated(inUseConfig, !replicated);
      try {
        db = env.openDatabase(null, TEST_DB, inUseConfig);
        fail("Should have caught config mismatch");
      } catch (IllegalArgumentException expected) {
      }
    }
  }
Example #7
0
  public void run(File envHomeDirectory) throws DatabaseException, IOException {

    /* Create the environment object. */
    EnvironmentConfig envConfig = new EnvironmentConfig();
    envConfig.setAllowCreate(true);
    Environment env = new Environment(envHomeDirectory, envConfig);

    /* Create the database object. */
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setAllowCreate(true);
    Database db = env.openDatabase(null, DB_NAME, dbConfig);

    /* Create the sequence oject. */
    SequenceConfig config = new SequenceConfig();
    config.setAllowCreate(true);
    DatabaseEntry key = new DatabaseEntry(KEY_NAME.getBytes("UTF-8"));
    Sequence seq = db.openSequence(null, key, config);

    /* Allocate a few sequence numbers. */
    for (int i = 0; i < 10; i++) {
      long seqnum = seq.get(null, 1);
      System.out.println("Got sequence number: " + seqnum);
    }

    /* Close all. */
    seq.close();
    db.close();
    env.close();
  }
 /* Close the Environments after finishing reading operations. */
 private void closeEnvironments() {
   if (master != null) {
     master.close();
   }
   if (replica != null) {
     replica.close();
   }
 }
 /** @see org.geogit.storage.ObjectDatabase#create() */
 @Override
 public void create() {
   txn = CurrentTransaction.getInstance(env);
   DatabaseConfig dbConfig = new DatabaseConfig();
   dbConfig.setAllowCreate(true);
   dbConfig.setTransactional(env.getConfig().getTransactional());
   this.objectDb = env.openDatabase(null, "BlobStore", dbConfig);
 }
 private void expectNothingToClean() {
   env.cleanLog();
   final EnvironmentStats stats = env.getStats(null);
   final String msg =
       String.format(
           "%d probes, %d non-probes", stats.getNCleanerProbeRuns(), stats.getNCleanerRuns());
   assertEquals(msg, 0, stats.getNCleanerRuns() - stats.getNCleanerProbeRuns());
 }
  @Override
  public void shutdown() {
    // ------ closing down everything
    // sync according to literature needed if no transaction used
    RelationBDB.close();
    NodeBDB.close();

    environment.sync();
    environment.close();
  }
 public EnvironmentStats getStats(String storeName, boolean fast) {
   StatsConfig config = new StatsConfig();
   config.setFast(fast);
   try {
     Environment env = getEnvironment(storeName);
     return env.getStats(config);
   } catch (DatabaseException e) {
     throw new VoldemortException(e);
   }
 }
  private void doReplicaHasGapNetworkRestore(boolean multiGaps) throws Throwable {

    Durability noAck =
        new Durability(SyncPolicy.NO_SYNC, SyncPolicy.NO_SYNC, ReplicaAckPolicy.NONE);
    db = null;
    try {
      Environment master = setupLogWithCleanedGaps(multiGaps);
      int masterIdx = findMasterIndex(master);
      /*
       * Write a record, so that we are sure that there will be a
       * network restore, because we have to cross a checkpoint.
       */
      generateData(master, 1, noAck, false);
      CheckpointConfig cc = new CheckpointConfig();
      master.checkpoint(cc);
      EnvironmentStats stats = master.getStats(clearConfig);
      assertEquals(0, stats.getCleanerBacklog());
      if (multiGaps) {
        logger.info("Multigap: deletion backlog = " + stats.getFileDeletionBacklog());
      } else {
        assertEquals(0, stats.getFileDeletionBacklog());
      }

      db.close();
      db = null;
      repEnvInfo[masterIdx].closeEnv();

      /* Start up the two replicas */
      openReplicas(masterIdx);

      /* Start the node that had been the master */
      try {
        repEnvInfo[masterIdx].openEnv();
        fail("Should be a network restore");
      } catch (InsufficientLogException ile) {
        repEnvInfo[masterIdx].closeEnv();
        NetworkRestore restore = new NetworkRestore();
        NetworkRestoreConfig config = new NetworkRestoreConfig();
        config.setRetainLogFiles(true);
        restore.execute(ile, config);
        repEnvInfo[masterIdx].openEnv();
      }

      /* Check its last VLSN and size. */

    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    } finally {
      if (db != null) {
        db.close();
      }
      RepTestUtils.shutdownRepEnvs(repEnvInfo);
    }
  }
  /**
   * Make a thread allocate a vlsn, but then fail before it's tracked by the vlsn index. This
   * happened in [#20919] when 1.rep environment close was called 2.the repNode was nulled out 3.a
   * concurrent writing thread got a NPE within its call to LogManager.log because the repNode was
   * null. This thread exited after it had bumped the vlsn, but before it had entered the vlsn in
   * the vlsnIndex 4.rep environment close tried to do a checkpoint, but the checkpoint hung. This
   * fix works by having (3) invalidate the environment, and by having (4) check for an invalidated
   * environment.
   */
  @Test
  public void testLoggingFailure() throws DatabaseException, IOException {

    /* Make a single replicated environment. */
    RepEnvInfo[] repEnvInfo = RepTestUtils.setupEnvInfos(envRoot, 1);
    RepTestUtils.joinGroup(repEnvInfo);

    /*
     * Disable cleaning and CBVLSN updating, to control vlsn creation
     * explicitly.
     */
    Environment env = repEnvInfo[0].getEnv();
    EnvironmentMutableConfig config = env.getMutableConfig();
    config.setConfigParam("je.env.runCleaner", "false");
    env.setMutableConfig(config);
    LocalCBVLSNUpdater.setSuppressGroupDBUpdates(false);

    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setTransactional(true);
    dbConfig.setAllowCreate(true);
    Database db = env.openDatabase(null, "foo", dbConfig);
    DatabaseEntry value = new DatabaseEntry(new byte[4]);

    EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env);
    LogManager logManager = DbInternal.getEnvironmentImpl(env).getLogManager();

    /*
     * Inject an exception into the next call to log() that is made
     * for a replicated log entry.
     */
    logManager.setDelayVLSNRegisterHook(new ForceException());

    VLSNIndex vlsnIndex = ((RepImpl) envImpl).getVLSNIndex();

    try {
      db.put(null, value, value);
      fail("Should throw exception");
    } catch (Exception expected) {
      assertTrue(
          "latest="
              + vlsnIndex.getLatestAllocatedVal()
              + " last mapped="
              + vlsnIndex.getRange().getLast().getSequence(),
          vlsnIndex.getLatestAllocatedVal() > vlsnIndex.getRange().getLast().getSequence());
    }

    try {
      VLSNIndex.AWAIT_CONSISTENCY_MS = 1000;
      envImpl.awaitVLSNConsistency();
      fail("Should throw and break out");
    } catch (DatabaseException expected) {
    }

    /* Before the fix, this test hung. */
  }
 @Test
 public void testCleanAfterMinUtilizationChange() {
   open(true /*runCleaner*/);
   writeFiles(4 /*nActive*/, 3 /*nObsolete*/);
   expectNothingToClean();
   final EnvironmentConfig envConfig = env.getConfig();
   envConfig.setConfigParam(EnvironmentConfig.CLEANER_MIN_UTILIZATION, "90");
   env.setMutableConfig(envConfig);
   expectBackgroundCleaning();
   close();
 }
 public void close() {
   synchronized (lock) {
     try {
       for (Environment environment : environments.values()) {
         environment.sync();
         environment.close();
       }
     } catch (DatabaseException e) {
       throw new VoldemortException(e);
     }
   }
 }
 /** Forceful cleanup the logs */
 @JmxOperation(description = "Forceful start the cleaner threads")
 public void cleanLogs() {
   synchronized (lock) {
     try {
       for (Environment environment : environments.values()) {
         environment.cleanLog();
       }
     } catch (DatabaseException e) {
       throw new VoldemortException(e);
     }
   }
 }
Example #18
0
  /**
   * Set an attribute value for the given environment.
   *
   * @param targetEnv The target JE environment. May be null if the environment is not open.
   * @param attribute name/value pair
   */
  public void setAttribute(Environment targetEnv, Attribute attribute)
      throws AttributeNotFoundException, InvalidAttributeValueException {

    if (attribute == null) {
      throw new AttributeNotFoundException("Attribute cannot be null");
    }

    /* Sanity check parameters. */
    String name = attribute.getName();
    Object value = attribute.getValue();

    if (name == null) {
      throw new AttributeNotFoundException("Attribute name cannot be null");
    }

    if (value == null) {
      throw new InvalidAttributeValueException(
          "Attribute value for attribute " + name + " cannot be null");
    }

    try {
      if (name.equals(ATT_SET_READ_ONLY)) {
        openConfig.setReadOnly(((Boolean) value).booleanValue());
      } else if (name.equals(ATT_SET_TRANSACTIONAL)) {
        openConfig.setTransactional(((Boolean) value).booleanValue());
      } else if (name.equals(ATT_SET_SERIALIZABLE)) {
        openConfig.setTxnSerializableIsolation(((Boolean) value).booleanValue());
      } else {
        /* Set the specified attribute if the environment is open. */
        if (targetEnv != null) {

          EnvironmentMutableConfig config = targetEnv.getMutableConfig();

          if (name.equals(ATT_CACHE_SIZE)) {
            config.setCacheSize(((Long) value).longValue());
            targetEnv.setMutableConfig(config);
          } else if (name.equals(ATT_CACHE_PERCENT)) {
            config.setCachePercent(((Integer) value).intValue());
            targetEnv.setMutableConfig(config);
          } else {
            throw new AttributeNotFoundException("attribute " + name + " is not valid.");
          }
        } else {
          throw new AttributeNotFoundException("attribute " + name + " is not valid.");
        }
      }
    } catch (NumberFormatException e) {
      throw new InvalidAttributeValueException("attribute name=" + name);
    } catch (DatabaseException e) {
      throw new InvalidAttributeValueException("attribute name=" + name + e.getMessage());
    }
  }
 public StorageEngine<ByteArray, byte[], byte[]> getStore(String storeName) {
   synchronized (lock) {
     try {
       Environment environment = getEnvironment(storeName);
       Database db = environment.openDatabase(null, storeName, databaseConfig);
       BdbRuntimeConfig runtimeConfig = new BdbRuntimeConfig(voldemortConfig);
       BdbStorageEngine engine = new BdbStorageEngine(storeName, environment, db, runtimeConfig);
       return engine;
     } catch (DatabaseException d) {
       throw new StorageInitializationException(d);
     }
   }
 }
Example #20
0
  public static void main(String[] args) {
    Environment env = null;
    Database db = null;
    EnvironmentConfig envconfig = new EnvironmentConfig();
    envconfig.setAllowCreate(true);
    try {
      env = new Environment(new File("D://bdb"), envconfig);
      DatabaseConfig dbconfig = new DatabaseConfig();
      dbconfig.setAllowCreate(true);
      db = env.openDatabase(null, "dbac.db", dbconfig);
      String key = "mykey";
      DatabaseEntry thekey = new DatabaseEntry();
      thekey.setData(key.getBytes("utf-8"));

      Long value = new Long(123456);
      DatabaseEntry thevalue = new DatabaseEntry();
      EntryBinding myBinging = TupleBinding.getPrimitiveBinding(Long.class);
      myBinging.objectToEntry(value, thevalue);
      // LongBinding myLongBinging=(LongBinding)TupleBinding.getPrimitiveBinding(Long.class);
      // myLongBinging.objectToEntry(value, thevalue);
      db.put(null, thekey, thevalue);
      DatabaseEntry valueEntry = new DatabaseEntry();
      OperationStatus status = db.get(null, thekey, valueEntry, LockMode.DEFAULT);
      if (status == OperationStatus.SUCCESS) {
        // Long number=myLongBinging.entryToObject(valueEntry);
        Long number = (Long) myBinging.entryToObject(valueEntry);
        System.out.println(env.getDatabaseNames());
        System.out.println(number);
      }
    } catch (EnvironmentLockedException e) {
      e.printStackTrace();
    } catch (Exception e) {
      e.printStackTrace();
    } finally {
      if (db != null) {
        try {
          db.close();
        } catch (DatabaseException e) {
          e.printStackTrace();
        }
      }
      if (env != null) {
        try {
          env.cleanLog();
          env.close();
        } catch (DatabaseException e) {
          e.printStackTrace();
        }
      }
    }
  }
  public BerkeleyStorage(File file, boolean readonly, boolean async) {
    if (file == null) {
      int pid = 0;
      try {
        pid = Integer.parseInt((new File("/proc/self")).getCanonicalFile().getName());
      } catch (NumberFormatException | IOException e) {
      }
      String path = "/tmp";
      String db_path = System.getenv("DB");
      if (db_path != null) {
        path = db_path;
      }
      file = new File(path + "/ringpaxos-db/" + pid);
      file.mkdirs();
    }
    EnvironmentConfig envConfig = new EnvironmentConfig();
    DatabaseConfig dbConfig = new DatabaseConfig();
    envConfig.setReadOnly(readonly);
    dbConfig.setReadOnly(readonly);
    envConfig.setAllowCreate(!readonly);
    dbConfig.setAllowCreate(!readonly);

    // performance settings
    envConfig.setTransactional(true);
    envConfig.setCacheMode(CacheMode.DEFAULT);
    // envConfig.setCacheSize(1000000*800); // 800M
    if (async) {
      dbConfig.setTransactional(false);
      envConfig.setDurability(Durability.COMMIT_NO_SYNC);
      dbConfig.setDeferredWrite(true);
    } else {
      dbConfig.setTransactional(true);
      envConfig.setDurability(Durability.COMMIT_SYNC);
      dbConfig.setDeferredWrite(false);
    }

    env = new Environment(file, envConfig);
    db = env.openDatabase(null, "paxosDB", dbConfig);
    classCatalogDb = env.openDatabase(null, "ClassCatalogDB", dbConfig);
    classCatalog = new StoredClassCatalog(classCatalogDb);
    keyBinding = TupleBinding.getPrimitiveBinding(Long.class);
    dataBinding = new SerialBinding<Decision>(classCatalog, Decision.class);
    ballotBinding = TupleBinding.getPrimitiveBinding(Integer.class);

    logger.info("BerkeleyStorage cache size: " + env.getMutableConfig().getCacheSize());
    logger.info(
        "BerkeleyStorage durability: " + env.getMutableConfig().getDurability().getLocalSync());
    logger.info("BerkeleyStorage deferred write: " + db.getConfig().getDeferredWrite());
  }
Example #22
0
  private void verifyDb(Hashtable dataMap, int dumpIndex) throws DatabaseException {

    DatabaseConfig config = new DatabaseConfig();
    config.setReadOnly(true);
    DbInternal.setUseExistingConfig(config, true);
    Database myDb = env.openDatabase(null, dbName + dumpIndex, config);
    Cursor cursor = myDb.openCursor(null, null);
    StringDbt foundKey = new StringDbt();
    StringDbt foundData = new StringDbt();
    OperationStatus status = cursor.getFirst(foundKey, foundData, LockMode.DEFAULT);
    while (status == OperationStatus.SUCCESS) {
      String foundKeyString = foundKey.getString();
      String foundDataString = foundData.getString();
      if (dataMap.get(foundKeyString) != null) {
        assertTrue(((String) dataMap.get(foundKeyString)).equals(foundDataString));
        dataMap.remove(foundKeyString);
      } else {
        fail("didn't find key in either map (" + foundKeyString + ")");
      }
      status = cursor.getNext(foundKey, foundData, LockMode.DEFAULT);
    }
    assertTrue(dataMap.size() == 0);
    cursor.close();
    myDb.close();
  }
  private static void closeEnv() {
    System.out.println("Closing env and databases");
    if (myDb != null) {
      try {
        myDb.close();
      } catch (DatabaseException e) {
        System.err.println("closeEnv: myDb: " + e.toString());
        e.printStackTrace();
      }
    }

    if (myClassDb != null) {
      try {
        myClassDb.close();
      } catch (DatabaseException e) {
        System.err.println("closeEnv: myClassDb: " + e.toString());
        e.printStackTrace();
      }
    }

    if (myEnv != null) {
      try {
        myEnv.close();
      } catch (DatabaseException e) {
        System.err.println("closeEnv: " + e.toString());
        e.printStackTrace();
      }
    }
  }
  /** Write data into the database. */
  private void generateData(
      Environment master, int numTxns, Durability durability, boolean doCommit) {

    /* Write some data. */
    DatabaseEntry key = new DatabaseEntry();
    byte[] dataPadding = new byte[1000];
    DatabaseEntry data = new DatabaseEntry(dataPadding);

    TransactionConfig txnConfig = new TransactionConfig();
    txnConfig.setDurability(durability);

    for (int i = 0; i < numTxns; i++) {
      final Transaction txn = master.beginTransaction(null, txnConfig);
      //            long keyPrefix = i << 10;
      //            LongBinding.longToEntry(keyPrefix + i, key);
      LongBinding.longToEntry(i, key);
      db.put(txn, key, data);

      if (doCommit) {
        txn.commit();
      } else {
        txn.abort();
      }
    }
  }
Example #25
0
 public ByteStoreBDB(File dir, String dbname, boolean ro) {
   this.dir = Files.initDirectory(dir);
   this.readonly = ro;
   settings = new SettingsJE();
   EnvironmentConfig bdb_eco = new EnvironmentConfig();
   bdb_eco.setReadOnly(ro);
   bdb_eco.setAllowCreate(!ro);
   bdb_eco.setTransactional(false);
   //          bdb_eco.setDurability(Durability.COMMIT_NO_SYNC);
   if (ro) {
     bdb_eco.setConfigParam(
         EnvironmentConfig.ENV_RUN_CLEANER, "false"); // Disable log cleaner thread
     bdb_eco.setCacheMode(CacheMode.EVICT_LN);
   }
   JEUtil.mergeSystemProperties(bdb_eco);
   SettingsJE.updateEnvironmentConfig(settings, bdb_eco);
   bdb_env = new Environment(dir, bdb_eco);
   bdb_cfg = new DatabaseConfig();
   bdb_cfg.setReadOnly(ro);
   bdb_cfg.setAllowCreate(true);
   bdb_cfg.setDeferredWrite(true);
   SettingsJE.updateDatabaseConfig(settings, bdb_cfg);
   bdb = bdb_env.openDatabase(null, dbname, bdb_cfg);
   if (log.isDebugEnabled()) {
     log.debug(SettingsJE.dumpDebug(bdb));
   }
 }
  /** Get a locker for a read or cursor operation. */
  private static Locker getReadableLocker(
      Environment env, Locker locker, boolean readCommittedIsolation) throws DatabaseException {

    EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env);

    if (locker == null) {
      Transaction xaTxn = env.getThreadTransaction();
      if (xaTxn != null) {
        return DbInternal.getLocker(xaTxn);
      }
    }

    if (locker == null) {
      /* Non-transactional user operations use ThreadLocker. */
      locker = ThreadLocker.createThreadLocker(envImpl);
    } else {

      /*
       * Use the given locker.  For read-committed, wrap the given
       * transactional locker in a special locker for that isolation
       * level.
       */
      if (readCommittedIsolation) {
        locker = ReadCommittedLocker.createReadCommittedLocker(envImpl, locker);
      }
    }
    return locker;
  }
  /**
   * Tests that when a file is opened with a lesser version than the current version, a new log file
   * is started for writing new log entries. This is important so that the new header version is
   * written even if no new log file is needed. If the new version were not written, an older
   * version of JE would not recognize that there had been a version change.
   */
  public void testLesserVersionNotUpdated() throws DatabaseException, IOException {

    TestUtils.loadLog(getClass(), Utils.MIN_VERSION_NAME, envHome);
    File logFile = new File(envHome, TestUtils.LOG_FILE_NAME);
    long origFileSize = logFile.length();

    EnvironmentConfig envConfig = TestUtils.initEnvConfig();
    envConfig.setAllowCreate(false);
    envConfig.setTransactional(true);

    Environment env = new Environment(envHome, envConfig);
    env.sync();
    env.close();

    assertEquals(origFileSize, logFile.length());
  }
    /* Insert 100 records begins with the beginKey. */
    private void doWork(Environment master, String dbName, int beginKey) throws Exception {

      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setAllowCreate(true);
      dbConfig.setTransactional(true);

      /* Insert/Update the records of the database. */
      Database db = master.openDatabase(null, dbName, dbConfig);
      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry data = new DatabaseEntry();

      for (int i = 0; i < 100; i++) {
        IntegerBinding.intToEntry(beginKey + i, key);
        StringBinding.stringToEntry("herococo", data);
        db.put(null, key, data);
      }
      db.close();

      /*
       * Do a sync at the end of the stage to make sure master and
       * replica have the same data set.
       */
      VLSN commitVLSN = RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
      RepTestUtils.checkNodeEquality(commitVLSN, false, repEnvInfo);
    }
Example #29
0
  public void put(WebURL url) throws DatabaseException {

    /*
     * The key that is used for storing URLs determines the order
     * they are crawled. Lower key values results in earlier crawling.
     * Here our keys are 6 bytes. The first byte comes from the URL priority.
     * The second byte comes from depth of crawl at which this URL is first found.
     * The rest of the 4 bytes come from the docid of the URL. As a result,
     * URLs with lower priority numbers will be crawled earlier. If priority
     * numbers are the same, those found at lower depths will be crawled earlier.
     * If depth is also equal, those found earlier (therefore, smaller docid) will
     * be crawled earlier.
     */
    byte[] keyData = new byte[6];
    keyData[0] = url.getPriority();
    keyData[1] = (url.getDepth() > Byte.MAX_VALUE ? Byte.MAX_VALUE : (byte) url.getDepth());
    Util.putIntInByteArray(url.getDocid(), keyData, 2);

    DatabaseEntry value = new DatabaseEntry();
    webURLBinding.objectToEntry(url, value);
    Transaction txn;
    if (resumable) {
      txn = env.beginTransaction(null, null);
    } else {
      txn = null;
    }
    urlsDB.put(txn, new DatabaseEntry(keyData), value);
    if (resumable) {
      if (txn != null) {
        txn.commit();
      }
    }
  }
Example #30
0
  public static void close() {
    long startTs = System.currentTimeMillis();

    try {
      if (database != null) {
        database.close(); // 会调用sync
      }
      env.cleanLog();
      env.close();
    } catch (Exception e) {
      // ingore
      logger.warn("close exception ", e);
    }

    logger.warn("close takes " + (System.currentTimeMillis() - startTs));
  }