Exemplo n.º 1
0
  public void put(WebURL url) throws DatabaseException {

    /*
     * The key that is used for storing URLs determines the order
     * they are crawled. Lower key values results in earlier crawling.
     * Here our keys are 6 bytes. The first byte comes from the URL priority.
     * The second byte comes from depth of crawl at which this URL is first found.
     * The rest of the 4 bytes come from the docid of the URL. As a result,
     * URLs with lower priority numbers will be crawled earlier. If priority
     * numbers are the same, those found at lower depths will be crawled earlier.
     * If depth is also equal, those found earlier (therefore, smaller docid) will
     * be crawled earlier.
     */
    byte[] keyData = new byte[6];
    keyData[0] = url.getPriority();
    keyData[1] = (url.getDepth() > Byte.MAX_VALUE ? Byte.MAX_VALUE : (byte) url.getDepth());
    Util.putIntInByteArray(url.getDocid(), keyData, 2);

    DatabaseEntry value = new DatabaseEntry();
    webURLBinding.objectToEntry(url, value);
    Transaction txn;
    if (resumable) {
      txn = env.beginTransaction(null, null);
    } else {
      txn = null;
    }
    urlsDB.put(txn, new DatabaseEntry(keyData), value);
    if (resumable) {
      if (txn != null) {
        txn.commit();
      }
    }
  }
Exemplo n.º 2
0
  public List<WebURL> get(int max) throws DatabaseException {
    synchronized (mutex) {
      int matches = 0;
      List<WebURL> results = new ArrayList<WebURL>(max);

      Cursor cursor = null;
      OperationStatus result;
      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry value = new DatabaseEntry();
      Transaction txn;
      if (resumable) {
        txn = env.beginTransaction(null, null);
      } else {
        txn = null;
      }
      try {
        cursor = urlsDB.openCursor(txn, null);
        result = cursor.getFirst(key, value, null);

        while (matches < max && result == OperationStatus.SUCCESS) {
          if (value.getData().length > 0) {
            results.add(webURLBinding.entryToObject(value));
            matches++;
          }
          result = cursor.getNext(key, value, null);
        }
      } catch (DatabaseException e) {
        if (txn != null) {
          txn.abort();
          txn = null;
        }
        throw e;
      } finally {
        if (cursor != null) {
          cursor.close();
        }
        if (txn != null) {
          txn.commit();
        }
      }
      return results;
    }
  }
Exemplo n.º 3
0
  public void delete(int count) throws DatabaseException {
    synchronized (mutex) {
      int matches = 0;

      Cursor cursor = null;
      OperationStatus result;
      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry value = new DatabaseEntry();
      Transaction txn;
      if (resumable) {
        txn = env.beginTransaction(null, null);
      } else {
        txn = null;
      }
      try {
        cursor = urlsDB.openCursor(txn, null);
        result = cursor.getFirst(key, value, null);

        while (matches < count && result == OperationStatus.SUCCESS) {
          cursor.delete();
          matches++;
          result = cursor.getNext(key, value, null);
        }
      } catch (DatabaseException e) {
        if (txn != null) {
          txn.abort();
          txn = null;
        }
        throw e;
      } finally {
        if (cursor != null) {
          cursor.close();
        }
        if (txn != null) {
          txn.commit();
        }
      }
    }
  }
  @Test
  public void testOpenReadOnly() throws Throwable {

    try {
      EnvironmentConfig envConfig = TestUtils.initEnvConfig();
      envConfig.setTransactional(true);
      envConfig.setAllowCreate(true);
      env = create(envHome, envConfig);

      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry data = new DatabaseEntry();

      Transaction txn = env.beginTransaction(null, null);
      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setTransactional(true);
      dbConfig.setAllowCreate(true);
      Database myDb = env.openDatabase(txn, "testDB2", dbConfig);

      key.setData(TestUtils.getTestArray(0));
      data.setData(TestUtils.getTestArray(0));
      try {
        myDb.put(txn, key, data);
      } catch (DatabaseException DBE) {
        fail("unexpected DatabaseException during put");
      }

      txn.commit();
      myDb.close();

      dbConfig = new DatabaseConfig();
      dbConfig.setTransactional(true);
      dbConfig.setReadOnly(true);
      txn = env.beginTransaction(null, null);
      myDb = env.openDatabase(txn, "testDB2", dbConfig);
      assertTrue(myDb.isTransactional());
      assertTrue(myDb.getConfig().getTransactional());

      key.setData(TestUtils.getTestArray(0));
      data.setData(TestUtils.getTestArray(0));
      try {
        myDb.put(txn, key, data);
        fail("expected UnsupportedOperationException " + "because open RDONLY");
      } catch (UnsupportedOperationException expected) {
      }

      key.setData(TestUtils.getTestArray(0));
      data.setData(TestUtils.getTestArray(0));
      assertEquals(OperationStatus.SUCCESS, myDb.get(txn, key, data, LockMode.DEFAULT));

      Cursor cursor = myDb.openCursor(txn, null);

      assertEquals(OperationStatus.SUCCESS, cursor.getFirst(key, data, LockMode.DEFAULT));

      try {
        cursor.delete();
        fail("expected Exception from delete on RD_ONLY db");
      } catch (UnsupportedOperationException e) {
      }

      key.setData(TestUtils.getTestArray(1));
      data.setData(TestUtils.getTestArray(1));
      try {
        myDb.put(txn, key, data);
        fail("expected UnsupportedOperationException because open RDONLY");
      } catch (UnsupportedOperationException expected) {
      }

      cursor.close();
      txn.commit();
      myDb.close();

      close(env);
    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    }
  }
  @Test
  public void testIsTransactional() throws Throwable {

    try {
      /* Open environment in transactional mode.*/
      EnvironmentConfig envConfig = TestUtils.initEnvConfig();
      envConfig.setTransactional(true);
      envConfig.setAllowCreate(true);
      env = create(envHome, envConfig);

      /* Create a db, open transactionally with implied auto-commit. */
      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setAllowCreate(true);
      dbConfig.setTransactional(true);
      Database myDb = env.openDatabase(null, "testDB", dbConfig);
      assertTrue(myDb.isTransactional());
      assertTrue(myDb.getConfig().getTransactional());
      myDb.close();

      /* Open an existing db, can open it non-transactionally. */
      dbConfig.setTransactional(false);
      myDb = env.openDatabase(null, "testDB", null);
      assertFalse(myDb.isTransactional());
      assertFalse(myDb.getConfig().getTransactional());
      myDb.close();

      /* Open another db, pass an explicit transaction. */
      dbConfig.setTransactional(true);
      Transaction txn = env.beginTransaction(null, null);
      myDb = env.openDatabase(txn, "testDB2", dbConfig);
      assertTrue(myDb.isTransactional());
      assertTrue(myDb.getConfig().getTransactional());

      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry data = new DatabaseEntry();
      key.setData(TestUtils.getTestArray(0));
      data.setData(TestUtils.getTestArray(0));
      try {
        myDb.put(null, key, data);
      } catch (DatabaseException DBE) {
        fail("didn't expect DatabaseException, implied autocommit");
      }

      key.setData(TestUtils.getTestArray(1));
      data.setData(TestUtils.getTestArray(1));
      try {
        myDb.put(txn, key, data);
      } catch (DatabaseException DBE) {
        fail("didn't expect DatabaseException with txn passed");
      }

      try {
        myDb.get(txn, key, data, LockMode.DEFAULT);
      } catch (DatabaseException DBE) {
        fail("didn't expect DatabaseException with txn passed");
      }

      txn.commit();

      try {
        myDb.get(null, key, data, LockMode.DEFAULT);
      } catch (DatabaseException DBE) {
        fail("didn't expect DatabaseException because no txn passed");
      }

      myDb.close();

      close(env);
    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    }
  }
  /** Test open and close of an environment. */
  public void testCacheStats() throws DatabaseException {

    EnvironmentConfig envConfig = TestUtils.initEnvConfig();
    envConfig.setTransactional(true);
    envConfig.setConfigParam(EnvironmentParams.NODE_MAX.getName(), "6");
    envConfig.setAllowCreate(true);
    env = new Environment(envHome, envConfig);
    EnvironmentStats stat = env.getStats(TestUtils.FAST_STATS);
    env.close();
    env = null;
    assertEquals(0, stat.getNCacheMiss());
    assertEquals(0, stat.getNNotResident());

    // Try to open and close again, now that the environment exists
    envConfig.setAllowCreate(false);
    envConfig.setConfigParam(EnvironmentParams.JE_LOGGING_LEVEL.getName(), "CONFIG");
    env = new Environment(envHome, envConfig);
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setTransactional(true);
    dbConfig.setAllowCreate(true);
    Database db = env.openDatabase(null, "foo", dbConfig);
    db.put(null, new DatabaseEntry(new byte[0]), new DatabaseEntry(new byte[0]));
    Transaction txn = env.beginTransaction(null, null);
    db.put(txn, new DatabaseEntry(new byte[0]), new DatabaseEntry(new byte[0]));
    stat = env.getStats(TestUtils.FAST_STATS);
    MemoryBudget mb = DbInternal.envGetEnvironmentImpl(env).getMemoryBudget();

    assertEquals(mb.getCacheMemoryUsage(), stat.getCacheTotalBytes());
    assertEquals(mb.getLogBufferBudget(), stat.getBufferBytes());
    assertEquals(mb.getTreeMemoryUsage() + mb.getTreeAdminMemoryUsage(), stat.getDataBytes());
    assertEquals(mb.getLockMemoryUsage(), stat.getLockBytes());
    assertEquals(mb.getAdminMemoryUsage(), stat.getAdminBytes());

    assertTrue(stat.getBufferBytes() > 0);
    assertTrue(stat.getDataBytes() > 0);
    assertTrue(stat.getLockBytes() > 0);
    assertTrue(stat.getAdminBytes() > 0);

    assertEquals(
        stat.getCacheTotalBytes(),
        stat.getBufferBytes() + stat.getDataBytes() + stat.getLockBytes() + stat.getAdminBytes());

    assertEquals(12, stat.getNCacheMiss());
    assertEquals(12, stat.getNNotResident());

    /* Test deprecated getCacheDataBytes method. */
    final EnvironmentStats finalStat = stat;
    final long expectCacheDataBytes = mb.getCacheMemoryUsage() - mb.getLogBufferBudget();
    (new Runnable() {
          @Deprecated
          public void run() {
            assertEquals(expectCacheDataBytes, finalStat.getCacheDataBytes());
          }
        })
        .run();

    txn.abort();
    db.close();
    env.close();
    env = null;
  }