예제 #1
0
  /** Check that primary put() writes to each secondary that is open. */
  private void checkSecondaryUpdate(
      Transaction txn,
      Database priDb,
      int val,
      Database secDb,
      boolean expectSecDbVal,
      Database secDb2,
      boolean expectSecDb2Val)
      throws DatabaseException {

    OperationStatus status;
    DatabaseEntry data = new DatabaseEntry();
    int secVal = KEY_OFFSET + val;

    status = priDb.put(txn, entry(val), entry(val));
    assertSame(OperationStatus.SUCCESS, status);

    status = secDb.get(txn, entry(secVal), data, LockMode.DEFAULT);
    assertSame(expectSecDbVal ? OperationStatus.SUCCESS : OperationStatus.NOTFOUND, status);

    status = secDb2.get(txn, entry(secVal), data, LockMode.DEFAULT);
    assertSame(expectSecDb2Val ? OperationStatus.SUCCESS : OperationStatus.NOTFOUND, status);

    status = priDb.delete(txn, entry(val));
    assertSame(OperationStatus.SUCCESS, status);
  }
예제 #2
0
  public void merge() throws Exception {
    LOG.info("start merge");
    Database crawldbDatabase = env.openDatabase(null, "crawldb", BerkeleyDBUtils.defaultDBConfig);
    /*合并fetch库*/
    LOG.info("merge fetch database");
    Database fetchDatabase = env.openDatabase(null, "fetch", BerkeleyDBUtils.defaultDBConfig);
    Cursor fetchCursor = fetchDatabase.openCursor(null, null);
    DatabaseEntry key = new DatabaseEntry();
    DatabaseEntry value = new DatabaseEntry();
    while (fetchCursor.getNext(key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
      crawldbDatabase.put(null, key, value);
    }
    fetchCursor.close();
    fetchDatabase.close();
    /*合并link库*/
    LOG.info("merge link database");
    Database linkDatabase = env.openDatabase(null, "link", BerkeleyDBUtils.defaultDBConfig);
    Cursor linkCursor = linkDatabase.openCursor(null, null);
    while (linkCursor.getNext(key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
      if (!(crawldbDatabase.get(null, key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS)) {
        crawldbDatabase.put(null, key, value);
      }
    }
    linkCursor.close();
    linkDatabase.close();
    LOG.info("end merge");
    crawldbDatabase.sync();
    crawldbDatabase.close();

    env.removeDatabase(null, "fetch");
    LOG.debug("remove fetch database");
    env.removeDatabase(null, "link");
    LOG.debug("remove link database");
  }
 @Override
 public long[] findRela(long gid) {
   DatabaseEntry key = longToEntry(gid);
   DatabaseEntry res = new DatabaseEntry();
   RelationBDB.get(null, key, res, null);
   return entryToAddr(res);
 }
예제 #4
0
  public boolean contains(Transaction txn, K key, LockMode lockMode) throws DatabaseException {

    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry dataEntry = NO_RETURN_ENTRY;
    keyBinding.objectToEntry(key, keyEntry);

    OperationStatus status = db.get(txn, keyEntry, dataEntry, lockMode);
    return (status == OperationStatus.SUCCESS);
  }
예제 #5
0
 @Override
 public byte[] get(byte[] key) {
   final DatabaseEntry dv = new DatabaseEntry();
   byte[] val;
   if (bdb.get(null, new DatabaseEntry(key), dv, lockMode) != opSuccess) {
     return null;
   }
   val = dv.getData();
   bytesIn.addAndGet(key.length + val.length);
   gets.incrementAndGet();
   return val;
 }
 @Override
 public synchronized int getBallot(Long instance) {
   keyBinding.objectToEntry(instance, key);
   Integer ballot = null;
   OperationStatus status = db.get(null, key, ballot_data, LockMode.DEFAULT);
   if (status == OperationStatus.SUCCESS) {
     ballot = ballotBinding.entryToObject(ballot_data);
   }
   if (logger.isDebugEnabled()) {
     logger.debug("DB get ballot " + ballot + " for instance " + instance + " " + status.name());
   }
   return ballot.intValue();
 }
 @Override
 public synchronized Decision getDecision(Long instance) {
   keyBinding.objectToEntry(instance, key);
   Decision decision = null;
   OperationStatus status = db.get(null, key, data, LockMode.DEFAULT);
   if (status == OperationStatus.SUCCESS) {
     decision = dataBinding.entryToObject(data);
   }
   if (logger.isDebugEnabled()) {
     logger.debug("DB get " + decision + " " + status.name());
   }
   return decision;
 }
 @Override
 public synchronized boolean containsDecision(Long instance) {
   boolean found = false;
   keyBinding.objectToEntry(instance, key);
   OperationStatus status = db.get(null, key, data, LockMode.DEFAULT);
   if (status == OperationStatus.SUCCESS) {
     found = true;
   }
   if (logger.isDebugEnabled()) {
     logger.debug("DB contains " + instance + " " + found + " (" + status.name() + ")");
   }
   return found;
 }
예제 #9
0
  public static void main(String[] args) {
    Environment env = null;
    Database db = null;
    EnvironmentConfig envconfig = new EnvironmentConfig();
    envconfig.setAllowCreate(true);
    try {
      env = new Environment(new File("D://bdb"), envconfig);
      DatabaseConfig dbconfig = new DatabaseConfig();
      dbconfig.setAllowCreate(true);
      db = env.openDatabase(null, "dbac.db", dbconfig);
      String key = "mykey";
      DatabaseEntry thekey = new DatabaseEntry();
      thekey.setData(key.getBytes("utf-8"));

      Long value = new Long(123456);
      DatabaseEntry thevalue = new DatabaseEntry();
      EntryBinding myBinging = TupleBinding.getPrimitiveBinding(Long.class);
      myBinging.objectToEntry(value, thevalue);
      // LongBinding myLongBinging=(LongBinding)TupleBinding.getPrimitiveBinding(Long.class);
      // myLongBinging.objectToEntry(value, thevalue);
      db.put(null, thekey, thevalue);
      DatabaseEntry valueEntry = new DatabaseEntry();
      OperationStatus status = db.get(null, thekey, valueEntry, LockMode.DEFAULT);
      if (status == OperationStatus.SUCCESS) {
        // Long number=myLongBinging.entryToObject(valueEntry);
        Long number = (Long) myBinging.entryToObject(valueEntry);
        System.out.println(env.getDatabaseNames());
        System.out.println(number);
      }
    } catch (EnvironmentLockedException e) {
      e.printStackTrace();
    } catch (Exception e) {
      e.printStackTrace();
    } finally {
      if (db != null) {
        try {
          db.close();
        } catch (DatabaseException e) {
          e.printStackTrace();
        }
      }
      if (env != null) {
        try {
          env.cleanLog();
          env.close();
        } catch (DatabaseException e) {
          e.printStackTrace();
        }
      }
    }
  }
예제 #10
0
  /** @see org.geogit.storage.ObjectDatabase#exists(org.geogit.api.ObjectId) */
  @Override
  public boolean exists(final ObjectId id) {
    Preconditions.checkNotNull(id, "id");

    DatabaseEntry key = new DatabaseEntry(id.getRawValue());
    DatabaseEntry data = new DatabaseEntry();
    // tell db not to retrieve data
    data.setPartial(0, 0, true);

    final LockMode lockMode = LockMode.DEFAULT;
    CurrentTransaction.getInstance(env);
    OperationStatus status = objectDb.get(txn.getTransaction(), key, data, lockMode);
    return SUCCESS == status;
  }
예제 #11
0
 public boolean isLocked() throws Exception {
   boolean isLocked = false;
   lockDatabase = env.openDatabase(null, "lock", BerkeleyDBUtils.defaultDBConfig);
   DatabaseEntry key = new DatabaseEntry("lock".getBytes("utf-8"));
   DatabaseEntry value = new DatabaseEntry();
   if (lockDatabase.get(null, key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
     String lockInfo = new String(value.getData(), "utf-8");
     if (lockInfo.equals("locked")) {
       isLocked = true;
     }
   }
   lockDatabase.close();
   return isLocked;
 }
예제 #12
0
  /** @see org.geogit.storage.ObjectDatabase#getRaw(org.geogit.api.ObjectId) */
  @Override
  protected InputStream getRawInternal(final ObjectId id) throws IOException {
    Preconditions.checkNotNull(id, "id");
    DatabaseEntry key = new DatabaseEntry(id.getRawValue());
    DatabaseEntry data = new DatabaseEntry();

    final LockMode lockMode = LockMode.READ_COMMITTED;
    Transaction transaction = txn.getTransaction();
    OperationStatus operationStatus = objectDb.get(transaction, key, data, lockMode);
    if (NOTFOUND.equals(operationStatus)) {
      throw new IllegalArgumentException("Object does not exist: " + id.toString());
    }
    final byte[] cData = data.getData();

    return new ByteArrayInputStream(cData);
  }
  private void readData(Environment env, int lastRecordVal) {
    Database readDb = null;
    DatabaseEntry key = new DatabaseEntry();
    DatabaseEntry data = new DatabaseEntry();

    try {
      readDb = openDatabase(env);

      for (int i = 0; i < lastRecordVal; i++) {
        LongBinding.longToEntry(i, key);
        assertEquals(OperationStatus.SUCCESS, db.get(null, key, data, LockMode.DEFAULT));
      }
    } finally {
      if (readDb != null) {
        readDb.close();
      }
    }
  }
  /*
   * Test that a replica using the jdb files copied from the master can join
   * the group.
   */
  @Test
  public void testCopyEnvJoin() throws Throwable {

    createGroup(1);
    assertTrue(repEnvInfo[0].isMaster());

    /* Create some data on the master. */
    populateDB(repEnvInfo[0].getEnv(), "testDB", 1000);

    /* Close the Environment before copy. */
    repEnvInfo[0].closeEnv();

    /* First check there is no jdb files in the second replica. */
    File repEnvHome = repEnvInfo[1].getEnvHome();
    File[] envFiles = repEnvHome.listFiles();
    for (File envFile : envFiles) {
      if (envFile.getName().contains(".jdb")) {
        throw new IllegalStateException("Replica home should not contain any jdb files");
      }
    }

    /* Copy the jdb files from the master to the replica. */
    SharedTestUtils.copyFiles(repEnvInfo[0].getEnvHome(), repEnvInfo[1].getEnvHome());

    /* Reopen the master. */
    repEnvInfo[0].openEnv();
    assertTrue(repEnvInfo[0].isMaster());

    /* Open the replica. */
    repEnvInfo[1].openEnv();
    assertTrue(repEnvInfo[1].isReplica());

    /* Read the data to make sure data is correctly copied. */
    Database db = repEnvInfo[1].getEnv().openDatabase(null, "testDB", dbconfig);
    for (int i = 0; i < 1000; i++) {
      IntegerBinding.intToEntry(i, key);
      db.get(null, key, data, null);
      assertEquals(i, (int) LongBinding.entryToLong(data));
    }
    db.close();
  }
예제 #15
0
  /**
   * gets object from databse
   *
   * @param key
   * @return
   * @throws Exception
   */
  public Object get(Object key) throws Exception {
    Serializer serializer = new Serializer();
    byte[] keyBytes = serializer.serialize(key);

    final DatabaseEntry keyEntry = new DatabaseEntry(keyBytes);
    Object result;

    final DatabaseEntry dataEntry = new DatabaseEntry();

    final Transaction transaction = myEnv.beginTransaction(null, null);
    final OperationStatus res = myDatabase.get(transaction, keyEntry, dataEntry, null);

    if (res != OperationStatus.SUCCESS) {
      // throw new Exception("Error retrieving from database");
      return null;
    } else {
      result = serializer.deserialize(dataEntry.getData());
    }
    transaction.commit();
    return result;
  }
예제 #16
0
  /** Test that null can be passed for the LockMode to all get methods. */
  public void testNullLockMode() throws DatabaseException {

    SecondaryDatabase secDb = initDb();
    Database priDb = secDb.getPrimaryDatabase();
    Transaction txn = txnBegin();

    DatabaseEntry key = entry(0);
    DatabaseEntry data = entry(0);
    DatabaseEntry secKey = entry(KEY_OFFSET);
    DatabaseEntry found = new DatabaseEntry();
    DatabaseEntry found2 = new DatabaseEntry();
    DatabaseEntry found3 = new DatabaseEntry();

    assertEquals(OperationStatus.SUCCESS, priDb.put(txn, key, data));
    assertEquals(OperationStatus.SUCCESS, priDb.put(txn, entry(1), data));
    assertEquals(OperationStatus.SUCCESS, priDb.put(txn, entry(2), entry(2)));

    /* Database operations. */

    assertEquals(OperationStatus.SUCCESS, priDb.get(txn, key, found, null));
    assertEquals(OperationStatus.SUCCESS, priDb.getSearchBoth(txn, key, data, null));
    assertEquals(OperationStatus.SUCCESS, secDb.get(txn, secKey, found, null));
    assertEquals(OperationStatus.SUCCESS, secDb.get(txn, secKey, found, found2, null));
    assertEquals(OperationStatus.SUCCESS, secDb.getSearchBoth(txn, secKey, key, found, null));

    /* Cursor operations. */

    txnCommit(txn);
    txn = txnBeginCursor();
    Cursor cursor = priDb.openCursor(txn, null);
    SecondaryCursor secCursor = secDb.openSecondaryCursor(txn, null);

    assertEquals(OperationStatus.SUCCESS, cursor.getSearchKey(key, found, null));
    assertEquals(OperationStatus.SUCCESS, cursor.getSearchBoth(key, data, null));
    assertEquals(OperationStatus.SUCCESS, cursor.getSearchKeyRange(key, found, null));
    assertEquals(OperationStatus.SUCCESS, cursor.getSearchBothRange(key, data, null));
    assertEquals(OperationStatus.SUCCESS, cursor.getFirst(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, cursor.getNext(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, cursor.getPrev(found, found2, null));
    assertEquals(OperationStatus.NOTFOUND, cursor.getNextDup(found, found2, null));
    assertEquals(OperationStatus.NOTFOUND, cursor.getPrevDup(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, cursor.getNextNoDup(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, cursor.getPrevNoDup(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, cursor.getLast(found, found2, null));

    assertEquals(OperationStatus.SUCCESS, secCursor.getSearchKey(secKey, found, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getSearchKeyRange(secKey, found, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getFirst(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getNext(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getPrev(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getNextDup(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getPrevDup(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getNextNoDup(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getPrevNoDup(found, found2, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getLast(found, found2, null));

    assertEquals(OperationStatus.SUCCESS, secCursor.getSearchKey(secKey, found, found2, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getSearchBoth(secKey, data, found, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getSearchKeyRange(secKey, found, found2, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getSearchBothRange(secKey, data, found, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getFirst(found, found2, found3, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getNext(found, found2, found3, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getPrev(found, found2, found3, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getNextDup(found, found2, found3, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getPrevDup(found, found2, found3, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getNextNoDup(found, found2, found3, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getPrevNoDup(found, found2, found3, null));
    assertEquals(OperationStatus.SUCCESS, secCursor.getLast(found, found2, found3, null));

    secCursor.close();
    cursor.close();
    txnCommit(txn);
    secDb.close();
    priDb.close();
    env.close();
    env = null;
  }
예제 #17
0
 @Override
 public boolean hasKey(byte[] key) {
   return bdb.get(null, new DatabaseEntry(key), new DatabaseEntry(), lockMode) == opSuccess;
 }
  @Test
  public void testOpenReadOnly() throws Throwable {

    try {
      EnvironmentConfig envConfig = TestUtils.initEnvConfig();
      envConfig.setTransactional(true);
      envConfig.setAllowCreate(true);
      env = create(envHome, envConfig);

      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry data = new DatabaseEntry();

      Transaction txn = env.beginTransaction(null, null);
      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setTransactional(true);
      dbConfig.setAllowCreate(true);
      Database myDb = env.openDatabase(txn, "testDB2", dbConfig);

      key.setData(TestUtils.getTestArray(0));
      data.setData(TestUtils.getTestArray(0));
      try {
        myDb.put(txn, key, data);
      } catch (DatabaseException DBE) {
        fail("unexpected DatabaseException during put");
      }

      txn.commit();
      myDb.close();

      dbConfig = new DatabaseConfig();
      dbConfig.setTransactional(true);
      dbConfig.setReadOnly(true);
      txn = env.beginTransaction(null, null);
      myDb = env.openDatabase(txn, "testDB2", dbConfig);
      assertTrue(myDb.isTransactional());
      assertTrue(myDb.getConfig().getTransactional());

      key.setData(TestUtils.getTestArray(0));
      data.setData(TestUtils.getTestArray(0));
      try {
        myDb.put(txn, key, data);
        fail("expected UnsupportedOperationException " + "because open RDONLY");
      } catch (UnsupportedOperationException expected) {
      }

      key.setData(TestUtils.getTestArray(0));
      data.setData(TestUtils.getTestArray(0));
      assertEquals(OperationStatus.SUCCESS, myDb.get(txn, key, data, LockMode.DEFAULT));

      Cursor cursor = myDb.openCursor(txn, null);

      assertEquals(OperationStatus.SUCCESS, cursor.getFirst(key, data, LockMode.DEFAULT));

      try {
        cursor.delete();
        fail("expected Exception from delete on RD_ONLY db");
      } catch (UnsupportedOperationException e) {
      }

      key.setData(TestUtils.getTestArray(1));
      data.setData(TestUtils.getTestArray(1));
      try {
        myDb.put(txn, key, data);
        fail("expected UnsupportedOperationException because open RDONLY");
      } catch (UnsupportedOperationException expected) {
      }

      cursor.close();
      txn.commit();
      myDb.close();

      close(env);
    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    }
  }
  @Test
  public void testIsTransactional() throws Throwable {

    try {
      /* Open environment in transactional mode.*/
      EnvironmentConfig envConfig = TestUtils.initEnvConfig();
      envConfig.setTransactional(true);
      envConfig.setAllowCreate(true);
      env = create(envHome, envConfig);

      /* Create a db, open transactionally with implied auto-commit. */
      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setAllowCreate(true);
      dbConfig.setTransactional(true);
      Database myDb = env.openDatabase(null, "testDB", dbConfig);
      assertTrue(myDb.isTransactional());
      assertTrue(myDb.getConfig().getTransactional());
      myDb.close();

      /* Open an existing db, can open it non-transactionally. */
      dbConfig.setTransactional(false);
      myDb = env.openDatabase(null, "testDB", null);
      assertFalse(myDb.isTransactional());
      assertFalse(myDb.getConfig().getTransactional());
      myDb.close();

      /* Open another db, pass an explicit transaction. */
      dbConfig.setTransactional(true);
      Transaction txn = env.beginTransaction(null, null);
      myDb = env.openDatabase(txn, "testDB2", dbConfig);
      assertTrue(myDb.isTransactional());
      assertTrue(myDb.getConfig().getTransactional());

      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry data = new DatabaseEntry();
      key.setData(TestUtils.getTestArray(0));
      data.setData(TestUtils.getTestArray(0));
      try {
        myDb.put(null, key, data);
      } catch (DatabaseException DBE) {
        fail("didn't expect DatabaseException, implied autocommit");
      }

      key.setData(TestUtils.getTestArray(1));
      data.setData(TestUtils.getTestArray(1));
      try {
        myDb.put(txn, key, data);
      } catch (DatabaseException DBE) {
        fail("didn't expect DatabaseException with txn passed");
      }

      try {
        myDb.get(txn, key, data, LockMode.DEFAULT);
      } catch (DatabaseException DBE) {
        fail("didn't expect DatabaseException with txn passed");
      }

      txn.commit();

      try {
        myDb.get(null, key, data, LockMode.DEFAULT);
      } catch (DatabaseException DBE) {
        fail("didn't expect DatabaseException because no txn passed");
      }

      myDb.close();

      close(env);
    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    }
  }
  private void experienceLogFlushTask(String sleepTime, boolean flushBeforeCrash) throws Throwable {

    try {
      createRepEnvInfo(sleepTime);

      ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
      long startTime = System.currentTimeMillis();

      StatsConfig stConfig = new StatsConfig();
      stConfig.setClear(true);

      /* Flush the existed dirty data before we do writes. */
      for (int i = 0; i < repEnvInfo.length; i++) {
        repEnvInfo[i].getEnv().sync();
        repEnvInfo[i].getEnv().getStats(stConfig);
      }

      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setAllowCreate(true);
      dbConfig.setTransactional(true);

      Database db = master.openDatabase(null, dbName, dbConfig);

      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry data = new DatabaseEntry();
      for (int i = 1; i <= 100; i++) {
        IntegerBinding.intToEntry(i, key);
        StringBinding.stringToEntry(value, data);
        db.put(null, key, data);
      }

      assertTrue(System.currentTimeMillis() - startTime < 15000);

      Thread.sleep(15000);

      long endTime = System.currentTimeMillis();

      for (int i = 0; i < repEnvInfo.length; i++) {
        EnvironmentStats envStats = repEnvInfo[i].getEnv().getStats(stConfig);
        LogFlusher flusher = repEnvInfo[i].getRepNode().getLogFlusher();
        if (flushBeforeCrash) {
          /* Make sure the LogFlushTask has been invoked. */
          assertTrue(flusher.getFlushTask().scheduledExecutionTime() > startTime);
          assertTrue(flusher.getFlushTask().scheduledExecutionTime() < endTime);

          /*
           * Since the log file size is not so big, we can't assure
           * all the data will be written in the same log file, but
           * we can sure that a flush does happen.
           */
          assertTrue(envStats.getNSequentialWrites() >= 1);
          assertTrue(envStats.getNLogFSyncs() == 1);
        } else {

          /*
           * Make sure the LogFlushTask is not invoked after making
           * the changes.
           */
          assertTrue(flusher.getFlushTask().scheduledExecutionTime() < startTime);
          assertTrue(envStats.getNSequentialWrites() == 0);
          assertTrue(envStats.getNLogFSyncs() == 0);
        }
        assertTrue(envStats.getNFSyncs() == 0);
      }

      File[] envHomes = new File[3];
      /* Close the replicas without doing a checkpoint. */
      for (int i = 0; i < repEnvInfo.length; i++) {
        envHomes[i] = repEnvInfo[i].getEnvHome();
        repEnvInfo[i].getRepImpl().abnormalClose();
      }

      /*
       * Open a read only standalone Environment on the replicas to see
       * whether the data has been synced to the disk.
       */
      EnvironmentConfig newConfig = new EnvironmentConfig();
      newConfig.setAllowCreate(false);
      newConfig.setReadOnly(true);
      newConfig.setTransactional(true);

      for (int i = 0; i < repEnvInfo.length; i++) {
        Environment env = new Environment(envHomes[i], newConfig);

        dbConfig.setAllowCreate(false);
        dbConfig.setReadOnly(true);

        try {
          db = env.openDatabase(null, dbName, dbConfig);
        } catch (DatabaseNotFoundException e) {

          /*
           * If the system crashes before the flush, the database is
           * not synced to the disk, so this database can't be found
           * at all, it's expected.
           */
          assertFalse(flushBeforeCrash);
        }

        if (flushBeforeCrash) {
          assertTrue(db.count() == 100);
          for (int index = 1; index <= 100; index++) {
            IntegerBinding.intToEntry(index, key);
            OperationStatus status = db.get(null, key, data, null);
            if (flushBeforeCrash) {
              assertTrue(status == OperationStatus.SUCCESS);
              assertEquals(value, StringBinding.entryToString(data));
            }
          }
        }

        if (flushBeforeCrash) {
          db.close();
        }
        env.close();
      }
    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    }
  }
  /**
   * Synchronize publications with pubmed using pmid
   *
   * @throws Exception if an error occurs
   */
  public void execute() throws Exception {
    // Needed so that STAX can find it's implementation classes
    ClassLoader cl = Thread.currentThread().getContextClassLoader();

    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());

    Database db = null;
    Transaction txn = null;
    try {
      if (osAlias == null) {
        throw new BuildException("osAlias attribute is not set");
      }
      if (outputFile == null) {
        throw new BuildException("outputFile attribute is not set");
      }

      // environment is transactional
      EnvironmentConfig envConfig = new EnvironmentConfig();
      envConfig.setTransactional(true);
      envConfig.setAllowCreate(true);

      Environment env = new Environment(new File(cacheDirName), envConfig);

      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setTransactional(true);
      dbConfig.setAllowCreate(true);
      dbConfig.setSortedDuplicates(true);

      db = env.openDatabase(null, "publications_db", dbConfig);

      txn = env.beginTransaction(null, null);

      LOG.info("Starting EntrezPublicationsRetriever");

      Writer writer = new FileWriter(outputFile);
      ObjectStore os = ObjectStoreFactory.getObjectStore(osAlias);

      Set<Integer> idsToFetch = new HashSet<Integer>();
      itemFactory = new ItemFactory(os.getModel(), "-1_");
      writer.write(FullRenderer.getHeader() + ENDL);
      for (Iterator<Publication> iter = getPublications(os).iterator(); iter.hasNext(); ) {
        String pubMedId = iter.next().getPubMedId();
        Integer pubMedIdInteger;
        try {
          pubMedIdInteger = Integer.valueOf(pubMedId);
        } catch (NumberFormatException e) {
          // not a pubmed id
          continue;
        }

        if (seenPubMeds.contains(pubMedIdInteger)) {
          continue;
        }
        DatabaseEntry key = new DatabaseEntry(pubMedId.getBytes());
        DatabaseEntry data = new DatabaseEntry();
        if (db.get(txn, key, data, null).equals(OperationStatus.SUCCESS)) {
          try {
            ByteArrayInputStream mapInputStream = new ByteArrayInputStream(data.getData());
            ObjectInputStream deserializer = new ObjectInputStream(mapInputStream);
            Map<String, Object> pubMap = (Map) deserializer.readObject();
            writeItems(writer, mapToItems(itemFactory, pubMap));
            seenPubMeds.add(pubMedIdInteger);
          } catch (EOFException e) {
            // ignore and fetch it again
            System.err.println(
                "found in cache, but igored due to cache problem: " + pubMedIdInteger);
          }
        } else {
          idsToFetch.add(pubMedIdInteger);
        }
      }

      Iterator<Integer> idIter = idsToFetch.iterator();
      Set<Integer> thisBatch = new HashSet<Integer>();
      while (idIter.hasNext()) {
        Integer pubMedIdInteger = idIter.next();
        thisBatch.add(pubMedIdInteger);
        if (thisBatch.size() == BATCH_SIZE || !idIter.hasNext() && thisBatch.size() > 0) {
          try {
            // the server may return less publications than we ask for, so keep a Map
            Map<String, Map<String, Object>> fromServerMap = null;

            for (int i = 0; i < MAX_TRIES; i++) {
              BufferedReader br = new BufferedReader(getReader(thisBatch));
              StringBuffer buf = new StringBuffer();
              String line;
              while ((line = br.readLine()) != null) {
                buf.append(line + "\n");
              }
              fromServerMap = new HashMap<String, Map<String, Object>>();
              Throwable throwable = null;
              try {
                if (loadFullRecord) {
                  SAXParser.parse(
                      new InputSource(new StringReader(buf.toString())),
                      new FullRecordHandler(fromServerMap),
                      false);
                } else {
                  SAXParser.parse(
                      new InputSource(new StringReader(buf.toString())),
                      new SummaryRecordHandler(fromServerMap),
                      false);
                }
              } catch (Throwable e) {
                LOG.error("Couldn't parse PubMed XML", e);
                // try again or re-throw the Throwable
                throwable = e;
              }
              if (i == MAX_TRIES) {
                throw new RuntimeException(
                    "failed to parse: " + buf.toString() + " - tried " + MAX_TRIES + " times",
                    throwable);
              } else {
                if (throwable != null) {
                  // try again
                  continue;
                }
              }

              for (String id : fromServerMap.keySet()) {
                writeItems(writer, mapToItems(itemFactory, fromServerMap.get(id)));
              }
              addToDb(txn, db, fromServerMap);
              break;
            }
            thisBatch.clear();
          } finally {
            txn.commit();
            // start a new transaction incase there is an exception while parsing
            txn = env.beginTransaction(null, null);
          }
        }
      }
      writeItems(writer, authorMap.values());
      writeItems(writer, meshTerms.values());
      writer.write(FullRenderer.getFooter() + ENDL);
      writer.flush();
      writer.close();
    } catch (Throwable e) {
      throw new RuntimeException("failed to get all publications", e);
    } finally {
      txn.commit();
      db.close();
      Thread.currentThread().setContextClassLoader(cl);
    }
  }
예제 #22
0
  public void testPutAndDelete() throws DatabaseException {

    SecondaryDatabase secDb = initDb();
    Database priDb = secDb.getPrimaryDatabase();

    DatabaseEntry data = new DatabaseEntry();
    DatabaseEntry key = new DatabaseEntry();
    OperationStatus status;
    Transaction txn = txnBegin();

    /* Database.put() */
    status = priDb.put(txn, entry(1), entry(2));
    assertSame(OperationStatus.SUCCESS, status);
    status = secDb.get(txn, entry(102), key, data, LockMode.DEFAULT);
    assertSame(OperationStatus.SUCCESS, status);
    assertDataEquals(entry(1), key);
    assertDataEquals(entry(2), data);

    /* Database.putNoOverwrite() */
    status = priDb.putNoOverwrite(txn, entry(1), entry(1));
    assertSame(OperationStatus.KEYEXIST, status);
    status = secDb.get(txn, entry(102), key, data, LockMode.DEFAULT);
    assertSame(OperationStatus.SUCCESS, status);
    assertDataEquals(entry(1), key);
    assertDataEquals(entry(2), data);

    /* Database.put() overwrite */
    status = priDb.put(txn, entry(1), entry(3));
    assertSame(OperationStatus.SUCCESS, status);
    status = secDb.get(txn, entry(102), key, data, LockMode.DEFAULT);
    assertSame(OperationStatus.NOTFOUND, status);
    status = secDb.get(txn, entry(103), key, data, LockMode.DEFAULT);
    assertSame(OperationStatus.SUCCESS, status);
    assertDataEquals(entry(1), key);
    assertDataEquals(entry(3), data);

    /* Database.delete() */
    status = priDb.delete(txn, entry(1));
    assertSame(OperationStatus.SUCCESS, status);
    status = priDb.delete(txn, entry(1));
    assertSame(OperationStatus.NOTFOUND, status);
    status = secDb.get(txn, entry(103), key, data, LockMode.DEFAULT);
    assertSame(OperationStatus.NOTFOUND, status);

    /* SecondaryDatabase.delete() */
    status = priDb.put(txn, entry(1), entry(1));
    assertSame(OperationStatus.SUCCESS, status);
    status = priDb.put(txn, entry(2), entry(1));
    assertSame(OperationStatus.SUCCESS, status);
    status = secDb.get(txn, entry(101), key, data, LockMode.DEFAULT);
    assertSame(OperationStatus.SUCCESS, status);
    assertDataEquals(entry(1), key);
    assertDataEquals(entry(1), data);
    status = secDb.delete(txn, entry(101));
    assertSame(OperationStatus.SUCCESS, status);
    status = secDb.delete(txn, entry(101));
    assertSame(OperationStatus.NOTFOUND, status);
    status = secDb.get(txn, entry(101), key, data, LockMode.DEFAULT);
    assertSame(OperationStatus.NOTFOUND, status);
    status = priDb.get(txn, entry(1), data, LockMode.DEFAULT);
    assertSame(OperationStatus.NOTFOUND, status);
    status = priDb.get(txn, entry(2), data, LockMode.DEFAULT);
    assertSame(OperationStatus.NOTFOUND, status);

    /*
     * Database.putNoDupData() cannot be called since the primary cannot be
     * configured for duplicates.
     */

    /* Primary and secondary are empty now. */

    /* Get a txn for a cursor. */
    txnCommit(txn);
    txn = txnBeginCursor();

    Cursor priCursor = null;
    SecondaryCursor secCursor = null;
    try {
      priCursor = priDb.openCursor(txn, null);
      secCursor = secDb.openSecondaryCursor(txn, null);

      /* Cursor.putNoOverwrite() */
      status = priCursor.putNoOverwrite(entry(1), entry(2));
      assertSame(OperationStatus.SUCCESS, status);
      status = secCursor.getSearchKey(entry(102), key, data, LockMode.DEFAULT);
      assertSame(OperationStatus.SUCCESS, status);
      assertDataEquals(entry(1), key);
      assertDataEquals(entry(2), data);

      /* Cursor.putCurrent() */
      status = priCursor.putCurrent(entry(3));
      assertSame(OperationStatus.SUCCESS, status);
      status = secCursor.getSearchKey(entry(102), key, data, LockMode.DEFAULT);
      assertSame(OperationStatus.NOTFOUND, status);
      status = secCursor.getSearchKey(entry(103), key, data, LockMode.DEFAULT);
      assertSame(OperationStatus.SUCCESS, status);
      assertDataEquals(entry(1), key);
      assertDataEquals(entry(3), data);

      /* Cursor.delete() */
      status = priCursor.delete();
      assertSame(OperationStatus.SUCCESS, status);
      status = priCursor.delete();
      assertSame(OperationStatus.KEYEMPTY, status);
      status = secCursor.getSearchKey(entry(103), key, data, LockMode.DEFAULT);
      assertSame(OperationStatus.NOTFOUND, status);
      status = priCursor.getSearchKey(entry(1), data, LockMode.DEFAULT);
      assertSame(OperationStatus.NOTFOUND, status);

      /* Cursor.put() */
      status = priCursor.put(entry(1), entry(4));
      assertSame(OperationStatus.SUCCESS, status);
      status = secCursor.getSearchKey(entry(104), key, data, LockMode.DEFAULT);
      assertSame(OperationStatus.SUCCESS, status);
      assertDataEquals(entry(1), key);
      assertDataEquals(entry(4), data);

      /* SecondaryCursor.delete() */
      status = secCursor.delete();
      assertSame(OperationStatus.SUCCESS, status);
      status = secCursor.delete();
      assertSame(OperationStatus.KEYEMPTY, status);
      status = secCursor.getCurrent(new DatabaseEntry(), key, data, LockMode.DEFAULT);
      assertSame(OperationStatus.KEYEMPTY, status);
      status = secCursor.getSearchKey(entry(104), key, data, LockMode.DEFAULT);
      assertSame(OperationStatus.NOTFOUND, status);
      status = priCursor.getSearchKey(entry(1), data, LockMode.DEFAULT);
      assertSame(OperationStatus.NOTFOUND, status);

      /*
       * Cursor.putNoDupData() cannot be called since the primary cannot
       * be configured for duplicates.
       */

      /* Primary and secondary are empty now. */
    } finally {
      if (secCursor != null) {
        secCursor.close();
      }
      if (priCursor != null) {
        priCursor.close();
      }
    }

    txnCommit(txn);
    secDb.close();
    priDb.close();
  }