Пример #1
0
  public void put(WebURL url) throws DatabaseException {

    /*
     * The key that is used for storing URLs determines the order
     * they are crawled. Lower key values results in earlier crawling.
     * Here our keys are 6 bytes. The first byte comes from the URL priority.
     * The second byte comes from depth of crawl at which this URL is first found.
     * The rest of the 4 bytes come from the docid of the URL. As a result,
     * URLs with lower priority numbers will be crawled earlier. If priority
     * numbers are the same, those found at lower depths will be crawled earlier.
     * If depth is also equal, those found earlier (therefore, smaller docid) will
     * be crawled earlier.
     */
    byte[] keyData = new byte[6];
    keyData[0] = url.getPriority();
    keyData[1] = (url.getDepth() > Byte.MAX_VALUE ? Byte.MAX_VALUE : (byte) url.getDepth());
    Util.putIntInByteArray(url.getDocid(), keyData, 2);

    DatabaseEntry value = new DatabaseEntry();
    webURLBinding.objectToEntry(url, value);
    Transaction txn;
    if (resumable) {
      txn = env.beginTransaction(null, null);
    } else {
      txn = null;
    }
    urlsDB.put(txn, new DatabaseEntry(keyData), value);
    if (resumable) {
      if (txn != null) {
        txn.commit();
      }
    }
  }
  public void testDuplicatesWithDeletion() throws Throwable {

    createEnvAndDbs(1 << 20, true, NUM_DBS);
    int numRecs = 10;
    int nDups = N_DUPLICATES_PER_KEY;

    try {
      /* Set up an repository of expected data. */
      Map<TestData, Set<TestData>> expectedData = new HashMap<TestData, Set<TestData>>();

      /* Insert all the data. */
      Transaction txn = env.beginTransaction(null, null);
      insertData(txn, 0, numRecs - 1, expectedData, nDups, true, NUM_DBS);

      /* Delete all the even records. */
      deleteData(txn, expectedData, false, true, NUM_DBS);
      txn.commit();

      /* Modify all the records. */
      //    modifyData(expectedData);

      closeEnv();

      recoverAndVerify(expectedData, NUM_DBS);
    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    }
  }
  /** Write data into the database. */
  private void generateData(
      Environment master, int numTxns, Durability durability, boolean doCommit) {

    /* Write some data. */
    DatabaseEntry key = new DatabaseEntry();
    byte[] dataPadding = new byte[1000];
    DatabaseEntry data = new DatabaseEntry(dataPadding);

    TransactionConfig txnConfig = new TransactionConfig();
    txnConfig.setDurability(durability);

    for (int i = 0; i < numTxns; i++) {
      final Transaction txn = master.beginTransaction(null, txnConfig);
      //            long keyPrefix = i << 10;
      //            LongBinding.longToEntry(keyPrefix + i, key);
      LongBinding.longToEntry(i, key);
      db.put(txn, key, data);

      if (doCommit) {
        txn.commit();
      } else {
        txn.abort();
      }
    }
  }
  public V remove(K key, V value) throws Exception {
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry foundEntry = new DatabaseEntry();
    keyBinding.objectToEntry(key, keyEntry);

    V ret = null;

    Transaction txn = env.beginTransaction(null, null);
    Cursor cursor = db.openCursor(txn, null);

    if ((cursor.getSearchKey(keyEntry, foundEntry, LockMode.DEFAULT)) == OperationStatus.SUCCESS) {
      do {
        V v = (V) dataBinding.entryToObject(foundEntry);
        if (value.equals(v)) {
          ret = v;
          cursor.delete();
          break;
        }
      } while ((cursor.getNextDup(keyEntry, foundEntry, LockMode.DEFAULT))
          == OperationStatus.SUCCESS);
    }
    // Cannot use Cursor#getSearchBoth() because it compare values byte-wise without Object#equal().

    cursor.close();
    txn.commit();

    return ret;
  }
Пример #5
0
  @Override
  public String saveDocument(
      DsSessionDto aSessionDto,
      DsDocumentDto aDocumentDto,
      String aKey,
      boolean aKeyIsNumeric,
      String aValue,
      String aSecurityPolicyName,
      String aContentFileKey) {

    if ("bug".equals(aSessionDto.getUsername())) {
      throw new IllegalStateException("this is a bug");
    }

    Transaction tx = entityStore.getEnvironment().beginTransaction(null, null);

    Document doc = new Document();
    doc.setFileKey(aContentFileKey);

    doc.setFileName(getFileName(aDocumentDto));
    documentPrimaryIndex.put(doc);
    tx.commit();

    return String.valueOf(doc.getId());
  }
Пример #6
0
 private void attemptCommit(Transaction transaction) {
   try {
     transaction.commit();
   } catch (DatabaseException e) {
     logger.error("Transaction commit failed!", e);
     attemptAbort(transaction);
     throw new PersistenceFailureException(e);
   }
 }
Пример #7
0
  @Override
  public void deleteDocument(DsSessionDto aSessionDto, String aDocumentId) {

    if ("bug".equals(aSessionDto.getUsername())) {
      throw new IllegalStateException("this is a bug");
    }

    Transaction tx = entityStore.getEnvironment().beginTransaction(null, null);
    documentPrimaryIndex.delete(Long.valueOf(aDocumentId));
    tx.commit();
  }
Пример #8
0
 /** {@inheritDoc} */
 public void commit() {
   try {
     if (xid != null) {
       env.commit(xid, true /* ignored */);
     } else {
       txn.commit();
     }
   } catch (DatabaseException e) {
     throw JeEnvironment.convertException(e, false);
   } catch (XAException e) {
     throw JeEnvironment.convertException(e, false);
   }
 }
  /*
   * See SR11455 for details.
   *
   * This test is checking that the maxTxnId gets recovered properly during
   * recovery.  The SR has to do with the INFileReader not including
   * DupCountLN_TX and DelDupLN_TX's in its txnIdTrackingMap.  When these
   * were not included, it was possible for a transaction to consist solely
   * of DupCountLN_TX/DelDupLN_TX pairs.  The "deleteData" transaction below
   * does just this.  If no checkpoint occurred following such a transaction,
   * then the correct current txnid would not be written to the log and
   * determining this value during recovery would be left up to the
   * INFileReader.  However, without reading the DupCountLN_TX/DelDupLN_TX
   * records, it would not recover the correct value.
   *
   * We take the poor man's way out of creating this situation by just
   * manually asserting the txn id is correct post-recovery.  The txnid of 12
   * was determined by looking through logs before and after the fix.
   */
  public void testSR11455() throws Throwable {

    createEnvAndDbs(1 << 20, true, 1);
    int numRecs = 1;
    int nDups = 3;

    try {
      /* Set up an repository of expected data. */
      Map<TestData, Set<TestData>> expectedData = new HashMap<TestData, Set<TestData>>();

      /* Insert all the data. */
      Transaction txn = env.beginTransaction(null, null);
      insertData(txn, 0, numRecs - 1, expectedData, nDups, true, 1);
      txn.commit();

      txn = env.beginTransaction(null, null);
      /* Delete all the even records. */
      deleteData(txn, expectedData, false, false, 1);
      txn.abort();
      closeEnv();

      /* Open it again, which will run recovery. */
      EnvironmentConfig recoveryConfig = TestUtils.initEnvConfig();
      recoveryConfig.setTransactional(true);
      recoveryConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
      recoveryConfig.setConfigParam(EnvironmentParams.ENV_RUN_EVICTOR.getName(), "false");
      env = new Environment(envHome, recoveryConfig);

      txn = env.beginTransaction(null, null);
      assertEquals(6, txn.getId());
      txn.commit();
      env.close();

    } catch (Throwable t) {
      t.printStackTrace();
      throw t;
    }
  }
Пример #10
0
 @Override
 public synchronized void commit() {
   try {
     if (!openedCursors.isEmpty()) {
       closeOpenedCursors();
       throw new BerkeleyDBPersistenceOpenedCursorsException();
     }
     dbTransaction.commit();
     getPersistenceManager().unregisterTransaction(this);
     super.commit();
   } catch (DatabaseException e) {
     throw getPersistenceManager().convertDatabaseException(e);
   }
 }
Пример #11
0
  /**
   * delete from the database
   *
   * @param key
   * @throws Exception
   */
  public void delete(Object key) throws Exception {
    Serializer serializer = new Serializer();
    byte[] keyBytes = serializer.serialize(key);

    final DatabaseEntry keyEntry = new DatabaseEntry(keyBytes);

    final Transaction transaction = myEnv.beginTransaction(null, null);
    final OperationStatus res = myDatabase.delete(transaction, keyEntry);

    if (res != OperationStatus.SUCCESS) {
      throw new Exception("Error retrieving from database");
    }

    transaction.commit();
    return;
  }
  public V put(K key, V value) throws Exception {
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry dataEntry = new DatabaseEntry();
    keyBinding.objectToEntry(key, keyEntry);
    dataBinding.objectToEntry(value, dataEntry);

    V ret = null;

    Transaction txn = env.beginTransaction(null, null);
    Cursor cursor = db.openCursor(txn, null);

    if ((cursor.getSearchKey(keyEntry, dataEntry, LockMode.DEFAULT)) == OperationStatus.SUCCESS) {
      V v = (V) dataBinding.entryToObject(dataEntry);
      if (value.equals(v)) { // found
        ret = v;
        cursor.delete(); // remove here. and put later.
      } else {
        while ((cursor.getNextDup(keyEntry, dataEntry, LockMode.DEFAULT))
            == OperationStatus.SUCCESS) {
          v = (V) dataBinding.entryToObject(dataEntry);
          if (value.equals(v)) { // found
            ret = v;
            cursor.delete(); // remove here. and put later.
            break;
          }
        }
      }
    }

    // put
    keyBinding.objectToEntry(key, keyEntry);
    dataBinding.objectToEntry(value, dataEntry);

    if ((cursor.put(keyEntry, dataEntry)) != OperationStatus.SUCCESS) {
      String msg = "Could not put: " + key + ", " + value;
      logger.log(Level.WARNING, msg);
      throw new DatabaseException(msg);
    }

    cursor.close();
    txn.commit();

    return ret;
  }
Пример #13
0
  public List<WebURL> get(int max) throws DatabaseException {
    synchronized (mutex) {
      int matches = 0;
      List<WebURL> results = new ArrayList<WebURL>(max);

      Cursor cursor = null;
      OperationStatus result;
      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry value = new DatabaseEntry();
      Transaction txn;
      if (resumable) {
        txn = env.beginTransaction(null, null);
      } else {
        txn = null;
      }
      try {
        cursor = urlsDB.openCursor(txn, null);
        result = cursor.getFirst(key, value, null);

        while (matches < max && result == OperationStatus.SUCCESS) {
          if (value.getData().length > 0) {
            results.add(webURLBinding.entryToObject(value));
            matches++;
          }
          result = cursor.getNext(key, value, null);
        }
      } catch (DatabaseException e) {
        if (txn != null) {
          txn.abort();
          txn = null;
        }
        throw e;
      } finally {
        if (cursor != null) {
          cursor.close();
        }
        if (txn != null) {
          txn.commit();
        }
      }
      return results;
    }
  }
  private void testMemberRemoveAckInteraction(final boolean delete) {
    createGroup(groupSize);
    Transaction txn;
    Database db;
    try {
      MasterTxn.setFactory(new TxnFactory(delete));
      ReplicatedEnvironment master = repEnvInfo[0].getEnv();

      txn = master.beginTransaction(null, null);
      /* Write to the environment. */
      db = master.openDatabase(txn, "random", dbconfig);
      db.close();
      txn.commit();
    } catch (InsufficientAcksException e) {
      fail("No exception expected.");
    } finally {
      MasterTxn.setFactory(null);
    }
  }
Пример #15
0
  /**
   * gets object from databse
   *
   * @param key
   * @return
   * @throws Exception
   */
  public Object get(Object key) throws Exception {
    Serializer serializer = new Serializer();
    byte[] keyBytes = serializer.serialize(key);

    final DatabaseEntry keyEntry = new DatabaseEntry(keyBytes);
    Object result;

    final DatabaseEntry dataEntry = new DatabaseEntry();

    final Transaction transaction = myEnv.beginTransaction(null, null);
    final OperationStatus res = myDatabase.get(transaction, keyEntry, dataEntry, null);

    if (res != OperationStatus.SUCCESS) {
      // throw new Exception("Error retrieving from database");
      return null;
    } else {
      result = serializer.deserialize(dataEntry.getData());
    }
    transaction.commit();
    return result;
  }
Пример #16
0
  /**
   * inserts into databse
   *
   * @param key
   * @param data
   * @throws Exception
   */
  public void insert(Object key, Object data) throws Exception {
    Serializer serializer = new Serializer();
    byte[] keyBytes = serializer.serialize(key);
    byte[] dataBytes = serializer.serialize(data);

    final DatabaseEntry keyEntry = new DatabaseEntry(keyBytes);
    final DatabaseEntry dataEntry = new DatabaseEntry(dataBytes);

    try {
      final Transaction transaction = myEnv.beginTransaction(null, null);
      final OperationStatus result = myDatabase.put(transaction, keyEntry, dataEntry);
      if (result != OperationStatus.SUCCESS) {
        System.out.println("operation status-failure");
        throw new Exception("Error");
      }
      transaction.commit();

    } catch (Exception DE) {
      System.out.println(DE);
    }
  }
Пример #17
0
  public void delete(int count) throws DatabaseException {
    synchronized (mutex) {
      int matches = 0;

      Cursor cursor = null;
      OperationStatus result;
      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry value = new DatabaseEntry();
      Transaction txn;
      if (resumable) {
        txn = env.beginTransaction(null, null);
      } else {
        txn = null;
      }
      try {
        cursor = urlsDB.openCursor(txn, null);
        result = cursor.getFirst(key, value, null);

        while (matches < count && result == OperationStatus.SUCCESS) {
          cursor.delete();
          matches++;
          result = cursor.getNext(key, value, null);
        }
      } catch (DatabaseException e) {
        if (txn != null) {
          txn.abort();
          txn = null;
        }
        throw e;
      } finally {
        if (cursor != null) {
          cursor.close();
        }
        if (txn != null) {
          txn.commit();
        }
      }
    }
  }
Пример #18
0
 @Override
 public synchronized boolean trim(Long instance) {
   if (instance == 0) {
     return true;
   } // fast track
   Transaction t = null;
   if (db.getConfig().getTransactional()) {
     t = env.beginTransaction(null, null);
   }
   Cursor cursor = db.openCursor(t, null);
   boolean dirty = false;
   try {
     while (cursor.getNext(key, data, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
       Long i = keyBinding.entryToObject(key);
       if (i < instance && cursor.delete() != OperationStatus.SUCCESS) {
         logger.error("Error deleting instance " + i + " from DB!");
         dirty = true;
       }
     }
   } finally {
     cursor.close();
     if (!dirty) {
       if (t != null) {
         t.commit();
       }
     } else {
       if (t != null) {
         t.abort();
       }
       return false;
     }
   }
   putDecision(-1L, new Decision(0, instance, 0, null));
   logger.debug("DB deltete up to instance " + instance);
   return true;
 }
  private Set<V> getAndRemove(K key, boolean remove) throws DatabaseException {
    DatabaseEntry searchKey = new DatabaseEntry();
    keyBinding.objectToEntry(key, searchKey);
    DatabaseEntry foundKey = new DatabaseEntry();
    DatabaseEntry foundData = new DatabaseEntry();

    Transaction txn = env.beginTransaction(null, null);
    Cursor cursor = db.openCursor(txn, null);

    Set<V> s = null;
    if ((cursor.getSearchKey(searchKey, foundData, LockMode.DEFAULT)) == OperationStatus.SUCCESS) {
      s = new HashSet<V>();
      s.add((V) dataBinding.entryToObject(foundData));

      while ((cursor.getNextDup(foundKey, foundData, LockMode.DEFAULT))
          == OperationStatus.SUCCESS) {
        //				K k = (K)keyBinding.entryToObject(foundKey);
        //				if (!key.equals(k)) {
        //					break;
        //				}

        s.add((V) dataBinding.entryToObject(foundData));
      }
    }

    // remove all entries associated with the given key
    if (remove) {
      db.delete(txn, searchKey);
      // does not check the result of this operation.
    }

    cursor.close();
    txn.commit();

    return s;
  }
Пример #20
0
 /** removes database */
 public void removeDatabase() {
   final Transaction txn = myEnv.beginTransaction(null, null);
   // close();
   myEnv.removeDatabase(txn, dbName);
   txn.commit();
 }
Пример #21
0
  public Store(Environment env, String storeName, StoreConfig config, boolean rawAccess)
      throws DatabaseException {

    this.env = env;
    this.storeName = storeName;
    this.rawAccess = rawAccess;

    if (env == null || storeName == null) {
      throw new NullPointerException("env and storeName parameters must not be null");
    }
    if (config != null) {
      model = config.getModel();
      mutations = config.getMutations();
    }
    if (config == null) {
      storeConfig = StoreConfig.DEFAULT;
    } else {
      storeConfig = config.cloneConfig();
    }

    storePrefix = NAME_PREFIX + storeName + NAME_SEPARATOR;
    priIndexMap = new HashMap<String, PrimaryIndex>();
    secIndexMap = new HashMap<String, SecondaryIndex>();
    priConfigMap = new HashMap<String, DatabaseConfig>();
    secConfigMap = new HashMap<String, SecondaryConfig>();
    keyBindingMap = new HashMap<String, PersistKeyBinding>();
    sequenceMap = new HashMap<String, Sequence>();
    sequenceConfigMap = new HashMap<String, SequenceConfig>();
    deferredWriteDatabases = new IdentityHashMap<Database, Object>();

    if (rawAccess) {
      /* Open a read-only catalog that uses the stored model. */
      if (model != null) {
        throw new IllegalArgumentException("A model may not be specified when opening a RawStore");
      }
      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setReadOnly(true);
      dbConfig.setTransactional(storeConfig.getTransactional());
      catalog =
          new PersistCatalog(
              null,
              env,
              storePrefix,
              storePrefix + CATALOG_DB,
              dbConfig,
              model,
              mutations,
              rawAccess,
              this);
    } else {
      /* Open the shared catalog that uses the current model. */
      synchronized (catalogPool) {
        Map<String, PersistCatalog> catalogMap = catalogPool.get(env);
        if (catalogMap == null) {
          catalogMap = new HashMap<String, PersistCatalog>();
          catalogPool.put(env, catalogMap);
        }
        catalog = catalogMap.get(storeName);
        if (catalog != null) {
          catalog.openExisting();
        } else {
          Transaction txn = null;
          if (storeConfig.getTransactional() && env.getThreadTransaction() == null) {
            txn = env.beginTransaction(null, null);
          }
          boolean success = false;
          try {
            DatabaseConfig dbConfig = new DatabaseConfig();
            dbConfig.setAllowCreate(storeConfig.getAllowCreate());
            dbConfig.setReadOnly(storeConfig.getReadOnly());
            dbConfig.setTransactional(storeConfig.getTransactional());
            catalog =
                new PersistCatalog(
                    txn,
                    env,
                    storePrefix,
                    storePrefix + CATALOG_DB,
                    dbConfig,
                    model,
                    mutations,
                    rawAccess,
                    this);
            catalogMap.put(storeName, catalog);
            success = true;
          } finally {
            if (txn != null) {
              if (success) {
                txn.commit();
              } else {
                txn.abort();
              }
            }
          }
        }
      }
    }

    /* Get the merged mutations from the catalog. */
    mutations = catalog.getMutations();

    /*
     * If there is no model parameter, use the default or stored model
     * obtained from the catalog.
     */
    model = catalog.getResolvedModel();

    /*
     * Give the model a reference to the catalog to fully initialize the
     * model.  Only then may we initialize the Converter mutations, which
     * themselves may call model methods and expect the model to be fully
     * initialized.
     */
    ModelInternal.setCatalog(model, catalog);
    for (Converter converter : mutations.getConverters()) {
      converter.getConversion().initialize(model);
    }

    /*
     * For each existing entity with a relatedEntity reference, create an
     * inverse map (back pointer) from the class named in the relatedEntity
     * to the class containing the secondary key.  This is used to open the
     * class containing the secondary key whenever we open the
     * relatedEntity class, to configure foreign key constraints. Note that
     * we do not need to update this map as new primary indexes are
     * created, because opening the new index will setup the foreign key
     * constraints. [#15358]
     */
    inverseRelatedEntityMap = new HashMap<String, Set<String>>();
    List<Format> entityFormats = new ArrayList<Format>();
    catalog.getEntityFormats(entityFormats);
    for (Format entityFormat : entityFormats) {
      EntityMetadata entityMeta = entityFormat.getEntityMetadata();
      for (SecondaryKeyMetadata secKeyMeta : entityMeta.getSecondaryKeys().values()) {
        String relatedClsName = secKeyMeta.getRelatedEntity();
        if (relatedClsName != null) {
          Set<String> inverseClassNames = inverseRelatedEntityMap.get(relatedClsName);
          if (inverseClassNames == null) {
            inverseClassNames = new HashSet<String>();
            inverseRelatedEntityMap.put(relatedClsName, inverseClassNames);
          }
          inverseClassNames.add(entityMeta.getClassName());
        }
      }
    }
  }
  /**
   * Synchronize publications with pubmed using pmid
   *
   * @throws Exception if an error occurs
   */
  public void execute() throws Exception {
    // Needed so that STAX can find it's implementation classes
    ClassLoader cl = Thread.currentThread().getContextClassLoader();

    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());

    Database db = null;
    Transaction txn = null;
    try {
      if (osAlias == null) {
        throw new BuildException("osAlias attribute is not set");
      }
      if (outputFile == null) {
        throw new BuildException("outputFile attribute is not set");
      }

      // environment is transactional
      EnvironmentConfig envConfig = new EnvironmentConfig();
      envConfig.setTransactional(true);
      envConfig.setAllowCreate(true);

      Environment env = new Environment(new File(cacheDirName), envConfig);

      DatabaseConfig dbConfig = new DatabaseConfig();
      dbConfig.setTransactional(true);
      dbConfig.setAllowCreate(true);
      dbConfig.setSortedDuplicates(true);

      db = env.openDatabase(null, "publications_db", dbConfig);

      txn = env.beginTransaction(null, null);

      LOG.info("Starting EntrezPublicationsRetriever");

      Writer writer = new FileWriter(outputFile);
      ObjectStore os = ObjectStoreFactory.getObjectStore(osAlias);

      Set<Integer> idsToFetch = new HashSet<Integer>();
      itemFactory = new ItemFactory(os.getModel(), "-1_");
      writer.write(FullRenderer.getHeader() + ENDL);
      for (Iterator<Publication> iter = getPublications(os).iterator(); iter.hasNext(); ) {
        String pubMedId = iter.next().getPubMedId();
        Integer pubMedIdInteger;
        try {
          pubMedIdInteger = Integer.valueOf(pubMedId);
        } catch (NumberFormatException e) {
          // not a pubmed id
          continue;
        }

        if (seenPubMeds.contains(pubMedIdInteger)) {
          continue;
        }
        DatabaseEntry key = new DatabaseEntry(pubMedId.getBytes());
        DatabaseEntry data = new DatabaseEntry();
        if (db.get(txn, key, data, null).equals(OperationStatus.SUCCESS)) {
          try {
            ByteArrayInputStream mapInputStream = new ByteArrayInputStream(data.getData());
            ObjectInputStream deserializer = new ObjectInputStream(mapInputStream);
            Map<String, Object> pubMap = (Map) deserializer.readObject();
            writeItems(writer, mapToItems(itemFactory, pubMap));
            seenPubMeds.add(pubMedIdInteger);
          } catch (EOFException e) {
            // ignore and fetch it again
            System.err.println(
                "found in cache, but igored due to cache problem: " + pubMedIdInteger);
          }
        } else {
          idsToFetch.add(pubMedIdInteger);
        }
      }

      Iterator<Integer> idIter = idsToFetch.iterator();
      Set<Integer> thisBatch = new HashSet<Integer>();
      while (idIter.hasNext()) {
        Integer pubMedIdInteger = idIter.next();
        thisBatch.add(pubMedIdInteger);
        if (thisBatch.size() == BATCH_SIZE || !idIter.hasNext() && thisBatch.size() > 0) {
          try {
            // the server may return less publications than we ask for, so keep a Map
            Map<String, Map<String, Object>> fromServerMap = null;

            for (int i = 0; i < MAX_TRIES; i++) {
              BufferedReader br = new BufferedReader(getReader(thisBatch));
              StringBuffer buf = new StringBuffer();
              String line;
              while ((line = br.readLine()) != null) {
                buf.append(line + "\n");
              }
              fromServerMap = new HashMap<String, Map<String, Object>>();
              Throwable throwable = null;
              try {
                if (loadFullRecord) {
                  SAXParser.parse(
                      new InputSource(new StringReader(buf.toString())),
                      new FullRecordHandler(fromServerMap),
                      false);
                } else {
                  SAXParser.parse(
                      new InputSource(new StringReader(buf.toString())),
                      new SummaryRecordHandler(fromServerMap),
                      false);
                }
              } catch (Throwable e) {
                LOG.error("Couldn't parse PubMed XML", e);
                // try again or re-throw the Throwable
                throwable = e;
              }
              if (i == MAX_TRIES) {
                throw new RuntimeException(
                    "failed to parse: " + buf.toString() + " - tried " + MAX_TRIES + " times",
                    throwable);
              } else {
                if (throwable != null) {
                  // try again
                  continue;
                }
              }

              for (String id : fromServerMap.keySet()) {
                writeItems(writer, mapToItems(itemFactory, fromServerMap.get(id)));
              }
              addToDb(txn, db, fromServerMap);
              break;
            }
            thisBatch.clear();
          } finally {
            txn.commit();
            // start a new transaction incase there is an exception while parsing
            txn = env.beginTransaction(null, null);
          }
        }
      }
      writeItems(writer, authorMap.values());
      writeItems(writer, meshTerms.values());
      writer.write(FullRenderer.getFooter() + ENDL);
      writer.flush();
      writer.close();
    } catch (Throwable e) {
      throw new RuntimeException("failed to get all publications", e);
    } finally {
      txn.commit();
      db.close();
      Thread.currentThread().setContextClassLoader(cl);
    }
  }
  /** Insert or retrieve data. */
  public void run() throws DatabaseException {

    /* Create a new, transactional database environment. */
    EnvironmentConfig envConfig = new EnvironmentConfig();
    envConfig.setTransactional(true);
    envConfig.setAllowCreate(true);
    Environment exampleEnv = new Environment(envDir, envConfig);

    /*
     * Make a database within that environment. Because this will be used
     * as a primary database, it must not allow duplicates. The primary key
     * of a primary database must be unique.
     */
    Transaction txn = exampleEnv.beginTransaction(null, null);
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setTransactional(true);
    dbConfig.setAllowCreate(true);
    Database exampleDb = exampleEnv.openDatabase(txn, "bindingsDb", dbConfig);

    /*
     * In our example, the database record is composed of an integer key
     * and and instance of the MyData class as data.
     *
     * A class catalog database is needed for storing class descriptions
     * for the serial binding used below.  This avoids storing class
     * descriptions redundantly in each record.
     */
    DatabaseConfig catalogConfig = new DatabaseConfig();
    catalogConfig.setTransactional(true);
    catalogConfig.setAllowCreate(true);
    Database catalogDb = exampleEnv.openDatabase(txn, "catalogDb", catalogConfig);
    StoredClassCatalog catalog = new StoredClassCatalog(catalogDb);

    /*
     * Create a serial binding for MyData data objects.  Serial
     * bindings can be used to store any Serializable object.
     */
    EntryBinding<MyData> dataBinding = new SerialBinding<MyData>(catalog, MyData.class);

    /*
     * Further below we'll use a tuple binding (IntegerBinding
     * specifically) for integer keys.  Tuples, unlike serialized
     * Java objects, have a well defined sort order.
     */

    /*
     * Define a String tuple binding for a secondary key.  The
     * secondary key is the msg field of the MyData object.
     */
    EntryBinding<String> secKeyBinding = TupleBinding.getPrimitiveBinding(String.class);

    /*
     * Open a secondary database to allow accessing the primary
     * database by the secondary key value.
     */
    SecondaryConfig secConfig = new SecondaryConfig();
    secConfig.setTransactional(true);
    secConfig.setAllowCreate(true);
    secConfig.setSortedDuplicates(true);
    secConfig.setKeyCreator(new MyKeyCreator(secKeyBinding, dataBinding));
    SecondaryDatabase exampleSecDb =
        exampleEnv.openSecondaryDatabase(txn, "bindingsSecDb", exampleDb, secConfig);
    txn.commit();

    /* DatabaseEntry represents the key and data of each record. */
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry dataEntry = new DatabaseEntry();

    if (doInsert) {

      /*
       * Put some data in.  Note that the primary database is always used
       * to add data.  Adding or changing data in the secondary database
       * is not allowed; however, deleting through the secondary database
       * is allowed.
       */
      for (int i = offset; i < numRecords + offset; i++) {
        txn = exampleEnv.beginTransaction(null, null);
        StringBuffer stars = new StringBuffer();
        for (int j = 0; j < i; j++) {
          stars.append('*');
        }
        MyData data = new MyData(i, stars.toString());

        IntegerBinding.intToEntry(i, keyEntry);
        dataBinding.objectToEntry(data, dataEntry);

        OperationStatus status = exampleDb.put(txn, keyEntry, dataEntry);

        /*
         * Note that put will throw a DatabaseException when error
         * conditions are found such as deadlock.  However, the status
         * return conveys a variety of information. For example, the
         * put might succeed, or it might not succeed if the record
         * exists and duplicates were not
         */
        if (status != OperationStatus.SUCCESS) {
          throw new RuntimeException("Data insertion got status " + status);
        }
        txn.commit();
      }
    } else {

      /*
       * Retrieve the data by secondary key by opening a cursor on the
       * secondary database.  The key parameter for a secondary cursor is
       * always the secondary key, but the data parameter is always the
       * data of the primary database.  You can cast the cursor to a
       * SecondaryCursor and use additional method signatures for
       * retrieving the primary key also.  Or you can call
       * openSecondaryCursor() to avoid casting.
       */
      txn = exampleEnv.beginTransaction(null, null);
      Cursor cursor = exampleSecDb.openCursor(txn, null);

      while (cursor.getNext(keyEntry, dataEntry, LockMode.DEFAULT) == OperationStatus.SUCCESS) {

        String key = secKeyBinding.entryToObject(keyEntry);
        MyData data = dataBinding.entryToObject(dataEntry);

        System.out.println("key=" + key + " data=" + data);
      }
      cursor.close();
      txn.commit();
    }

    /*
     * Always close secondary databases before closing their associated
     * primary database.
     */
    catalogDb.close();
    exampleSecDb.close();
    exampleDb.close();
    exampleEnv.close();
  }
Пример #24
0
  /**
   * A getPrimaryIndex with extra parameters for opening a raw store. primaryKeyClass and
   * entityClass are used for generic typing; for a raw store, these should always be Object.class
   * and RawObject.class. primaryKeyClassName is used for consistency checking and should be null
   * for a raw store only. entityClassName is used to identify the store and may not be null.
   */
  public synchronized <PK, E> PrimaryIndex<PK, E> getPrimaryIndex(
      Class<PK> primaryKeyClass,
      String primaryKeyClassName,
      Class<E> entityClass,
      String entityClassName)
      throws DatabaseException {

    assert (rawAccess && entityClass == RawObject.class)
        || (!rawAccess && entityClass != RawObject.class);
    assert (rawAccess && primaryKeyClassName == null)
        || (!rawAccess && primaryKeyClassName != null);

    checkOpen();

    PrimaryIndex<PK, E> priIndex = priIndexMap.get(entityClassName);
    if (priIndex == null) {

      /* Check metadata. */
      EntityMetadata entityMeta = checkEntityClass(entityClassName);
      PrimaryKeyMetadata priKeyMeta = entityMeta.getPrimaryKey();
      if (primaryKeyClassName == null) {
        primaryKeyClassName = priKeyMeta.getClassName();
      } else {
        String expectClsName = SimpleCatalog.keyClassName(priKeyMeta.getClassName());
        if (!primaryKeyClassName.equals(expectClsName)) {
          throw new IllegalArgumentException(
              "Wrong primary key class: "
                  + primaryKeyClassName
                  + " Correct class is: "
                  + expectClsName);
        }
      }

      /* Create bindings. */
      PersistEntityBinding entityBinding =
          new PersistEntityBinding(catalog, entityClassName, rawAccess);
      PersistKeyBinding keyBinding = getKeyBinding(primaryKeyClassName);

      /* If not read-only, get the primary key sequence. */
      String seqName = priKeyMeta.getSequenceName();
      if (!storeConfig.getReadOnly() && seqName != null) {
        entityBinding.keyAssigner =
            new PersistKeyAssigner(keyBinding, entityBinding, getSequence(seqName));
      }

      /*
       * Use a single transaction for opening the primary DB and its
       * secondaries.  If opening any secondary fails, abort the
       * transaction and undo the changes to the state of the store.
       * Also support undo if the store is non-transactional.
       */
      Transaction txn = null;
      DatabaseConfig dbConfig = getPrimaryConfig(entityMeta);
      if (dbConfig.getTransactional() && env.getThreadTransaction() == null) {
        txn = env.beginTransaction(null, null);
      }
      PrimaryOpenState priOpenState = new PrimaryOpenState(entityClassName);
      boolean success = false;
      try {

        /* Open the primary database. */
        String dbName = storePrefix + entityClassName;
        Database db = env.openDatabase(txn, dbName, dbConfig);
        priOpenState.addDatabase(db);

        /* Create index object. */
        priIndex = new PrimaryIndex(db, primaryKeyClass, keyBinding, entityClass, entityBinding);

        /* Update index and database maps. */
        priIndexMap.put(entityClassName, priIndex);
        if (DbCompat.getDeferredWrite(dbConfig)) {
          deferredWriteDatabases.put(db, null);
        }

        /* If not read-only, open all associated secondaries. */
        if (!dbConfig.getReadOnly()) {
          openSecondaryIndexes(txn, entityMeta, priOpenState);

          /*
           * To enable foreign key contratints, also open all primary
           * indexes referring to this class via a relatedEntity
           * property in another entity. [#15358]
           */
          Set<String> inverseClassNames = inverseRelatedEntityMap.get(entityClassName);
          if (inverseClassNames != null) {
            for (String relatedClsName : inverseClassNames) {
              getRelatedIndex(relatedClsName);
            }
          }
        }
        success = true;
      } finally {
        if (success) {
          if (txn != null) {
            txn.commit();
          }
        } else {
          if (txn != null) {
            txn.abort();
          } else {
            priOpenState.closeDatabases();
          }
          priOpenState.undoState();
        }
      }
    }
    return priIndex;
  }
  /** Insert or retrieve data */
  public void run() throws DatabaseException {
    /* Create a new, transactional database environment */
    EnvironmentConfig envConfig = new EnvironmentConfig();
    envConfig.setTransactional(true);
    envConfig.setAllowCreate(true);
    Environment exampleEnv = new Environment(envDir, envConfig);

    /* Make a database within that environment */
    Transaction txn = exampleEnv.beginTransaction(null, null);
    DatabaseConfig dbConfig = new DatabaseConfig();
    dbConfig.setTransactional(true);
    dbConfig.setAllowCreate(true);
    dbConfig.setSortedDuplicates(true);
    Database exampleDb = exampleEnv.openDatabase(txn, "bindingsDb", dbConfig);

    /*
     * In our example, the database record is composed of an integer
     * key and and instance of the MyData class as data.
     *
     * A class catalog database is needed for storing class descriptions
     * for the serial binding used below.  This avoids storing class
     * descriptions redundantly in each record.
     */
    DatabaseConfig catalogConfig = new DatabaseConfig();
    catalogConfig.setTransactional(true);
    catalogConfig.setAllowCreate(true);
    Database catalogDb = exampleEnv.openDatabase(txn, "catalogDb", catalogConfig);
    StoredClassCatalog catalog = new StoredClassCatalog(catalogDb);

    /*
     * Create a serial binding for MyData data objects.  Serial bindings
     * can be used to store any Serializable object.
     */
    EntryBinding<MyData> dataBinding = new SerialBinding<MyData>(catalog, MyData.class);

    txn.commit();

    /*
     * Further below we'll use a tuple binding (IntegerBinding
     * specifically) for integer keys.  Tuples, unlike serialized Java
     * objects, have a well defined sort order.
     */

    /* DatabaseEntry represents the key and data of each record */
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry dataEntry = new DatabaseEntry();

    if (doInsert) {

      /* put some data in */
      for (int i = offset; i < numRecords + offset; i++) {

        StringBuilder stars = new StringBuilder();
        for (int j = 0; j < i; j++) {
          stars.append('*');
        }
        MyData data = new MyData(i, stars.toString());

        IntegerBinding.intToEntry(i, keyEntry);
        dataBinding.objectToEntry(data, dataEntry);

        txn = exampleEnv.beginTransaction(null, null);
        OperationStatus status = exampleDb.put(txn, keyEntry, dataEntry);

        /*
         * Note that put will throw a DatabaseException when
         * error conditions are found such as deadlock.
         * However, the status return conveys a variety of
         * information. For example, the put might succeed,
         * or it might not succeed if the record exists
         * and duplicates were not
         */
        if (status != OperationStatus.SUCCESS) {
          throw new RuntimeException("Data insertion got status " + status);
        }
        txn.commit();
      }
    } else {

      /* retrieve the data */
      Cursor cursor = exampleDb.openCursor(null, null);

      while (cursor.getNext(keyEntry, dataEntry, LockMode.DEFAULT) == OperationStatus.SUCCESS) {

        int key = IntegerBinding.entryToInt(keyEntry);
        MyData data = dataBinding.entryToObject(dataEntry);

        System.out.println("key=" + key + " data=" + data);
      }
      cursor.close();
    }

    catalogDb.close();
    exampleDb.close();
    exampleEnv.close();
  }
Пример #26
0
  private void evolveIndex(Format format, EvolveEvent event, EvolveListener listener)
      throws DatabaseException {

    Class entityClass = format.getType();
    String entityClassName = format.getClassName();
    EntityMetadata meta = model.getEntityMetadata(entityClassName);
    String keyClassName = meta.getPrimaryKey().getClassName();
    keyClassName = SimpleCatalog.keyClassName(keyClassName);
    DatabaseConfig dbConfig = getPrimaryConfig(meta);

    PrimaryIndex<Object, Object> index =
        getPrimaryIndex(Object.class, keyClassName, entityClass, entityClassName);
    Database db = index.getDatabase();

    EntityBinding binding = index.getEntityBinding();
    DatabaseEntry key = new DatabaseEntry();
    DatabaseEntry data = new DatabaseEntry();

    Cursor readCursor = db.openCursor(null, CursorConfig.READ_UNCOMMITTED);
    try {
      while (readCursor.getNext(key, data, null) == OperationStatus.SUCCESS) {
        if (evolveNeeded(key, data, binding)) {
          Transaction txn = null;
          if (dbConfig.getTransactional()) {
            boolean success = false;
            txn = env.beginTransaction(null, null);
          }
          boolean doCommit = false;
          Cursor writeCursor = null;
          try {
            writeCursor = db.openCursor(txn, null);
            if (writeCursor.getSearchKey(key, data, LockMode.RMW) == OperationStatus.SUCCESS) {
              boolean written = false;
              if (evolveNeeded(key, data, binding)) {
                writeCursor.putCurrent(data);
                written = true;
              }
              if (listener != null) {
                EvolveInternal.updateEvent(event, entityClassName, 1, written ? 1 : 0);
                if (!listener.evolveProgress(event)) {
                  break;
                }
              }
            }
          } finally {
            if (writeCursor != null) {
              writeCursor.close();
            }
            if (txn != null) {
              if (doCommit) {
                txn.commit();
              } else {
                txn.abort();
              }
            }
          }
        }
      }
    } finally {
      readCursor.close();
    }
  }