public void put(WebURL url) throws DatabaseException {
/*
* The key that is used for storing URLs determines the order
* they are crawled. Lower key values results in earlier crawling.
* Here our keys are 6 bytes. The first byte comes from the URL priority.
* The second byte comes from depth of crawl at which this URL is first found.
* The rest of the 4 bytes come from the docid of the URL. As a result,
* URLs with lower priority numbers will be crawled earlier. If priority
* numbers are the same, those found at lower depths will be crawled earlier.
* If depth is also equal, those found earlier (therefore, smaller docid) will
* be crawled earlier.
*/
byte[] keyData = new byte[6];
keyData[0] = url.getPriority();
keyData[1] = (url.getDepth() > Byte.MAX_VALUE ? Byte.MAX_VALUE : (byte) url.getDepth());
Util.putIntInByteArray(url.getDocid(), keyData, 2);
DatabaseEntry value = new DatabaseEntry();
webURLBinding.objectToEntry(url, value);
Transaction txn;
if (resumable) {
txn = env.beginTransaction(null, null);
} else {
txn = null;
}
urlsDB.put(txn, new DatabaseEntry(keyData), value);
if (resumable) {
if (txn != null) {
txn.commit();
}
}
}
public void testDuplicatesWithDeletion() throws Throwable {
createEnvAndDbs(1 << 20, true, NUM_DBS);
int numRecs = 10;
int nDups = N_DUPLICATES_PER_KEY;
try {
/* Set up an repository of expected data. */
Map<TestData, Set<TestData>> expectedData = new HashMap<TestData, Set<TestData>>();
/* Insert all the data. */
Transaction txn = env.beginTransaction(null, null);
insertData(txn, 0, numRecs - 1, expectedData, nDups, true, NUM_DBS);
/* Delete all the even records. */
deleteData(txn, expectedData, false, true, NUM_DBS);
txn.commit();
/* Modify all the records. */
// modifyData(expectedData);
closeEnv();
recoverAndVerify(expectedData, NUM_DBS);
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}
/** Write data into the database. */
private void generateData(
Environment master, int numTxns, Durability durability, boolean doCommit) {
/* Write some data. */
DatabaseEntry key = new DatabaseEntry();
byte[] dataPadding = new byte[1000];
DatabaseEntry data = new DatabaseEntry(dataPadding);
TransactionConfig txnConfig = new TransactionConfig();
txnConfig.setDurability(durability);
for (int i = 0; i < numTxns; i++) {
final Transaction txn = master.beginTransaction(null, txnConfig);
// long keyPrefix = i << 10;
// LongBinding.longToEntry(keyPrefix + i, key);
LongBinding.longToEntry(i, key);
db.put(txn, key, data);
if (doCommit) {
txn.commit();
} else {
txn.abort();
}
}
}
public V remove(K key, V value) throws Exception {
DatabaseEntry keyEntry = new DatabaseEntry();
DatabaseEntry foundEntry = new DatabaseEntry();
keyBinding.objectToEntry(key, keyEntry);
V ret = null;
Transaction txn = env.beginTransaction(null, null);
Cursor cursor = db.openCursor(txn, null);
if ((cursor.getSearchKey(keyEntry, foundEntry, LockMode.DEFAULT)) == OperationStatus.SUCCESS) {
do {
V v = (V) dataBinding.entryToObject(foundEntry);
if (value.equals(v)) {
ret = v;
cursor.delete();
break;
}
} while ((cursor.getNextDup(keyEntry, foundEntry, LockMode.DEFAULT))
== OperationStatus.SUCCESS);
}
// Cannot use Cursor#getSearchBoth() because it compare values byte-wise without Object#equal().
cursor.close();
txn.commit();
return ret;
}
@Override
public String saveDocument(
DsSessionDto aSessionDto,
DsDocumentDto aDocumentDto,
String aKey,
boolean aKeyIsNumeric,
String aValue,
String aSecurityPolicyName,
String aContentFileKey) {
if ("bug".equals(aSessionDto.getUsername())) {
throw new IllegalStateException("this is a bug");
}
Transaction tx = entityStore.getEnvironment().beginTransaction(null, null);
Document doc = new Document();
doc.setFileKey(aContentFileKey);
doc.setFileName(getFileName(aDocumentDto));
documentPrimaryIndex.put(doc);
tx.commit();
return String.valueOf(doc.getId());
}
private void attemptCommit(Transaction transaction) {
try {
transaction.commit();
} catch (DatabaseException e) {
logger.error("Transaction commit failed!", e);
attemptAbort(transaction);
throw new PersistenceFailureException(e);
}
}
@Override
public void deleteDocument(DsSessionDto aSessionDto, String aDocumentId) {
if ("bug".equals(aSessionDto.getUsername())) {
throw new IllegalStateException("this is a bug");
}
Transaction tx = entityStore.getEnvironment().beginTransaction(null, null);
documentPrimaryIndex.delete(Long.valueOf(aDocumentId));
tx.commit();
}
/** {@inheritDoc} */
public void commit() {
try {
if (xid != null) {
env.commit(xid, true /* ignored */);
} else {
txn.commit();
}
} catch (DatabaseException e) {
throw JeEnvironment.convertException(e, false);
} catch (XAException e) {
throw JeEnvironment.convertException(e, false);
}
}
/*
* See SR11455 for details.
*
* This test is checking that the maxTxnId gets recovered properly during
* recovery. The SR has to do with the INFileReader not including
* DupCountLN_TX and DelDupLN_TX's in its txnIdTrackingMap. When these
* were not included, it was possible for a transaction to consist solely
* of DupCountLN_TX/DelDupLN_TX pairs. The "deleteData" transaction below
* does just this. If no checkpoint occurred following such a transaction,
* then the correct current txnid would not be written to the log and
* determining this value during recovery would be left up to the
* INFileReader. However, without reading the DupCountLN_TX/DelDupLN_TX
* records, it would not recover the correct value.
*
* We take the poor man's way out of creating this situation by just
* manually asserting the txn id is correct post-recovery. The txnid of 12
* was determined by looking through logs before and after the fix.
*/
public void testSR11455() throws Throwable {
createEnvAndDbs(1 << 20, true, 1);
int numRecs = 1;
int nDups = 3;
try {
/* Set up an repository of expected data. */
Map<TestData, Set<TestData>> expectedData = new HashMap<TestData, Set<TestData>>();
/* Insert all the data. */
Transaction txn = env.beginTransaction(null, null);
insertData(txn, 0, numRecs - 1, expectedData, nDups, true, 1);
txn.commit();
txn = env.beginTransaction(null, null);
/* Delete all the even records. */
deleteData(txn, expectedData, false, false, 1);
txn.abort();
closeEnv();
/* Open it again, which will run recovery. */
EnvironmentConfig recoveryConfig = TestUtils.initEnvConfig();
recoveryConfig.setTransactional(true);
recoveryConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
recoveryConfig.setConfigParam(EnvironmentParams.ENV_RUN_EVICTOR.getName(), "false");
env = new Environment(envHome, recoveryConfig);
txn = env.beginTransaction(null, null);
assertEquals(6, txn.getId());
txn.commit();
env.close();
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}
@Override
public synchronized void commit() {
try {
if (!openedCursors.isEmpty()) {
closeOpenedCursors();
throw new BerkeleyDBPersistenceOpenedCursorsException();
}
dbTransaction.commit();
getPersistenceManager().unregisterTransaction(this);
super.commit();
} catch (DatabaseException e) {
throw getPersistenceManager().convertDatabaseException(e);
}
}
/**
* delete from the database
*
* @param key
* @throws Exception
*/
public void delete(Object key) throws Exception {
Serializer serializer = new Serializer();
byte[] keyBytes = serializer.serialize(key);
final DatabaseEntry keyEntry = new DatabaseEntry(keyBytes);
final Transaction transaction = myEnv.beginTransaction(null, null);
final OperationStatus res = myDatabase.delete(transaction, keyEntry);
if (res != OperationStatus.SUCCESS) {
throw new Exception("Error retrieving from database");
}
transaction.commit();
return;
}
public V put(K key, V value) throws Exception {
DatabaseEntry keyEntry = new DatabaseEntry();
DatabaseEntry dataEntry = new DatabaseEntry();
keyBinding.objectToEntry(key, keyEntry);
dataBinding.objectToEntry(value, dataEntry);
V ret = null;
Transaction txn = env.beginTransaction(null, null);
Cursor cursor = db.openCursor(txn, null);
if ((cursor.getSearchKey(keyEntry, dataEntry, LockMode.DEFAULT)) == OperationStatus.SUCCESS) {
V v = (V) dataBinding.entryToObject(dataEntry);
if (value.equals(v)) { // found
ret = v;
cursor.delete(); // remove here. and put later.
} else {
while ((cursor.getNextDup(keyEntry, dataEntry, LockMode.DEFAULT))
== OperationStatus.SUCCESS) {
v = (V) dataBinding.entryToObject(dataEntry);
if (value.equals(v)) { // found
ret = v;
cursor.delete(); // remove here. and put later.
break;
}
}
}
}
// put
keyBinding.objectToEntry(key, keyEntry);
dataBinding.objectToEntry(value, dataEntry);
if ((cursor.put(keyEntry, dataEntry)) != OperationStatus.SUCCESS) {
String msg = "Could not put: " + key + ", " + value;
logger.log(Level.WARNING, msg);
throw new DatabaseException(msg);
}
cursor.close();
txn.commit();
return ret;
}
public List<WebURL> get(int max) throws DatabaseException {
synchronized (mutex) {
int matches = 0;
List<WebURL> results = new ArrayList<WebURL>(max);
Cursor cursor = null;
OperationStatus result;
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry value = new DatabaseEntry();
Transaction txn;
if (resumable) {
txn = env.beginTransaction(null, null);
} else {
txn = null;
}
try {
cursor = urlsDB.openCursor(txn, null);
result = cursor.getFirst(key, value, null);
while (matches < max && result == OperationStatus.SUCCESS) {
if (value.getData().length > 0) {
results.add(webURLBinding.entryToObject(value));
matches++;
}
result = cursor.getNext(key, value, null);
}
} catch (DatabaseException e) {
if (txn != null) {
txn.abort();
txn = null;
}
throw e;
} finally {
if (cursor != null) {
cursor.close();
}
if (txn != null) {
txn.commit();
}
}
return results;
}
}
private void testMemberRemoveAckInteraction(final boolean delete) {
createGroup(groupSize);
Transaction txn;
Database db;
try {
MasterTxn.setFactory(new TxnFactory(delete));
ReplicatedEnvironment master = repEnvInfo[0].getEnv();
txn = master.beginTransaction(null, null);
/* Write to the environment. */
db = master.openDatabase(txn, "random", dbconfig);
db.close();
txn.commit();
} catch (InsufficientAcksException e) {
fail("No exception expected.");
} finally {
MasterTxn.setFactory(null);
}
}
/**
* gets object from databse
*
* @param key
* @return
* @throws Exception
*/
public Object get(Object key) throws Exception {
Serializer serializer = new Serializer();
byte[] keyBytes = serializer.serialize(key);
final DatabaseEntry keyEntry = new DatabaseEntry(keyBytes);
Object result;
final DatabaseEntry dataEntry = new DatabaseEntry();
final Transaction transaction = myEnv.beginTransaction(null, null);
final OperationStatus res = myDatabase.get(transaction, keyEntry, dataEntry, null);
if (res != OperationStatus.SUCCESS) {
// throw new Exception("Error retrieving from database");
return null;
} else {
result = serializer.deserialize(dataEntry.getData());
}
transaction.commit();
return result;
}
/**
* inserts into databse
*
* @param key
* @param data
* @throws Exception
*/
public void insert(Object key, Object data) throws Exception {
Serializer serializer = new Serializer();
byte[] keyBytes = serializer.serialize(key);
byte[] dataBytes = serializer.serialize(data);
final DatabaseEntry keyEntry = new DatabaseEntry(keyBytes);
final DatabaseEntry dataEntry = new DatabaseEntry(dataBytes);
try {
final Transaction transaction = myEnv.beginTransaction(null, null);
final OperationStatus result = myDatabase.put(transaction, keyEntry, dataEntry);
if (result != OperationStatus.SUCCESS) {
System.out.println("operation status-failure");
throw new Exception("Error");
}
transaction.commit();
} catch (Exception DE) {
System.out.println(DE);
}
}
public void delete(int count) throws DatabaseException {
synchronized (mutex) {
int matches = 0;
Cursor cursor = null;
OperationStatus result;
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry value = new DatabaseEntry();
Transaction txn;
if (resumable) {
txn = env.beginTransaction(null, null);
} else {
txn = null;
}
try {
cursor = urlsDB.openCursor(txn, null);
result = cursor.getFirst(key, value, null);
while (matches < count && result == OperationStatus.SUCCESS) {
cursor.delete();
matches++;
result = cursor.getNext(key, value, null);
}
} catch (DatabaseException e) {
if (txn != null) {
txn.abort();
txn = null;
}
throw e;
} finally {
if (cursor != null) {
cursor.close();
}
if (txn != null) {
txn.commit();
}
}
}
}
@Override
public synchronized boolean trim(Long instance) {
if (instance == 0) {
return true;
} // fast track
Transaction t = null;
if (db.getConfig().getTransactional()) {
t = env.beginTransaction(null, null);
}
Cursor cursor = db.openCursor(t, null);
boolean dirty = false;
try {
while (cursor.getNext(key, data, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
Long i = keyBinding.entryToObject(key);
if (i < instance && cursor.delete() != OperationStatus.SUCCESS) {
logger.error("Error deleting instance " + i + " from DB!");
dirty = true;
}
}
} finally {
cursor.close();
if (!dirty) {
if (t != null) {
t.commit();
}
} else {
if (t != null) {
t.abort();
}
return false;
}
}
putDecision(-1L, new Decision(0, instance, 0, null));
logger.debug("DB deltete up to instance " + instance);
return true;
}
private Set<V> getAndRemove(K key, boolean remove) throws DatabaseException {
DatabaseEntry searchKey = new DatabaseEntry();
keyBinding.objectToEntry(key, searchKey);
DatabaseEntry foundKey = new DatabaseEntry();
DatabaseEntry foundData = new DatabaseEntry();
Transaction txn = env.beginTransaction(null, null);
Cursor cursor = db.openCursor(txn, null);
Set<V> s = null;
if ((cursor.getSearchKey(searchKey, foundData, LockMode.DEFAULT)) == OperationStatus.SUCCESS) {
s = new HashSet<V>();
s.add((V) dataBinding.entryToObject(foundData));
while ((cursor.getNextDup(foundKey, foundData, LockMode.DEFAULT))
== OperationStatus.SUCCESS) {
// K k = (K)keyBinding.entryToObject(foundKey);
// if (!key.equals(k)) {
// break;
// }
s.add((V) dataBinding.entryToObject(foundData));
}
}
// remove all entries associated with the given key
if (remove) {
db.delete(txn, searchKey);
// does not check the result of this operation.
}
cursor.close();
txn.commit();
return s;
}
/** removes database */
public void removeDatabase() {
final Transaction txn = myEnv.beginTransaction(null, null);
// close();
myEnv.removeDatabase(txn, dbName);
txn.commit();
}
public Store(Environment env, String storeName, StoreConfig config, boolean rawAccess)
throws DatabaseException {
this.env = env;
this.storeName = storeName;
this.rawAccess = rawAccess;
if (env == null || storeName == null) {
throw new NullPointerException("env and storeName parameters must not be null");
}
if (config != null) {
model = config.getModel();
mutations = config.getMutations();
}
if (config == null) {
storeConfig = StoreConfig.DEFAULT;
} else {
storeConfig = config.cloneConfig();
}
storePrefix = NAME_PREFIX + storeName + NAME_SEPARATOR;
priIndexMap = new HashMap<String, PrimaryIndex>();
secIndexMap = new HashMap<String, SecondaryIndex>();
priConfigMap = new HashMap<String, DatabaseConfig>();
secConfigMap = new HashMap<String, SecondaryConfig>();
keyBindingMap = new HashMap<String, PersistKeyBinding>();
sequenceMap = new HashMap<String, Sequence>();
sequenceConfigMap = new HashMap<String, SequenceConfig>();
deferredWriteDatabases = new IdentityHashMap<Database, Object>();
if (rawAccess) {
/* Open a read-only catalog that uses the stored model. */
if (model != null) {
throw new IllegalArgumentException("A model may not be specified when opening a RawStore");
}
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setReadOnly(true);
dbConfig.setTransactional(storeConfig.getTransactional());
catalog =
new PersistCatalog(
null,
env,
storePrefix,
storePrefix + CATALOG_DB,
dbConfig,
model,
mutations,
rawAccess,
this);
} else {
/* Open the shared catalog that uses the current model. */
synchronized (catalogPool) {
Map<String, PersistCatalog> catalogMap = catalogPool.get(env);
if (catalogMap == null) {
catalogMap = new HashMap<String, PersistCatalog>();
catalogPool.put(env, catalogMap);
}
catalog = catalogMap.get(storeName);
if (catalog != null) {
catalog.openExisting();
} else {
Transaction txn = null;
if (storeConfig.getTransactional() && env.getThreadTransaction() == null) {
txn = env.beginTransaction(null, null);
}
boolean success = false;
try {
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(storeConfig.getAllowCreate());
dbConfig.setReadOnly(storeConfig.getReadOnly());
dbConfig.setTransactional(storeConfig.getTransactional());
catalog =
new PersistCatalog(
txn,
env,
storePrefix,
storePrefix + CATALOG_DB,
dbConfig,
model,
mutations,
rawAccess,
this);
catalogMap.put(storeName, catalog);
success = true;
} finally {
if (txn != null) {
if (success) {
txn.commit();
} else {
txn.abort();
}
}
}
}
}
}
/* Get the merged mutations from the catalog. */
mutations = catalog.getMutations();
/*
* If there is no model parameter, use the default or stored model
* obtained from the catalog.
*/
model = catalog.getResolvedModel();
/*
* Give the model a reference to the catalog to fully initialize the
* model. Only then may we initialize the Converter mutations, which
* themselves may call model methods and expect the model to be fully
* initialized.
*/
ModelInternal.setCatalog(model, catalog);
for (Converter converter : mutations.getConverters()) {
converter.getConversion().initialize(model);
}
/*
* For each existing entity with a relatedEntity reference, create an
* inverse map (back pointer) from the class named in the relatedEntity
* to the class containing the secondary key. This is used to open the
* class containing the secondary key whenever we open the
* relatedEntity class, to configure foreign key constraints. Note that
* we do not need to update this map as new primary indexes are
* created, because opening the new index will setup the foreign key
* constraints. [#15358]
*/
inverseRelatedEntityMap = new HashMap<String, Set<String>>();
List<Format> entityFormats = new ArrayList<Format>();
catalog.getEntityFormats(entityFormats);
for (Format entityFormat : entityFormats) {
EntityMetadata entityMeta = entityFormat.getEntityMetadata();
for (SecondaryKeyMetadata secKeyMeta : entityMeta.getSecondaryKeys().values()) {
String relatedClsName = secKeyMeta.getRelatedEntity();
if (relatedClsName != null) {
Set<String> inverseClassNames = inverseRelatedEntityMap.get(relatedClsName);
if (inverseClassNames == null) {
inverseClassNames = new HashSet<String>();
inverseRelatedEntityMap.put(relatedClsName, inverseClassNames);
}
inverseClassNames.add(entityMeta.getClassName());
}
}
}
}
/**
* Synchronize publications with pubmed using pmid
*
* @throws Exception if an error occurs
*/
public void execute() throws Exception {
// Needed so that STAX can find it's implementation classes
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
Database db = null;
Transaction txn = null;
try {
if (osAlias == null) {
throw new BuildException("osAlias attribute is not set");
}
if (outputFile == null) {
throw new BuildException("outputFile attribute is not set");
}
// environment is transactional
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
Environment env = new Environment(new File(cacheDirName), envConfig);
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
dbConfig.setSortedDuplicates(true);
db = env.openDatabase(null, "publications_db", dbConfig);
txn = env.beginTransaction(null, null);
LOG.info("Starting EntrezPublicationsRetriever");
Writer writer = new FileWriter(outputFile);
ObjectStore os = ObjectStoreFactory.getObjectStore(osAlias);
Set<Integer> idsToFetch = new HashSet<Integer>();
itemFactory = new ItemFactory(os.getModel(), "-1_");
writer.write(FullRenderer.getHeader() + ENDL);
for (Iterator<Publication> iter = getPublications(os).iterator(); iter.hasNext(); ) {
String pubMedId = iter.next().getPubMedId();
Integer pubMedIdInteger;
try {
pubMedIdInteger = Integer.valueOf(pubMedId);
} catch (NumberFormatException e) {
// not a pubmed id
continue;
}
if (seenPubMeds.contains(pubMedIdInteger)) {
continue;
}
DatabaseEntry key = new DatabaseEntry(pubMedId.getBytes());
DatabaseEntry data = new DatabaseEntry();
if (db.get(txn, key, data, null).equals(OperationStatus.SUCCESS)) {
try {
ByteArrayInputStream mapInputStream = new ByteArrayInputStream(data.getData());
ObjectInputStream deserializer = new ObjectInputStream(mapInputStream);
Map<String, Object> pubMap = (Map) deserializer.readObject();
writeItems(writer, mapToItems(itemFactory, pubMap));
seenPubMeds.add(pubMedIdInteger);
} catch (EOFException e) {
// ignore and fetch it again
System.err.println(
"found in cache, but igored due to cache problem: " + pubMedIdInteger);
}
} else {
idsToFetch.add(pubMedIdInteger);
}
}
Iterator<Integer> idIter = idsToFetch.iterator();
Set<Integer> thisBatch = new HashSet<Integer>();
while (idIter.hasNext()) {
Integer pubMedIdInteger = idIter.next();
thisBatch.add(pubMedIdInteger);
if (thisBatch.size() == BATCH_SIZE || !idIter.hasNext() && thisBatch.size() > 0) {
try {
// the server may return less publications than we ask for, so keep a Map
Map<String, Map<String, Object>> fromServerMap = null;
for (int i = 0; i < MAX_TRIES; i++) {
BufferedReader br = new BufferedReader(getReader(thisBatch));
StringBuffer buf = new StringBuffer();
String line;
while ((line = br.readLine()) != null) {
buf.append(line + "\n");
}
fromServerMap = new HashMap<String, Map<String, Object>>();
Throwable throwable = null;
try {
if (loadFullRecord) {
SAXParser.parse(
new InputSource(new StringReader(buf.toString())),
new FullRecordHandler(fromServerMap),
false);
} else {
SAXParser.parse(
new InputSource(new StringReader(buf.toString())),
new SummaryRecordHandler(fromServerMap),
false);
}
} catch (Throwable e) {
LOG.error("Couldn't parse PubMed XML", e);
// try again or re-throw the Throwable
throwable = e;
}
if (i == MAX_TRIES) {
throw new RuntimeException(
"failed to parse: " + buf.toString() + " - tried " + MAX_TRIES + " times",
throwable);
} else {
if (throwable != null) {
// try again
continue;
}
}
for (String id : fromServerMap.keySet()) {
writeItems(writer, mapToItems(itemFactory, fromServerMap.get(id)));
}
addToDb(txn, db, fromServerMap);
break;
}
thisBatch.clear();
} finally {
txn.commit();
// start a new transaction incase there is an exception while parsing
txn = env.beginTransaction(null, null);
}
}
}
writeItems(writer, authorMap.values());
writeItems(writer, meshTerms.values());
writer.write(FullRenderer.getFooter() + ENDL);
writer.flush();
writer.close();
} catch (Throwable e) {
throw new RuntimeException("failed to get all publications", e);
} finally {
txn.commit();
db.close();
Thread.currentThread().setContextClassLoader(cl);
}
}
/** Insert or retrieve data. */
public void run() throws DatabaseException {
/* Create a new, transactional database environment. */
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
Environment exampleEnv = new Environment(envDir, envConfig);
/*
* Make a database within that environment. Because this will be used
* as a primary database, it must not allow duplicates. The primary key
* of a primary database must be unique.
*/
Transaction txn = exampleEnv.beginTransaction(null, null);
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
Database exampleDb = exampleEnv.openDatabase(txn, "bindingsDb", dbConfig);
/*
* In our example, the database record is composed of an integer key
* and and instance of the MyData class as data.
*
* A class catalog database is needed for storing class descriptions
* for the serial binding used below. This avoids storing class
* descriptions redundantly in each record.
*/
DatabaseConfig catalogConfig = new DatabaseConfig();
catalogConfig.setTransactional(true);
catalogConfig.setAllowCreate(true);
Database catalogDb = exampleEnv.openDatabase(txn, "catalogDb", catalogConfig);
StoredClassCatalog catalog = new StoredClassCatalog(catalogDb);
/*
* Create a serial binding for MyData data objects. Serial
* bindings can be used to store any Serializable object.
*/
EntryBinding<MyData> dataBinding = new SerialBinding<MyData>(catalog, MyData.class);
/*
* Further below we'll use a tuple binding (IntegerBinding
* specifically) for integer keys. Tuples, unlike serialized
* Java objects, have a well defined sort order.
*/
/*
* Define a String tuple binding for a secondary key. The
* secondary key is the msg field of the MyData object.
*/
EntryBinding<String> secKeyBinding = TupleBinding.getPrimitiveBinding(String.class);
/*
* Open a secondary database to allow accessing the primary
* database by the secondary key value.
*/
SecondaryConfig secConfig = new SecondaryConfig();
secConfig.setTransactional(true);
secConfig.setAllowCreate(true);
secConfig.setSortedDuplicates(true);
secConfig.setKeyCreator(new MyKeyCreator(secKeyBinding, dataBinding));
SecondaryDatabase exampleSecDb =
exampleEnv.openSecondaryDatabase(txn, "bindingsSecDb", exampleDb, secConfig);
txn.commit();
/* DatabaseEntry represents the key and data of each record. */
DatabaseEntry keyEntry = new DatabaseEntry();
DatabaseEntry dataEntry = new DatabaseEntry();
if (doInsert) {
/*
* Put some data in. Note that the primary database is always used
* to add data. Adding or changing data in the secondary database
* is not allowed; however, deleting through the secondary database
* is allowed.
*/
for (int i = offset; i < numRecords + offset; i++) {
txn = exampleEnv.beginTransaction(null, null);
StringBuffer stars = new StringBuffer();
for (int j = 0; j < i; j++) {
stars.append('*');
}
MyData data = new MyData(i, stars.toString());
IntegerBinding.intToEntry(i, keyEntry);
dataBinding.objectToEntry(data, dataEntry);
OperationStatus status = exampleDb.put(txn, keyEntry, dataEntry);
/*
* Note that put will throw a DatabaseException when error
* conditions are found such as deadlock. However, the status
* return conveys a variety of information. For example, the
* put might succeed, or it might not succeed if the record
* exists and duplicates were not
*/
if (status != OperationStatus.SUCCESS) {
throw new RuntimeException("Data insertion got status " + status);
}
txn.commit();
}
} else {
/*
* Retrieve the data by secondary key by opening a cursor on the
* secondary database. The key parameter for a secondary cursor is
* always the secondary key, but the data parameter is always the
* data of the primary database. You can cast the cursor to a
* SecondaryCursor and use additional method signatures for
* retrieving the primary key also. Or you can call
* openSecondaryCursor() to avoid casting.
*/
txn = exampleEnv.beginTransaction(null, null);
Cursor cursor = exampleSecDb.openCursor(txn, null);
while (cursor.getNext(keyEntry, dataEntry, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
String key = secKeyBinding.entryToObject(keyEntry);
MyData data = dataBinding.entryToObject(dataEntry);
System.out.println("key=" + key + " data=" + data);
}
cursor.close();
txn.commit();
}
/*
* Always close secondary databases before closing their associated
* primary database.
*/
catalogDb.close();
exampleSecDb.close();
exampleDb.close();
exampleEnv.close();
}
/**
* A getPrimaryIndex with extra parameters for opening a raw store. primaryKeyClass and
* entityClass are used for generic typing; for a raw store, these should always be Object.class
* and RawObject.class. primaryKeyClassName is used for consistency checking and should be null
* for a raw store only. entityClassName is used to identify the store and may not be null.
*/
public synchronized <PK, E> PrimaryIndex<PK, E> getPrimaryIndex(
Class<PK> primaryKeyClass,
String primaryKeyClassName,
Class<E> entityClass,
String entityClassName)
throws DatabaseException {
assert (rawAccess && entityClass == RawObject.class)
|| (!rawAccess && entityClass != RawObject.class);
assert (rawAccess && primaryKeyClassName == null)
|| (!rawAccess && primaryKeyClassName != null);
checkOpen();
PrimaryIndex<PK, E> priIndex = priIndexMap.get(entityClassName);
if (priIndex == null) {
/* Check metadata. */
EntityMetadata entityMeta = checkEntityClass(entityClassName);
PrimaryKeyMetadata priKeyMeta = entityMeta.getPrimaryKey();
if (primaryKeyClassName == null) {
primaryKeyClassName = priKeyMeta.getClassName();
} else {
String expectClsName = SimpleCatalog.keyClassName(priKeyMeta.getClassName());
if (!primaryKeyClassName.equals(expectClsName)) {
throw new IllegalArgumentException(
"Wrong primary key class: "
+ primaryKeyClassName
+ " Correct class is: "
+ expectClsName);
}
}
/* Create bindings. */
PersistEntityBinding entityBinding =
new PersistEntityBinding(catalog, entityClassName, rawAccess);
PersistKeyBinding keyBinding = getKeyBinding(primaryKeyClassName);
/* If not read-only, get the primary key sequence. */
String seqName = priKeyMeta.getSequenceName();
if (!storeConfig.getReadOnly() && seqName != null) {
entityBinding.keyAssigner =
new PersistKeyAssigner(keyBinding, entityBinding, getSequence(seqName));
}
/*
* Use a single transaction for opening the primary DB and its
* secondaries. If opening any secondary fails, abort the
* transaction and undo the changes to the state of the store.
* Also support undo if the store is non-transactional.
*/
Transaction txn = null;
DatabaseConfig dbConfig = getPrimaryConfig(entityMeta);
if (dbConfig.getTransactional() && env.getThreadTransaction() == null) {
txn = env.beginTransaction(null, null);
}
PrimaryOpenState priOpenState = new PrimaryOpenState(entityClassName);
boolean success = false;
try {
/* Open the primary database. */
String dbName = storePrefix + entityClassName;
Database db = env.openDatabase(txn, dbName, dbConfig);
priOpenState.addDatabase(db);
/* Create index object. */
priIndex = new PrimaryIndex(db, primaryKeyClass, keyBinding, entityClass, entityBinding);
/* Update index and database maps. */
priIndexMap.put(entityClassName, priIndex);
if (DbCompat.getDeferredWrite(dbConfig)) {
deferredWriteDatabases.put(db, null);
}
/* If not read-only, open all associated secondaries. */
if (!dbConfig.getReadOnly()) {
openSecondaryIndexes(txn, entityMeta, priOpenState);
/*
* To enable foreign key contratints, also open all primary
* indexes referring to this class via a relatedEntity
* property in another entity. [#15358]
*/
Set<String> inverseClassNames = inverseRelatedEntityMap.get(entityClassName);
if (inverseClassNames != null) {
for (String relatedClsName : inverseClassNames) {
getRelatedIndex(relatedClsName);
}
}
}
success = true;
} finally {
if (success) {
if (txn != null) {
txn.commit();
}
} else {
if (txn != null) {
txn.abort();
} else {
priOpenState.closeDatabases();
}
priOpenState.undoState();
}
}
}
return priIndex;
}
/** Insert or retrieve data */
public void run() throws DatabaseException {
/* Create a new, transactional database environment */
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
Environment exampleEnv = new Environment(envDir, envConfig);
/* Make a database within that environment */
Transaction txn = exampleEnv.beginTransaction(null, null);
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
dbConfig.setSortedDuplicates(true);
Database exampleDb = exampleEnv.openDatabase(txn, "bindingsDb", dbConfig);
/*
* In our example, the database record is composed of an integer
* key and and instance of the MyData class as data.
*
* A class catalog database is needed for storing class descriptions
* for the serial binding used below. This avoids storing class
* descriptions redundantly in each record.
*/
DatabaseConfig catalogConfig = new DatabaseConfig();
catalogConfig.setTransactional(true);
catalogConfig.setAllowCreate(true);
Database catalogDb = exampleEnv.openDatabase(txn, "catalogDb", catalogConfig);
StoredClassCatalog catalog = new StoredClassCatalog(catalogDb);
/*
* Create a serial binding for MyData data objects. Serial bindings
* can be used to store any Serializable object.
*/
EntryBinding<MyData> dataBinding = new SerialBinding<MyData>(catalog, MyData.class);
txn.commit();
/*
* Further below we'll use a tuple binding (IntegerBinding
* specifically) for integer keys. Tuples, unlike serialized Java
* objects, have a well defined sort order.
*/
/* DatabaseEntry represents the key and data of each record */
DatabaseEntry keyEntry = new DatabaseEntry();
DatabaseEntry dataEntry = new DatabaseEntry();
if (doInsert) {
/* put some data in */
for (int i = offset; i < numRecords + offset; i++) {
StringBuilder stars = new StringBuilder();
for (int j = 0; j < i; j++) {
stars.append('*');
}
MyData data = new MyData(i, stars.toString());
IntegerBinding.intToEntry(i, keyEntry);
dataBinding.objectToEntry(data, dataEntry);
txn = exampleEnv.beginTransaction(null, null);
OperationStatus status = exampleDb.put(txn, keyEntry, dataEntry);
/*
* Note that put will throw a DatabaseException when
* error conditions are found such as deadlock.
* However, the status return conveys a variety of
* information. For example, the put might succeed,
* or it might not succeed if the record exists
* and duplicates were not
*/
if (status != OperationStatus.SUCCESS) {
throw new RuntimeException("Data insertion got status " + status);
}
txn.commit();
}
} else {
/* retrieve the data */
Cursor cursor = exampleDb.openCursor(null, null);
while (cursor.getNext(keyEntry, dataEntry, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
int key = IntegerBinding.entryToInt(keyEntry);
MyData data = dataBinding.entryToObject(dataEntry);
System.out.println("key=" + key + " data=" + data);
}
cursor.close();
}
catalogDb.close();
exampleDb.close();
exampleEnv.close();
}
private void evolveIndex(Format format, EvolveEvent event, EvolveListener listener)
throws DatabaseException {
Class entityClass = format.getType();
String entityClassName = format.getClassName();
EntityMetadata meta = model.getEntityMetadata(entityClassName);
String keyClassName = meta.getPrimaryKey().getClassName();
keyClassName = SimpleCatalog.keyClassName(keyClassName);
DatabaseConfig dbConfig = getPrimaryConfig(meta);
PrimaryIndex<Object, Object> index =
getPrimaryIndex(Object.class, keyClassName, entityClass, entityClassName);
Database db = index.getDatabase();
EntityBinding binding = index.getEntityBinding();
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
Cursor readCursor = db.openCursor(null, CursorConfig.READ_UNCOMMITTED);
try {
while (readCursor.getNext(key, data, null) == OperationStatus.SUCCESS) {
if (evolveNeeded(key, data, binding)) {
Transaction txn = null;
if (dbConfig.getTransactional()) {
boolean success = false;
txn = env.beginTransaction(null, null);
}
boolean doCommit = false;
Cursor writeCursor = null;
try {
writeCursor = db.openCursor(txn, null);
if (writeCursor.getSearchKey(key, data, LockMode.RMW) == OperationStatus.SUCCESS) {
boolean written = false;
if (evolveNeeded(key, data, binding)) {
writeCursor.putCurrent(data);
written = true;
}
if (listener != null) {
EvolveInternal.updateEvent(event, entityClassName, 1, written ? 1 : 0);
if (!listener.evolveProgress(event)) {
break;
}
}
}
} finally {
if (writeCursor != null) {
writeCursor.close();
}
if (txn != null) {
if (doCommit) {
txn.commit();
} else {
txn.abort();
}
}
}
}
}
} finally {
readCursor.close();
}
}