public void getNextURLs(int max, List<WebURL> result) { while (true) { synchronized (mutex) { if (isFinished) { return; } try { List<WebURL> curResults = workQueues.get(max); workQueues.delete(curResults.size()); if (inProcessPages != null) { for (WebURL curPage : curResults) { inProcessPages.put(curPage); } } result.addAll(curResults); } catch (DatabaseException e) { logger.error("Error while getting next urls: " + e.getMessage()); e.printStackTrace(); } if (result.size() > 0) { return; } } try { synchronized (waitingList) { waitingList.wait(); } } catch (InterruptedException ignored) { // Do nothing } if (isFinished) { return; } } }
public void doHook() { try { EnvironmentImpl envImpl = DbInternal.getEnvironmentImpl(env); SingleItemEntry<CheckpointStart> startEntry = SingleItemEntry.create(LogEntryType.LOG_CKPT_START, new CheckpointStart(100, "test")); long checkpointStart = envImpl.getLogManager().log(startEntry, ReplicationContext.NO_REPLICATE); CheckpointEnd ckptEnd = new CheckpointEnd( "test", checkpointStart, envImpl.getRootLsn(), envImpl.getTxnManager().getFirstActiveLsn(), envImpl.getNodeSequence().getLastLocalNodeId(), envImpl.getNodeSequence().getLastReplicatedNodeId(), envImpl.getDbTree().getLastLocalDbId(), envImpl.getDbTree().getLastReplicatedDbId(), envImpl.getTxnManager().getLastLocalTxnId(), envImpl.getTxnManager().getLastReplicatedTxnId(), 100, true /*cleanedFilesToDelete*/, envImpl.getCleaner().getLogSummary()); SingleItemEntry<CheckpointEnd> endEntry = SingleItemEntry.create(LogEntryType.LOG_CKPT_END, ckptEnd); envImpl .getLogManager() .logForceFlush( endEntry, true, // fsyncRequired ReplicationContext.NO_REPLICATE); } catch (DatabaseException e) { fail(e.getMessage()); } }
public Frontier(Environment env, CrawlConfig config, DocIDServer docIdServer) { super(config); this.counters = new Counters(env, config); this.docIdServer = docIdServer; try { workQueues = new WorkQueues(env, "PendingURLsDB", config.isResumableCrawling()); if (config.isResumableCrawling()) { scheduledPages = counters.getValue(ReservedCounterNames.SCHEDULED_PAGES); inProcessPages = new InProcessPagesDB(env); long numPreviouslyInProcessPages = inProcessPages.getLength(); if (numPreviouslyInProcessPages > 0) { logger.info("Rescheduling " + numPreviouslyInProcessPages + " URLs from previous crawl."); scheduledPages -= numPreviouslyInProcessPages; while (true) { List<WebURL> urls = inProcessPages.get(100); if (urls.size() == 0) { break; } scheduleAll(urls); inProcessPages.delete(urls.size()); } } } else { inProcessPages = null; scheduledPages = 0; } } catch (DatabaseException e) { logger.error("Error while initializing the Frontier: " + e.getMessage()); workQueues = null; } }
public void close() { try { urlsDB.close(); } catch (DatabaseException e) { e.printStackTrace(); } }
private void openEnvironment(File file) throws DatabaseInitException { EnvironmentConfig envConfig = new EnvironmentConfig(); envConfig.setCachePercent(30); envConfig.setAllowCreate(true); envConfig.setTransactional(true); envConfig.setTxnTimeout(10, TimeUnit.SECONDS); envConfig.setLockTimeout(2000, TimeUnit.MILLISECONDS); try { environment = new Environment(file, envConfig); transactionConfig = new TransactionConfig(); transactionConfig.setReadCommitted(true); cursorConfig = new CursorConfig(); cursorConfig.setReadCommitted(true); } catch (EnvironmentLockedException e) { String message = "Environment locked exception. Another process is using the same database, or the current user has no write access (database location: \"" + file.getAbsolutePath() + "\")"; throw new DatabaseInitException(message); } catch (DatabaseException e) { String message = "A database initialisation error has occured (" + e.getMessage() + ")"; throw new DatabaseInitException(message); } }
public void remove(String csName) { try { m_env.removeDatabase(null, csName); } catch (DatabaseException e) { e.printStackTrace(); } m_env = null; }
/** Closes the database. */ @Override public void close() { try { database.close(); } catch (DatabaseException e) { LOGGER.error("Ignored: Could not close database: " + e.getMessage()); } }
private static void attemptClose(Cursor cursor) { try { if (cursor != null) cursor.close(); } catch (DatabaseException e) { logger.error("Error closing cursor.", e); throw new PersistenceFailureException(e.getMessage(), e); } }
/** * Method used by BdbFrontier during checkpointing. * * <p>The backing bdbje database has been marked deferred write so we save on writes to disk. * Means no guarantees disk will have whats in memory unless a sync is called (Calling sync on the * bdbje Environment is not sufficent). * * <p>Package access only because only Frontiers of this package would ever need access. * * @see <a href="http://www.sleepycat.com/jedocs/GettingStartedGuide/DB.html">Deferred Write * Databases</a> */ void sync() { if (this.pendingUrisDB == null) { return; } try { this.pendingUrisDB.sync(); } catch (DatabaseException e) { e.printStackTrace(); } }
/* (non-Javadoc) * @see de.tudarmstadt.ukp.dkpro.ir.util.berkeleydb.BerkeleyDbEnvironmentResource#close() */ public void close() { if (environment != null) { try { environment.close(); } catch (DatabaseException e) { // TODO use logger instead of out e.printStackTrace(); } } }
protected void deleteItem(final WorkQueueFrontier frontier, final CrawlURI peekItem) throws IOException { try { final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier).getWorkQueues(); queues.delete(peekItem); } catch (DatabaseException e) { e.printStackTrace(); throw IoUtils.wrapAsIOException(e); } }
private static void closeEnv() { System.out.println("Closing env and databases"); if (myDb != null) { try { myDb.close(); } catch (DatabaseException e) { System.err.println("closeEnv: myDb: " + e.toString()); e.printStackTrace(); } } if (myClassDb != null) { try { myClassDb.close(); } catch (DatabaseException e) { System.err.println("closeEnv: myClassDb: " + e.toString()); e.printStackTrace(); } } if (myEnv != null) { try { myEnv.close(); } catch (DatabaseException e) { System.err.println("closeEnv: " + e.toString()); e.printStackTrace(); } } }
public static void main(String args[]) { ExampleInventoryRead eir = new ExampleInventoryRead(); try { eir.run(args); } catch (DatabaseException dbe) { System.err.println("ExampleInventoryRead: " + dbe.toString()); dbe.printStackTrace(); } finally { myDbEnv.close(); } System.out.println("All done."); }
public BtreeFile createBtreeFile(String csName) // , boolean bCanSortMultiThreads) { // Now open, or create and open, the database try { Database bdb = m_env.openDatabase(null, csName, m_DbConfig); BtreeFile btreeFile = new BtreeFile(bdb); // , bCanSortMultiThreads); return btreeFile; } catch (DatabaseException e) { e.printStackTrace(); return null; } }
/** * Set an attribute value for the given environment. * * @param targetEnv The target JE environment. May be null if the environment is not open. * @param attribute name/value pair */ public void setAttribute(Environment targetEnv, Attribute attribute) throws AttributeNotFoundException, InvalidAttributeValueException { if (attribute == null) { throw new AttributeNotFoundException("Attribute cannot be null"); } /* Sanity check parameters. */ String name = attribute.getName(); Object value = attribute.getValue(); if (name == null) { throw new AttributeNotFoundException("Attribute name cannot be null"); } if (value == null) { throw new InvalidAttributeValueException( "Attribute value for attribute " + name + " cannot be null"); } try { if (name.equals(ATT_SET_READ_ONLY)) { openConfig.setReadOnly(((Boolean) value).booleanValue()); } else if (name.equals(ATT_SET_TRANSACTIONAL)) { openConfig.setTransactional(((Boolean) value).booleanValue()); } else if (name.equals(ATT_SET_SERIALIZABLE)) { openConfig.setTxnSerializableIsolation(((Boolean) value).booleanValue()); } else { /* Set the specified attribute if the environment is open. */ if (targetEnv != null) { EnvironmentMutableConfig config = targetEnv.getMutableConfig(); if (name.equals(ATT_CACHE_SIZE)) { config.setCacheSize(((Long) value).longValue()); targetEnv.setMutableConfig(config); } else if (name.equals(ATT_CACHE_PERCENT)) { config.setCachePercent(((Integer) value).intValue()); targetEnv.setMutableConfig(config); } else { throw new AttributeNotFoundException("attribute " + name + " is not valid."); } } else { throw new AttributeNotFoundException("attribute " + name + " is not valid."); } } } catch (NumberFormatException e) { throw new InvalidAttributeValueException("attribute name=" + name); } catch (DatabaseException e) { throw new InvalidAttributeValueException("attribute name=" + name + e.getMessage()); } }
public void sync() { if (resumable) { return; } if (urlsDB == null) { return; } try { urlsDB.sync(); } catch (DatabaseException e) { e.printStackTrace(); } }
public static StateStore.Factory deserialize(Element store) throws XMLParseException { String type = store.getAttribute("type"); if (type.compareToIgnoreCase("hash") == 0) { return HashTableStore.Factory.deserialize(store); } else if (type.compareToIgnoreCase("bdb") == 0) { try { return BerkeleyDBStore.Factory.deserialize(store); } catch (DatabaseException e) { throw new XMLParseException(e.getMessage(), e.getCause()); } } throw new XMLParseException("Unrecognized state store type \'" + type + "\'", store); }
public static void main(String[] args) { Environment env = null; Database db = null; EnvironmentConfig envconfig = new EnvironmentConfig(); envconfig.setAllowCreate(true); try { env = new Environment(new File("D://bdb"), envconfig); DatabaseConfig dbconfig = new DatabaseConfig(); dbconfig.setAllowCreate(true); db = env.openDatabase(null, "dbac.db", dbconfig); String key = "mykey"; DatabaseEntry thekey = new DatabaseEntry(); thekey.setData(key.getBytes("utf-8")); Long value = new Long(123456); DatabaseEntry thevalue = new DatabaseEntry(); EntryBinding myBinging = TupleBinding.getPrimitiveBinding(Long.class); myBinging.objectToEntry(value, thevalue); // LongBinding myLongBinging=(LongBinding)TupleBinding.getPrimitiveBinding(Long.class); // myLongBinging.objectToEntry(value, thevalue); db.put(null, thekey, thevalue); DatabaseEntry valueEntry = new DatabaseEntry(); OperationStatus status = db.get(null, thekey, valueEntry, LockMode.DEFAULT); if (status == OperationStatus.SUCCESS) { // Long number=myLongBinging.entryToObject(valueEntry); Long number = (Long) myBinging.entryToObject(valueEntry); System.out.println(env.getDatabaseNames()); System.out.println(number); } } catch (EnvironmentLockedException e) { e.printStackTrace(); } catch (Exception e) { e.printStackTrace(); } finally { if (db != null) { try { db.close(); } catch (DatabaseException e) { e.printStackTrace(); } } if (env != null) { try { env.cleanLog(); env.close(); } catch (DatabaseException e) { e.printStackTrace(); } } } }
/** * Invoke an operation for the given environment. * * @param targetEnv The target JE environment. May be null if the environment is not open. * @param actionName operation name. * @param params operation parameters. May be null. * @param signature operation signature. May be null. * @return the operation result */ public Object invoke( Environment targetEnv, String actionName, Object[] params, String[] signature) throws MBeanException { /* Sanity checking. */ if (actionName == null) { throw new IllegalArgumentException("actionName cannot be null"); } try { if (targetEnv != null) { if (actionName.equals(OP_CLEAN)) { int numFiles = targetEnv.cleanLog(); return new Integer(numFiles); } else if (actionName.equals(OP_EVICT)) { targetEnv.evictMemory(); return null; } else if (actionName.equals(OP_CHECKPOINT)) { CheckpointConfig config = new CheckpointConfig(); if ((params != null) && (params.length > 0)) { Boolean force = (Boolean) params[0]; config.setForce(force.booleanValue()); } targetEnv.checkpoint(config); return null; } else if (actionName.equals(OP_SYNC)) { targetEnv.sync(); return null; } else if (actionName.equals(OP_ENV_STAT)) { return targetEnv.getStats(getStatsConfig(params)); } else if (actionName.equals(OP_LOCK_STAT)) { return targetEnv.getLockStats(getStatsConfig(params)); } else if (actionName.equals(OP_TXN_STAT)) { return targetEnv.getTransactionStats(getStatsConfig(params)); } else if (actionName.equals(OP_DB_NAMES)) { return targetEnv.getDatabaseNames(); } else if (actionName.equals(OP_DB_STAT)) { return getDatabaseStats(targetEnv, params); } } return new IllegalArgumentException("actionName: " + actionName + " is not valid"); } catch (DatabaseException e) { /* * Add both the message and the exception for easiest * deciphering of the problem. Sometimes the original exception * stacktrace gets hidden in server logs. */ throw new MBeanException(e, e.getMessage()); } }
/** {@inheritDoc} */ @Override() public void finalizeBackend() { // Deregister as a change listener. cfg.removeLocalDBChangeListener(this); // Deregister our base DNs. for (DN dn : rootContainer.getBaseDNs()) { try { DirectoryServer.deregisterBaseDN(dn); } catch (Exception e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } } DirectoryServer.deregisterMonitorProvider(rootContainerMonitor); DirectoryServer.deregisterMonitorProvider(diskMonitor); // We presume the server will prevent more operations coming into this // backend, but there may be existing operations already in the // backend. We need to wait for them to finish. waitUntilQuiescent(); // Close the database. try { rootContainer.close(); rootContainer = null; } catch (DatabaseException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } Message message = ERR_JEB_DATABASE_EXCEPTION.get(e.getMessage()); logError(message); } // Checksum this db environment and register its offline state id/checksum. DirectoryServer.registerOfflineBackendStateID(this.getBackendID(), checksumDbEnv()); // Deregister the alert generator. DirectoryServer.deregisterAlertGenerator(this); // Make sure the thread counts are zero for next initialization. threadTotalCount.set(0); threadWriteCount.set(0); // Log an informational message. Message message = NOTE_BACKEND_OFFLINE.get(cfg.getBackendId()); logError(message); }
private RootContainer initializeRootContainer(EnvironmentConfig envConfig) throws ConfigException, InitializationException { // Open the database environment try { RootContainer rc = new RootContainer(this, cfg); rc.open(envConfig); return rc; } catch (DatabaseException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } Message message = ERR_JEB_OPEN_ENV_FAIL.get(e.getMessage()); throw new InitializationException(message, e); } }
public Groundhog getGroundhog(String groundhogName) { Groundhog groundhog = null; File file = new File(baseDirectoryPath + groundhogName); if (file.isDirectory()) { try { groundhog = new Groundhog(file, defaultCacheSize); return groundhog; } catch (DatabaseException e) { // TODO Auto-generated catch block e.printStackTrace(); } } return groundhog; }
// Close the environment public boolean close() { if (m_env != null) { try { // StatsConfig config = new StatsConfig(); // config.setClear(true); // System.err.println(m_env.getStats(config)); // Finally, close the environment. m_env.close(); } catch (DatabaseException dbe) { System.err.println("Error closing MyDbEnv: " + dbe.toString()); return false; } } return true; }
public void close() { if (myEnv != null) { try { Iterator<Entry<String, Database>> it = myDbHashHandle.entrySet().iterator(); while (it.hasNext()) { Database thisDb = it.next().getValue(); if (thisDb != null) { thisDb.close(); } } myEnv.close(); } catch (DatabaseException dbe) { System.err.println("Error closing Envorionment" + dbe.toString()); } } }
/** Main */ public static void main(String argv[]) { if (argv.length < 2) { usage(); return; } File envHomeDirectory = new File(argv[0]); boolean doInsertArg = false; if (argv[1].equalsIgnoreCase("insert")) { doInsertArg = true; } else if (argv[1].equalsIgnoreCase("retrieve")) { doInsertArg = false; } else { usage(); } int startOffset = 0; int numRecordsVal = 0; if (doInsertArg) { if (argv.length > 2) { numRecordsVal = Integer.parseInt(argv[2]); } else { usage(); return; } if (argv.length > 3) { startOffset = Integer.parseInt(argv[3]); } } try { BindingExample app = new BindingExample(numRecordsVal, doInsertArg, envHomeDirectory, startOffset); app.run(); } catch (DatabaseException e) { e.printStackTrace(); System.exit(EXIT_FAILURE); } System.exit(EXIT_SUCCESS); }
/** * Tests that an exception is thrown when a log header is read with a newer version than the * current version. The maxversion.jdb log file is loaded as a resource by this test and written * as a regular log file. When the environment is opened, we expect a LogException. */ public void testGreaterVersionNotAllowed() throws DatabaseException, IOException { TestUtils.loadLog(getClass(), Utils.MAX_VERSION_NAME, envHome); EnvironmentConfig envConfig = TestUtils.initEnvConfig(); envConfig.setAllowCreate(false); envConfig.setTransactional(true); try { Environment env = new Environment(envHome, envConfig); try { env.close(); } catch (Exception ignore) { } } catch (DatabaseException e) { if (e.getCause() instanceof LogException) { /* Got LogException as expected. */ return; } } fail("Expected LogException"); }
/** * Creates a customized DirectoryException from the DatabaseException thrown by JE backend. * * @param e The DatabaseException to be converted. * @return DirectoryException created from exception. */ DirectoryException createDirectoryException(DatabaseException e) { ResultCode resultCode = DirectoryServer.getServerErrorResultCode(); Message message; if ((e instanceof EnvironmentFailureException) && !rootContainer.isValid()) { message = NOTE_BACKEND_ENVIRONMENT_UNUSABLE.get(getBackendID()); logError(message); DirectoryServer.sendAlertNotification( DirectoryServer.getInstance(), ALERT_TYPE_BACKEND_ENVIRONMENT_UNUSABLE, message); } String jeMessage = e.getMessage(); if (jeMessage == null) { jeMessage = stackTraceToSingleLineString(e); } message = ERR_JEB_DATABASE_EXCEPTION.get(jeMessage); return new DirectoryException(resultCode, message, e); }
private void doClose(boolean doCheckpoint) throws DatabaseException { StringBuffer errors = new StringBuffer(); try { // refined trace Tracer.trace(Level.FINE, this, "Close of // environment " + envHome + " started"); try { envState.checkState(DbEnvState.VALID_FOR_CLOSE, DbEnvState.CLOSED); } catch (DatabaseException DBE) { throw DBE; } /* * Begin shutdown of the deamons before checkpointing. Cleaning * during the checkpoint is wasted and slows down the checkpoint. */ requestShutdownDaemons(); /* Checkpoint to bound recovery time. */ if (doCheckpoint && !isReadOnly && (envState != DbEnvState.INVALID) && logManager.getLastLsnAtRecovery() != fileManager.getLastUsedLsn()) { /* * Force a checkpoint. Don't allow deltas (minimize recovery * time) because they cause inefficiencies for two reasons: (1) * recovering BINDeltas causes extra random I/O in order to * reconstitute BINS, which can greatly increase recovery time, * and (2) logging deltas during close causes redundant logging * by the full checkpoint after recovery. */ CheckpointConfig ckptConfig = new CheckpointConfig(); ckptConfig.setForce(true); ckptConfig.setMinimizeRecoveryTime(true); try { invokeCheckpoint( ckptConfig, false, // flushAll "close"); } catch (DatabaseException IE) { errors.append("\nException performing checkpoint: "); errors.append(IE.toString()).append("\n"); } } try { shutdownDaemons(); } catch (InterruptedException IE) { errors.append("\nException shutting down daemon threads: "); errors.append(IE.toString()).append("\n"); } /* Flush log. */ // refined trace Tracer.trace(Level.FINE, this, "Env " + envHome + " // daemons shutdown"); try { logManager.flush(); } catch (DatabaseException DBE) { errors.append("\nException flushing log manager: "); errors.append(DBE.toString()).append("\n"); } try { fileManager.clear(); } catch (IOException IOE) { errors.append("\nException clearing file manager: "); errors.append(IOE.toString()).append("\n"); } catch (DatabaseException DBE) { errors.append("\nException clearing file manager: "); errors.append(DBE.toString()).append("\n"); } try { fileManager.close(); } catch (IOException IOE) { errors.append("\nException clearing file manager: "); errors.append(IOE.toString()).append("\n"); } catch (DatabaseException DBE) { errors.append("\nException clearing file manager: "); errors.append(DBE.toString()).append("\n"); } try { inMemoryINs.clear(); } catch (DatabaseException DBE) { errors.append("\nException closing file manager: "); errors.append(DBE.toString()).append("\n"); } hook_afterDoClose(errors); } finally { envState = DbEnvState.CLOSED; } if (errors.length() > 0 && savedInvalidatingException == null) { /* Don't whine again if we've already whined. */ throw new RunRecoveryException(this, errors.toString()); } }
/** {@inheritDoc} */ @Override() public void initializeBackend() throws ConfigException, InitializationException { // Checksum this db environment and register its offline state id/checksum. DirectoryServer.registerOfflineBackendStateID(this.getBackendID(), checksumDbEnv()); if (rootContainer == null) { EnvironmentConfig envConfig = ConfigurableEnvironment.parseConfigEntry(cfg); rootContainer = initializeRootContainer(envConfig); } // Preload the database cache. rootContainer.preload(cfg.getPreloadTimeLimit()); try { // Log an informational message about the number of entries. Message message = NOTE_JEB_BACKEND_STARTED.get(cfg.getBackendId(), rootContainer.getEntryCount()); logError(message); } catch (DatabaseException databaseException) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, databaseException); } Message message = WARN_JEB_GET_ENTRY_COUNT_FAILED.get(databaseException.getMessage()); throw new InitializationException(message, databaseException); } for (DN dn : cfg.getBaseDN()) { try { DirectoryServer.registerBaseDN(dn, this, false); } catch (Exception e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } Message message = ERR_BACKEND_CANNOT_REGISTER_BASEDN.get(String.valueOf(dn), String.valueOf(e)); throw new InitializationException(message, e); } } // Register a monitor provider for the environment. rootContainerMonitor = rootContainer.getMonitorProvider(); DirectoryServer.registerMonitorProvider(rootContainerMonitor); // Register as disk space monitor handler File parentDirectory = getFileForPath(cfg.getDBDirectory()); File backendDirectory = new File(parentDirectory, cfg.getBackendId()); diskMonitor = new DiskSpaceMonitor( getBackendID() + " backend", backendDirectory, cfg.getDiskLowThreshold(), cfg.getDiskFullThreshold(), 5, TimeUnit.SECONDS, this); diskMonitor.initializeMonitorProvider(null); DirectoryServer.registerMonitorProvider(diskMonitor); // Register as an AlertGenerator. DirectoryServer.registerAlertGenerator(this); // Register this backend as a change listener. cfg.addLocalDBChangeListener(this); }