@Override public void run() { long latestCount = reader.getEntriesRead() + 0; long deltaCount = latestCount - previousCount; long latestTime = System.currentTimeMillis(); long deltaTime = latestTime - previousTime; if (deltaTime == 0) { return; } long entriesRead = reader.getEntriesRead(); long entriesIgnored = reader.getEntriesIgnored(); long entriesRejected = reader.getEntriesRejected(); float rate = 1000f * deltaCount / deltaTime; logger.info(NOTE_IMPORT_PROGRESS_REPORT, entriesRead, entriesIgnored, entriesRejected, rate); previousCount = latestCount; previousTime = latestTime; }
/** {@inheritDoc} */ @Override() public LDIFImportResult importLDIF(LDIFImportConfig importConfig) throws DirectoryException { RuntimeInformation.logInfo(); // If the backend already has the root container open, we must use the same // underlying root container boolean openRootContainer = rootContainer == null; // If the rootContainer is open, the backend is initialized by something // else. // We can't do import while the backend is online. if (!openRootContainer) { Message message = ERR_JEB_IMPORT_BACKEND_ONLINE.get(); throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message); } try { EnvironmentConfig envConfig = new EnvironmentConfig(); envConfig.setAllowCreate(true); envConfig.setTransactional(false); envConfig.setDurability(Durability.COMMIT_NO_SYNC); envConfig.setLockTimeout(0, TimeUnit.SECONDS); envConfig.setTxnTimeout(0, TimeUnit.SECONDS); envConfig.setConfigParam( EnvironmentConfig.CLEANER_MIN_FILE_UTILIZATION, String.valueOf(cfg.getDBCleanerMinUtilization())); envConfig.setConfigParam( EnvironmentConfig.LOG_FILE_MAX, String.valueOf(cfg.getDBLogFileMax())); if (!importConfig.appendToExistingData()) { if (importConfig.clearBackend() || cfg.getBaseDN().size() <= 1) { // We have the writer lock on the environment, now delete the // environment and re-open it. Only do this when we are // importing to all the base DNs in the backend or if the backend only // have one base DN. File parentDirectory = getFileForPath(cfg.getDBDirectory()); File backendDirectory = new File(parentDirectory, cfg.getBackendId()); // If the backend does not exist the import will create it. if (backendDirectory.exists()) { EnvManager.removeFiles(backendDirectory.getPath()); } } } Importer importer = new Importer(importConfig, cfg, envConfig); rootContainer = initializeRootContainer(envConfig); return importer.processImport(rootContainer); } catch (ExecutionException execEx) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, execEx); } if (execEx.getCause() instanceof DirectoryException) { throw ((DirectoryException) execEx.getCause()); } else { Message message = ERR_EXECUTION_ERROR.get(execEx.getMessage()); throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message); } } catch (InterruptedException intEx) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, intEx); } Message message = ERR_INTERRUPTED_ERROR.get(intEx.getMessage()); throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message); } catch (JebException je) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, je); } throw new DirectoryException( DirectoryServer.getServerErrorResultCode(), je.getMessageObject()); } catch (InitializationException ie) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, ie); } throw new DirectoryException( DirectoryServer.getServerErrorResultCode(), ie.getMessageObject()); } catch (ConfigException ce) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, ce); } throw new DirectoryException( DirectoryServer.getServerErrorResultCode(), ce.getMessageObject()); } finally { // leave the backend in the same state. try { if (rootContainer != null) { long startTime = System.currentTimeMillis(); rootContainer.close(); long finishTime = System.currentTimeMillis(); long closeTime = (finishTime - startTime) / 1000; Message msg = NOTE_JEB_IMPORT_LDIF_ROOTCONTAINER_CLOSE.get(closeTime); logError(msg); rootContainer = null; } // Sync the environment to disk. if (debugEnabled()) { Message message = NOTE_JEB_IMPORT_CLOSING_DATABASE.get(); TRACER.debugInfo(message.toString()); } } catch (DatabaseException de) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, de); } } } }
/** {@inheritDoc} */ @Override public LDIFImportResult importLDIF( LDIFImportConfig importConfig, RootContainer rootContainer, ServerContext serverContext) throws DirectoryException { try { ScheduledThreadPoolExecutor timerService = new ScheduledThreadPoolExecutor(1); try { final LDIFReader reader; try { reader = new LDIFReader(importConfig); } catch (Exception e) { LocalizableMessage m = ERR_LDIF_BACKEND_CANNOT_CREATE_LDIF_READER.get(stackTraceToSingleLineString(e)); throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), m, e); } long importCount = 0; final long startTime = System.currentTimeMillis(); timerService.scheduleAtFixedRate( new ImportProgress(reader), IMPORT_PROGRESS_INTERVAL, IMPORT_PROGRESS_INTERVAL, TimeUnit.MILLISECONDS); while (true) { final Entry entry; try { entry = reader.readEntry(); if (entry == null) { break; } } catch (LDIFException le) { if (!le.canContinueReading()) { LocalizableMessage m = ERR_LDIF_BACKEND_ERROR_READING_LDIF.get(stackTraceToSingleLineString(le)); throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), m, le); } continue; } final DN dn = entry.getName(); final EntryContainer ec = rootContainer.getEntryContainer(dn); if (ec == null) { final LocalizableMessage m = ERR_LDIF_SKIP.get(dn); logger.error(m); reader.rejectLastEntry(m); continue; } try { ec.addEntry(entry, null); importCount++; } catch (DirectoryException e) { switch (e.getResultCode().asEnum()) { case ENTRY_ALREADY_EXISTS: if (importConfig.replaceExistingEntries()) { final Entry oldEntry = ec.getEntry(entry.getName()); ec.replaceEntry(oldEntry, entry, null); } else { reader.rejectLastEntry(WARN_IMPORT_ENTRY_EXISTS.get()); } break; case NO_SUCH_OBJECT: reader.rejectLastEntry(ERR_IMPORT_PARENT_NOT_FOUND.get(dn.parent())); break; default: // Not sure why it failed. reader.rejectLastEntry(e.getMessageObject()); break; } } } final long finishTime = System.currentTimeMillis(); waitForShutdown(timerService); final long importTime = finishTime - startTime; float rate = 0; if (importTime > 0) { rate = 1000f * reader.getEntriesRead() / importTime; } logger.info( NOTE_IMPORT_FINAL_STATUS, reader.getEntriesRead(), importCount, reader.getEntriesIgnored(), reader.getEntriesRejected(), 0, importTime / 1000, rate); return new LDIFImportResult( reader.getEntriesRead(), reader.getEntriesRejected(), reader.getEntriesIgnored()); } finally { rootContainer.close(); // if not already stopped, then stop it waitForShutdown(timerService); } } catch (DirectoryException e) { logger.traceException(e); throw e; } catch (OpenDsException e) { logger.traceException(e); throw new DirectoryException(getServerErrorResultCode(), e.getMessageObject()); } catch (Exception e) { logger.traceException(e); throw new DirectoryException( getServerErrorResultCode(), LocalizableMessage.raw(e.getMessage())); } }
private void testGetOldestNewestCSNs(final int max, final int counterWindow) throws Exception { String tn = "testDBCount(" + max + "," + counterWindow + ")"; debugInfo(tn, "Starting test"); File testRoot = null; ReplicationServer replicationServer = null; ReplicationDbEnv dbEnv = null; JEReplicaDB replicaDB = null; try { TestCaseUtils.startServer(); replicationServer = configureReplicationServer(100000, 10); testRoot = createCleanDir(); dbEnv = new ReplicationDbEnv(testRoot.getPath(), replicationServer); replicaDB = new JEReplicaDB(1, TEST_ROOT_DN, replicationServer, dbEnv); replicaDB.setCounterRecordWindowSize(counterWindow); // Populate the db with 'max' msg int mySeqnum = 1; CSN csns[] = new CSN[2 * (max + 1)]; long now = System.currentTimeMillis(); for (int i = 1; i <= max; i++) { csns[i] = new CSN(now + i, mySeqnum, 1); replicaDB.add(new DeleteMsg(TEST_ROOT_DN, csns[i], "uid")); mySeqnum += 2; } assertEquals(replicaDB.getOldestCSN(), csns[1], "Wrong oldest CSN"); assertEquals(replicaDB.getNewestCSN(), csns[max], "Wrong newest CSN"); // Now we want to test that after closing and reopening the db, the // counting algo is well reinitialized and when new messages are added // the new counter are correctly generated. debugInfo(tn, "SHUTDOWN replicaDB and recreate"); replicaDB.shutdown(); replicaDB = new JEReplicaDB(1, TEST_ROOT_DN, replicationServer, dbEnv); replicaDB.setCounterRecordWindowSize(counterWindow); assertEquals(replicaDB.getOldestCSN(), csns[1], "Wrong oldest CSN"); assertEquals(replicaDB.getNewestCSN(), csns[max], "Wrong newest CSN"); // Populate the db with 'max' msg for (int i = max + 1; i <= 2 * max; i++) { csns[i] = new CSN(now + i, mySeqnum, 1); replicaDB.add(new DeleteMsg(TEST_ROOT_DN, csns[i], "uid")); mySeqnum += 2; } assertEquals(replicaDB.getOldestCSN(), csns[1], "Wrong oldest CSN"); assertEquals(replicaDB.getNewestCSN(), csns[2 * max], "Wrong newest CSN"); replicaDB.purgeUpTo(new CSN(Long.MAX_VALUE, 0, 0)); String testcase = "AFTER PURGE (oldest, newest)="; debugInfo(tn, testcase + replicaDB.getOldestCSN() + replicaDB.getNewestCSN()); assertEquals(replicaDB.getNewestCSN(), csns[2 * max], "Newest="); // Clear ... debugInfo(tn, "clear:"); replicaDB.clear(); // Check the db is cleared. assertEquals(null, replicaDB.getOldestCSN()); assertEquals(null, replicaDB.getNewestCSN()); debugInfo(tn, "Success"); } finally { shutdown(replicaDB); if (dbEnv != null) { dbEnv.shutdown(); } remove(replicationServer); TestCaseUtils.deleteDirectory(testRoot); } }
@DataProvider Object[][] cursorData() { // create 7 csns final CSN[] sevenCsns = generateCSNs(1, System.currentTimeMillis(), 7); CSN beforeCsn = sevenCsns[0]; CSN middleCsn = sevenCsns[3]; // will be between csns[1] and csns[2] CSN afterCsn = sevenCsns[6]; // but use only 4 of them for update msg // beforeCsn, middleCsn and afterCsn are not used // in order to test cursor generation from a key not present in the log (before, in the middle, // after) final List<CSN> usedCsns = new ArrayList<CSN>(Arrays.asList(sevenCsns)); usedCsns.remove(beforeCsn); usedCsns.remove(middleCsn); usedCsns.remove(afterCsn); final CSN[] csns = usedCsns.toArray(new CSN[4]); return new Object[][] { // equal matching {csns, beforeCsn, EQUAL_TO_KEY, ON_MATCHING_KEY, -1, -1}, {csns, csns[0], EQUAL_TO_KEY, ON_MATCHING_KEY, 0, 3}, {csns, csns[1], EQUAL_TO_KEY, ON_MATCHING_KEY, 1, 3}, {csns, middleCsn, EQUAL_TO_KEY, ON_MATCHING_KEY, -1, -1}, {csns, csns[2], EQUAL_TO_KEY, ON_MATCHING_KEY, 2, 3}, {csns, csns[3], EQUAL_TO_KEY, ON_MATCHING_KEY, 3, 3}, {csns, afterCsn, EQUAL_TO_KEY, ON_MATCHING_KEY, -1, -1}, {csns, beforeCsn, EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1}, {csns, csns[0], EQUAL_TO_KEY, AFTER_MATCHING_KEY, 1, 3}, {csns, csns[1], EQUAL_TO_KEY, AFTER_MATCHING_KEY, 2, 3}, {csns, middleCsn, EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1}, {csns, csns[2], EQUAL_TO_KEY, AFTER_MATCHING_KEY, 3, 3}, {csns, csns[3], EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1}, {csns, afterCsn, EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1}, // less than or equal matching {csns, beforeCsn, LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, -1, -1}, {csns, csns[0], LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 0, 3}, {csns, csns[1], LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 1, 3}, {csns, middleCsn, LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 1, 3}, {csns, csns[2], LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 2, 3}, {csns, csns[3], LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 3, 3}, {csns, afterCsn, LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 3, 3}, {csns, beforeCsn, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1}, {csns, csns[0], LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 1, 3}, {csns, csns[1], LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 2, 3}, {csns, middleCsn, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 2, 3}, {csns, csns[2], LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 3, 3}, {csns, csns[3], LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1}, {csns, afterCsn, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1}, // greater than or equal matching {csns, beforeCsn, GREATER_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 0, 3}, {csns, csns[0], GREATER_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 0, 3}, {csns, csns[1], GREATER_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 1, 3}, {csns, middleCsn, GREATER_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 2, 3}, {csns, csns[2], GREATER_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 2, 3}, {csns, csns[3], GREATER_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 3, 3}, {csns, afterCsn, GREATER_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, -1, -1}, {csns, beforeCsn, GREATER_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 0, 3}, {csns, csns[0], GREATER_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 1, 3}, {csns, csns[1], GREATER_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 2, 3}, {csns, middleCsn, GREATER_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 2, 3}, {csns, csns[2], GREATER_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 3, 3}, {csns, csns[3], GREATER_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1}, {csns, afterCsn, GREATER_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1}, {null, null, null, null, -1, -1} // stop line }; }