/** Run a checkpoint of the crawler */
public synchronized String requestCrawlCheckpoint() throws IllegalStateException {
if (isCheckpointing()) {
throw new IllegalStateException("Checkpoint already running.");
}
// prevent redundant auto-checkpoints when crawler paused
if (controller.isPaused()) {
if (controller.getStatisticsTracker().getSnapshot().sameProgressAs(lastCheckpointSnapshot)) {
LOGGER.info("no progress since last checkpoint; ignoring");
System.err.println("no progress since last checkpoint; ignoring");
return null;
}
}
Map<String, Checkpointable> toCheckpoint = appCtx.getBeansOfType(Checkpointable.class);
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("checkpointing beans " + toCheckpoint);
}
checkpointInProgress = new Checkpoint();
try {
checkpointInProgress.generateFrom(getCheckpointsDir(), getNextCheckpointNumber());
// pre (incl. acquire necessary locks)
// long startMs = System.currentTimeMillis();
for (Checkpointable c : toCheckpoint.values()) {
c.startCheckpoint(checkpointInProgress);
}
// long duration = System.currentTimeMillis() - startMs;
// System.err.println("all startCheckpoint() completed in "+duration+"ms");
// flush/write
for (Checkpointable c : toCheckpoint.values()) {
// long doMs = System.currentTimeMillis();
c.doCheckpoint(checkpointInProgress);
// long doDuration = System.currentTimeMillis() - doMs;
// System.err.println("doCheckpoint() "+c+" in "+doDuration+"ms");
}
checkpointInProgress.setSuccess(true);
appCtx.publishEvent(new CheckpointSuccessEvent(this, checkpointInProgress));
} catch (Exception e) {
checkpointFailed(e);
} finally {
checkpointInProgress.writeValidity(controller.getStatisticsTracker().getProgressStamp());
lastCheckpointSnapshot = controller.getStatisticsTracker().getSnapshot();
// close (incl. release locks)
for (Checkpointable c : toCheckpoint.values()) {
c.finishCheckpoint(checkpointInProgress);
}
}
this.nextCheckpointNumber++;
LOGGER.info("finished checkpoint " + checkpointInProgress.getName());
String nameToReport = checkpointInProgress.getSuccess() ? checkpointInProgress.getName() : null;
this.checkpointInProgress = null;
return nameToReport;
}
