/** Run a checkpoint of the crawler */
public synchronized String requestCrawlCheckpoint() throws IllegalStateException {
if (isCheckpointing()) {
throw new IllegalStateException("Checkpoint already running.");
}
// prevent redundant auto-checkpoints when crawler paused
if (controller.isPaused()) {
if (controller.getStatisticsTracker().getSnapshot().sameProgressAs(lastCheckpointSnapshot)) {
LOGGER.info("no progress since last checkpoint; ignoring");
System.err.println("no progress since last checkpoint; ignoring");
return null;
}
}
Map<String, Checkpointable> toCheckpoint = appCtx.getBeansOfType(Checkpointable.class);
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("checkpointing beans " + toCheckpoint);
}
checkpointInProgress = new Checkpoint();
try {
checkpointInProgress.generateFrom(getCheckpointsDir(), getNextCheckpointNumber());
// pre (incl. acquire necessary locks)
// long startMs = System.currentTimeMillis();
for (Checkpointable c : toCheckpoint.values()) {
c.startCheckpoint(checkpointInProgress);
}
// long duration = System.currentTimeMillis() - startMs;
// System.err.println("all startCheckpoint() completed in "+duration+"ms");
// flush/write
for (Checkpointable c : toCheckpoint.values()) {
// long doMs = System.currentTimeMillis();
c.doCheckpoint(checkpointInProgress);
// long doDuration = System.currentTimeMillis() - doMs;
// System.err.println("doCheckpoint() "+c+" in "+doDuration+"ms");
}
checkpointInProgress.setSuccess(true);
appCtx.publishEvent(new CheckpointSuccessEvent(this, checkpointInProgress));
} catch (Exception e) {
checkpointFailed(e);
} finally {
checkpointInProgress.writeValidity(controller.getStatisticsTracker().getProgressStamp());
lastCheckpointSnapshot = controller.getStatisticsTracker().getSnapshot();
// close (incl. release locks)
for (Checkpointable c : toCheckpoint.values()) {
c.finishCheckpoint(checkpointInProgress);
}
}
this.nextCheckpointNumber++;
LOGGER.info("finished checkpoint " + checkpointInProgress.getName());
String nameToReport = checkpointInProgress.getSuccess() ? checkpointInProgress.getName() : null;
this.checkpointInProgress = null;
return nameToReport;
}
/**
* Given the name of a valid checkpoint subdirectory in the checkpoints directory, create a
* Checkpoint instance, and insert it into all Checkpointable beans.
*
* @param selectedCheckpoint
*/
public synchronized void setRecoveryCheckpointByName(String selectedCheckpoint) {
if (isRunning) {
throw new RuntimeException("may not set recovery Checkpoint after launch");
}
Checkpoint recoveryCheckpoint = new Checkpoint();
recoveryCheckpoint.getCheckpointDir().setBase(getCheckpointsDir());
recoveryCheckpoint.getCheckpointDir().setPath(selectedCheckpoint);
recoveryCheckpoint.getCheckpointDir().setConfigurer(appCtx.getBean(ConfigPathConfigurer.class));
recoveryCheckpoint.afterPropertiesSet();
setRecoveryCheckpoint(recoveryCheckpoint);
Map<String, Checkpointable> toSetRecovery = appCtx.getBeansOfType(Checkpointable.class);
for (Checkpointable c : toSetRecovery.values()) {
c.setRecoveryCheckpoint(recoveryCheckpoint);
}
}