/**
* Given the name of a valid checkpoint subdirectory in the checkpoints directory, create a
* Checkpoint instance, and insert it into all Checkpointable beans.
*
* @param selectedCheckpoint
*/
public synchronized void setRecoveryCheckpointByName(String selectedCheckpoint) {
if (isRunning) {
throw new RuntimeException("may not set recovery Checkpoint after launch");
}
Checkpoint recoveryCheckpoint = new Checkpoint();
recoveryCheckpoint.getCheckpointDir().setBase(getCheckpointsDir());
recoveryCheckpoint.getCheckpointDir().setPath(selectedCheckpoint);
recoveryCheckpoint.getCheckpointDir().setConfigurer(appCtx.getBean(ConfigPathConfigurer.class));
recoveryCheckpoint.afterPropertiesSet();
setRecoveryCheckpoint(recoveryCheckpoint);
Map<String, Checkpointable> toSetRecovery = appCtx.getBeansOfType(Checkpointable.class);
for (Checkpointable c : toSetRecovery.values()) {
c.setRecoveryCheckpoint(recoveryCheckpoint);
}
}
/** Run a checkpoint of the crawler */
public synchronized String requestCrawlCheckpoint() throws IllegalStateException {
if (isCheckpointing()) {
throw new IllegalStateException("Checkpoint already running.");
}
// prevent redundant auto-checkpoints when crawler paused
if (controller.isPaused()) {
if (controller.getStatisticsTracker().getSnapshot().sameProgressAs(lastCheckpointSnapshot)) {
LOGGER.info("no progress since last checkpoint; ignoring");
System.err.println("no progress since last checkpoint; ignoring");
return null;
}
}
Map<String, Checkpointable> toCheckpoint = appCtx.getBeansOfType(Checkpointable.class);
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("checkpointing beans " + toCheckpoint);
}
checkpointInProgress = new Checkpoint();
try {
checkpointInProgress.generateFrom(getCheckpointsDir(), getNextCheckpointNumber());
// pre (incl. acquire necessary locks)
// long startMs = System.currentTimeMillis();
for (Checkpointable c : toCheckpoint.values()) {
c.startCheckpoint(checkpointInProgress);
}
// long duration = System.currentTimeMillis() - startMs;
// System.err.println("all startCheckpoint() completed in "+duration+"ms");
// flush/write
for (Checkpointable c : toCheckpoint.values()) {
// long doMs = System.currentTimeMillis();
c.doCheckpoint(checkpointInProgress);
// long doDuration = System.currentTimeMillis() - doMs;
// System.err.println("doCheckpoint() "+c+" in "+doDuration+"ms");
}
checkpointInProgress.setSuccess(true);
appCtx.publishEvent(new CheckpointSuccessEvent(this, checkpointInProgress));
} catch (Exception e) {
checkpointFailed(e);
} finally {
checkpointInProgress.writeValidity(controller.getStatisticsTracker().getProgressStamp());
lastCheckpointSnapshot = controller.getStatisticsTracker().getSnapshot();
// close (incl. release locks)
for (Checkpointable c : toCheckpoint.values()) {
c.finishCheckpoint(checkpointInProgress);
}
}
this.nextCheckpointNumber++;
LOGGER.info("finished checkpoint " + checkpointInProgress.getName());
String nameToReport = checkpointInProgress.getSuccess() ? checkpointInProgress.getName() : null;
this.checkpointInProgress = null;
return nameToReport;
}
public synchronized void start() {
if (isRunning) {
return;
}
// report if checkpoint incomplete/invalid
if (getRecoveryCheckpoint() != null) {
File cpDir = getRecoveryCheckpoint().getCheckpointDir().getFile();
if (!Checkpoint.hasValidStamp(cpDir)) {
LOGGER.severe(
"checkpoint '"
+ cpDir.getAbsolutePath()
+ "' missing validity stamp file; checkpoint data "
+ "may be missing or otherwise corrupt.");
}
}
this.isRunning = true;
setupCheckpointTask();
}
/**
* Returns a list of available, valid (contains 'valid' file) checkpoint directories, as File
* instances, with the more recently-written appearing first.
*
* @return List of valid checkpoint directory File instances
*/
@SuppressWarnings("unchecked")
public List<File> findAvailableCheckpointDirectories() {
File[] dirs =
getCheckpointsDir().getFile().listFiles((FileFilter) FileFilterUtils.directoryFileFilter());
if (dirs == null) {
return Collections.EMPTY_LIST;
}
Arrays.sort(dirs, LastModifiedFileComparator.LASTMODIFIED_REVERSE);
LinkedList<File> dirsList = new LinkedList<File>(Arrays.asList(dirs));
Iterator<File> iter = dirsList.iterator();
while (iter.hasNext()) {
File cpDir = iter.next();
if (!Checkpoint.hasValidStamp(cpDir)) {
LOGGER.warning("checkpoint '" + cpDir + "' missing validity stamp file; ignoring");
iter.remove();
}
}
return dirsList;
}
@Autowired(required = false)
public void setRecoveryCheckpoint(Checkpoint checkpoint) {
this.recoveryCheckpoint = checkpoint;
checkpoint.getCheckpointDir().setBase(getCheckpointsDir());
}
