/**
* Cleanup a file after it has been successfully processed.
*
* <p>This can through both IOExceptions and runtime exceptions due to Preconditions failures.
*
* <p>According to the link below, Solaris (I assume POSIX/linux) does atomic rename but Windows
* does not guarantee it. http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4017593 To be truly
* correct, I need to check the return value (will likely fail in unix if moving from one volume
* to another instead of just within same volume)
*/
synchronized void changeState(String tag, State oldState, State newState) throws IOException {
DFOData data = table.get(tag);
Preconditions.checkArgument(data != null, "Tag " + tag + " has no data");
Preconditions.checkArgument(
tag.equals(data.tag), "Data associated with tag didn't match tag " + tag);
if (LOG.isDebugEnabled()) {
LOG.debug("Change " + data.s + "/" + oldState + " to " + newState + " : " + tag);
}
// null allows any previous state.
if (oldState == null) {
oldState = data.s;
}
Preconditions.checkState(
data.s == oldState, "Expected state to be " + oldState + " but was " + data.s);
if (oldState == State.ERROR) {
throw new IllegalStateException("Cannot move from error state");
}
// SENT is terminal state, no where to move, just delete it.
if (newState == State.SENT) {
getQueue(oldState).remove(tag);
File sentFile = getFile(tag);
data.s = newState;
if (!sentFile.delete()) {
LOG.error("Couldn't delete " + sentFile + " - can be safely manually deleted");
}
// TODO (jon) need to eventually age off sent files entry to not exhaust
// memory
return;
}
// move files to other directories to making state change durable.
File orig = getFile(tag);
File newf = new File(getDir(newState), tag);
boolean success = orig.renameTo(newf);
if (!success) {
throw new IOException("Move " + orig + " -> " + newf + "failed!");
}
// is successful, update queues.
LOG.debug("old state is " + oldState);
getQueue(oldState).remove(tag);
BlockingQueue<String> q = getQueue(newState);
if (q != null) {
q.add(tag);
}
data.s = newState;
}
/** This is a hook that imports external files to the dfo bypassing the default append */
public synchronized void importData() throws IOException {
// move all writing into the logged dir.
for (String fn : importDir.list()) {
// add to logging queue
DFOData data = DFOData.recovered(fn);
synchronized (this) {
table.put(fn, data);
loggedQ.add(fn);
importedCount.incrementAndGet();
}
}
}
/**
* This looks at directory structure and recovers state based on where files are in the file
* system.
*
* <p>For a first cut, we will just get everything into the logged state and restart from there.
* Optimizations will recover at finer grain and be more performant.
*/
public synchronized void recover() throws IOException {
// move all writing into the logged dir.
for (String f : writingDir.list()) {
File old = new File(writingDir, f);
if (!old.isFile() || !old.renameTo(new File(loggedDir, f))) {
throw new IOException(
"Unable to recover - couldn't rename " + old + " to " + loggedDir + f);
}
LOG.debug("Recover moved " + f + " from WRITING to LOGGED");
}
// move all sending into the logged dir
for (String f : sendingDir.list()) {
File old = new File(sendingDir, f);
if (!old.isFile() || !old.renameTo(new File(loggedDir, f))) {
throw new IOException(
"Unable to recover - couldn't rename " + old + " to " + loggedDir + f);
}
LOG.debug("Recover moved " + f + " from SENDING to LOGGED");
}
// add all logged to loggedQ and table
for (String f : loggedDir.list()) {
// File log = new File(loggedDir, f);
DFOData data = DFOData.recovered(f);
table.put(f, data);
loggedQ.add(f);
recoverCount.incrementAndGet();
LOG.debug("Recover loaded " + f);
}
// carry on now on your merry way.
}
static DFOData recovered(String tag) {
DFOData data = new DFOData(tag);
data.s = State.LOGGED;
return data;
}
