/**
* Check whether the rolling upgrade marker file exists for each BP storage root. If it does
* exist, then the marker file is cleared and more importantly the layout upgrade is finalized.
*/
public void clearRollingUpgradeMarkers(List<StorageDirectory> dnStorageDirs) throws IOException {
for (StorageDirectory sd : dnStorageDirs) {
File bpRoot = getBpRoot(blockpoolID, sd.getCurrentDir());
File markerFile = new File(bpRoot, ROLLING_UPGRADE_MARKER_FILE);
if (!storagesWithoutRollingUpgradeMarker.contains(bpRoot.toString())) {
if (markerFile.exists()) {
LOG.info("Deleting " + markerFile);
doFinalize(sd.getCurrentDir());
if (!markerFile.delete()) {
LOG.warn("Failed to delete " + markerFile);
}
}
storagesWithoutRollingUpgradeMarker.add(bpRoot.toString());
storagesWithRollingUpgradeMarker.remove(bpRoot.toString());
}
}
}
/**
* Upgrade -- Move current storage into a backup directory, and hardlink all its blocks into the
* new current directory.
*
* <p>Upgrade from pre-0.22 to 0.22 or later release e.g. 0.19/0.20/ => 0.22/0.23
*
* <ul>
* <li>If <SD>/previous exists then delete it
* <li>Rename <SD>/current to <SD>/previous.tmp
* <li>Create new <SD>/current/<bpid>/current directory
* <li>
* <ul>
* <li>Hard links for block files are created from <SD>/previous.tmp to
* <SD>/current/<bpid>/current
* <li>Saves new version file in <SD>/current/<bpid>/current directory
* </ul>
* <li>Rename <SD>/previous.tmp to <SD>/previous
* </ul>
*
* There should be only ONE namenode in the cluster for first time upgrade to 0.22
*
* @param sd storage directory
* @throws IOException on error
*/
void doUpgrade(StorageDirectory sd, NamespaceInfo nsInfo) throws IOException {
if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) {
clusterID = nsInfo.getClusterID();
layoutVersion = nsInfo.getLayoutVersion();
writeProperties(sd);
return;
}
LOG.info(
"Upgrading storage directory "
+ sd.getRoot()
+ ".\n old LV = "
+ this.getLayoutVersion()
+ "; old CTime = "
+ this.getCTime()
+ ".\n new LV = "
+ nsInfo.getLayoutVersion()
+ "; new CTime = "
+ nsInfo.getCTime());
File curDir = sd.getCurrentDir();
File prevDir = sd.getPreviousDir();
File bbwDir = new File(sd.getRoot(), Storage.STORAGE_1_BBW);
assert curDir.exists() : "Data node current directory must exist.";
// Cleanup directory "detach"
cleanupDetachDir(new File(curDir, STORAGE_DIR_DETACHED));
// 1. delete <SD>/previous dir before upgrading
if (prevDir.exists()) deleteDir(prevDir);
// get previous.tmp directory, <SD>/previous.tmp
File tmpDir = sd.getPreviousTmp();
assert !tmpDir.exists() : "Data node previous.tmp directory must not exist.";
// 2. Rename <SD>/current to <SD>/previous.tmp
rename(curDir, tmpDir);
// 3. Format BP and hard link blocks from previous directory
File curBpDir = BlockPoolSliceStorage.getBpRoot(nsInfo.getBlockPoolID(), curDir);
BlockPoolSliceStorage bpStorage =
new BlockPoolSliceStorage(
nsInfo.getNamespaceID(),
nsInfo.getBlockPoolID(),
nsInfo.getCTime(),
nsInfo.getClusterID());
bpStorage.format(curDir, nsInfo);
linkAllBlocks(tmpDir, bbwDir, new File(curBpDir, STORAGE_DIR_CURRENT));
// 4. Write version file under <SD>/current
layoutVersion = HdfsConstants.LAYOUT_VERSION;
clusterID = nsInfo.getClusterID();
writeProperties(sd);
// 5. Rename <SD>/previous.tmp to <SD>/previous
rename(tmpDir, prevDir);
LOG.info("Upgrade of " + sd.getRoot() + " is complete");
addBlockPoolStorage(nsInfo.getBlockPoolID(), bpStorage);
}
private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOException {
LOG.info("Formatting block pool " + blockpoolID + " directory " + bpSdir.getCurrentDir());
bpSdir.clearDirectory(); // create directory
this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
this.cTime = nsInfo.getCTime();
this.namespaceID = nsInfo.getNamespaceID();
this.blockpoolID = nsInfo.getBlockPoolID();
writeProperties(bpSdir);
}
/**
* Upgrade to any release after 0.22 (0.22 included) release e.g. 0.22 => 0.23 Upgrade procedure
* is as follows:
*
* <ol>
* <li>If <SD>/current/<bpid>/previous exists then delete it
* <li>Rename <SD>/current/<bpid>/current to <SD>/current/bpid/current/previous.tmp
* <li>Create new <SD>current/<bpid>/current directory
* <ol>
* <li>Hard links for block files are created from previous.tmp to current
* <li>Save new version file in current directory
* </ol>
* <li>Rename previous.tmp to previous
* </ol>
*
* @param bpSd storage directory <SD>/current/<bpid>
* @param nsInfo Namespace Info from the namenode
* @throws IOException on error
*/
void doUpgrade(DataNode datanode, StorageDirectory bpSd, NamespaceInfo nsInfo)
throws IOException {
// Upgrading is applicable only to release with federation or after
if (!DataNodeLayoutVersion.supports(LayoutVersion.Feature.FEDERATION, layoutVersion)) {
return;
}
LOG.info(
"Upgrading block pool storage directory "
+ bpSd.getRoot()
+ ".\n old LV = "
+ this.getLayoutVersion()
+ "; old CTime = "
+ this.getCTime()
+ ".\n new LV = "
+ HdfsConstants.DATANODE_LAYOUT_VERSION
+ "; new CTime = "
+ nsInfo.getCTime());
// get <SD>/previous directory
String dnRoot = getDataNodeStorageRoot(bpSd.getRoot().getCanonicalPath());
StorageDirectory dnSdStorage = new StorageDirectory(new File(dnRoot));
File dnPrevDir = dnSdStorage.getPreviousDir();
// If <SD>/previous directory exists delete it
if (dnPrevDir.exists()) {
deleteDir(dnPrevDir);
}
File bpCurDir = bpSd.getCurrentDir();
File bpPrevDir = bpSd.getPreviousDir();
assert bpCurDir.exists() : "BP level current directory must exist.";
cleanupDetachDir(new File(bpCurDir, DataStorage.STORAGE_DIR_DETACHED));
// 1. Delete <SD>/current/<bpid>/previous dir before upgrading
if (bpPrevDir.exists()) {
deleteDir(bpPrevDir);
}
File bpTmpDir = bpSd.getPreviousTmp();
assert !bpTmpDir.exists() : "previous.tmp directory must not exist.";
// 2. Rename <SD>/current/<bpid>/current to
// <SD>/current/<bpid>/previous.tmp
rename(bpCurDir, bpTmpDir);
// 3. Create new <SD>/current with block files hardlinks and VERSION
linkAllBlocks(datanode, bpTmpDir, bpCurDir);
this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
assert this.namespaceID == nsInfo.getNamespaceID()
: "Data-node and name-node layout versions must be the same.";
this.cTime = nsInfo.getCTime();
writeProperties(bpSd);
// 4.rename <SD>/current/<bpid>/previous.tmp to
// <SD>/current/<bpid>/previous
rename(bpTmpDir, bpPrevDir);
LOG.info("Upgrade of block pool " + blockpoolID + " at " + bpSd.getRoot() + " is complete");
}
void finalizeUpgrade(String bpID) throws IOException {
for (StorageDirectory sd : storageDirs) {
File prevDir = sd.getPreviousDir();
if (prevDir.exists()) {
throw new RuntimeException("未实现");
} else {
BlockPoolSliceStorage bpStorage = bpStorageMap.get(bpID);
bpStorage.doFinalize(sd.getCurrentDir());
}
}
}
/**
* Prepare directories for a new checkpoint.
*
* <p>Rename <code>current</code> to <code>lastcheckpoint.tmp</code> and recreate <code>current
* </code>.
*
* @throws IOException
*/
void startCheckpoint() throws IOException {
for (StorageDirectory sd : storageDirs) {
File curDir = sd.getCurrentDir();
File tmpCkptDir = sd.getLastCheckpointTmp();
assert !tmpCkptDir.exists() : tmpCkptDir.getName() + " directory must not exist.";
if (curDir.exists()) {
// rename current to tmp
rename(curDir, tmpCkptDir);
}
if (!curDir.mkdir()) throw new IOException("Cannot create directory " + curDir);
}
}
public void clearRollingUpgradeMarkers(List<StorageDirectory> dnStorageDirs) throws IOException {
for (StorageDirectory sd : dnStorageDirs) {
File bpRoot = getBpRoot(blockpoolID, sd.getCurrentDir());
File markerFile = new File(bpRoot, ROLLING_UPGRADE_MARKER_FILE);
if (!storagesWithoutRollingUpgradeMarker.contains(bpRoot.toString())) {
if (markerFile.exists()) {
throw new RuntimeException("未实现");
}
storagesWithoutRollingUpgradeMarker.add(bpRoot.toString());
storagesWithRollingUpgradeMarker.remove(bpRoot.toString());
}
}
}
/**
* Create a rolling upgrade marker file for each BP storage root, if it does not exist already.
*/
public void setRollingUpgradeMarkers(List<StorageDirectory> dnStorageDirs) throws IOException {
for (StorageDirectory sd : dnStorageDirs) {
File bpRoot = getBpRoot(blockpoolID, sd.getCurrentDir());
File markerFile = new File(bpRoot, ROLLING_UPGRADE_MARKER_FILE);
if (!storagesWithRollingUpgradeMarker.contains(bpRoot.toString())) {
if (!markerFile.exists() && markerFile.createNewFile()) {
LOG.info("Created " + markerFile);
} else {
LOG.info(markerFile + " already exists.");
}
storagesWithRollingUpgradeMarker.add(bpRoot.toString());
storagesWithoutRollingUpgradeMarker.remove(bpRoot.toString());
}
}
}
/*
* Finalize the upgrade for a block pool
*/
void finalizeUpgrade(String bpID) throws IOException {
// To handle finalizing a snapshot taken at datanode level while
// upgrading to federation, if datanode level snapshot previous exists,
// then finalize it. Else finalize the corresponding BP.
for (StorageDirectory sd : storageDirs) {
File prevDir = sd.getPreviousDir();
if (prevDir.exists()) {
// data node level storage finalize
doFinalize(sd);
} else {
// block pool storage finalize using specific bpID
BlockPoolSliceStorage bpStorage = bpStorageMap.get(bpID);
bpStorage.doFinalize(sd.getCurrentDir());
}
}
}
/*
* Roll back to old snapshot at the block pool level
* If previous directory exists:
* <ol>
* <li>Rename <SD>/current/<bpid>/current to removed.tmp</li>
* <li>Rename * <SD>/current/<bpid>/previous to current</li>
* <li>Remove removed.tmp</li>
* </ol>
*
* Do nothing if previous directory does not exist.
* @param bpSd Block pool storage directory at <SD>/current/<bpid>
*/
void doRollback(StorageDirectory bpSd, NamespaceInfo nsInfo) throws IOException {
File prevDir = bpSd.getPreviousDir();
// regular startup if previous dir does not exist
if (!prevDir.exists()) return;
// read attributes out of the VERSION file of previous directory
BlockPoolSliceStorage prevInfo = new BlockPoolSliceStorage();
prevInfo.readPreviousVersionProperties(bpSd);
// We allow rollback to a state, which is either consistent with
// the namespace state or can be further upgraded to it.
// In another word, we can only roll back when ( storedLV >= software LV)
// && ( DN.previousCTime <= NN.ctime)
if (!(prevInfo.getLayoutVersion() >= HdfsConstants.DATANODE_LAYOUT_VERSION
&& prevInfo.getCTime() <= nsInfo.getCTime())) { // cannot rollback
throw new InconsistentFSStateException(
bpSd.getRoot(),
"Cannot rollback to a newer state.\nDatanode previous state: LV = "
+ prevInfo.getLayoutVersion()
+ " CTime = "
+ prevInfo.getCTime()
+ " is newer than the namespace state: LV = "
+ HdfsConstants.DATANODE_LAYOUT_VERSION
+ " CTime = "
+ nsInfo.getCTime());
}
LOG.info(
"Rolling back storage directory "
+ bpSd.getRoot()
+ ".\n target LV = "
+ nsInfo.getLayoutVersion()
+ "; target CTime = "
+ nsInfo.getCTime());
File tmpDir = bpSd.getRemovedTmp();
assert !tmpDir.exists() : "removed.tmp directory must not exist.";
// 1. rename current to tmp
File curDir = bpSd.getCurrentDir();
assert curDir.exists() : "Current directory must exist.";
rename(curDir, tmpDir);
// 2. rename previous to current
rename(prevDir, curDir);
// 3. delete removed.tmp dir
deleteDir(tmpDir);
LOG.info("Rollback of " + bpSd.getRoot() + " is complete");
}
void doFinalize(File dnCurDir) throws IOException {
File bpRoot = getBpRoot(blockpoolID, dnCurDir);
StorageDirectory bpSd = new StorageDirectory(bpRoot);
File prevDir = bpSd.getPreviousDir();
if (!prevDir.exists()) {
return; // already finalized
}
final String dataDirPath = bpSd.getRoot().getCanonicalPath();
LOG.info(
"Finalizing upgrade for storage directory "
+ dataDirPath
+ ".\n cur LV = "
+ this.getLayoutVersion()
+ "; cur CTime = "
+ this.getCTime());
assert bpSd.getCurrentDir().exists() : "Current directory must exist.";
throw new RuntimeException("未实现");
}
/**
* Finalize procedure deletes an existing snapshot.
*
* <ol>
* <li>Rename previous to finalized.tmp directory
* <li>Fully delete the finalized.tmp directory
* </ol>
*
* Do nothing, if previous directory does not exist
*/
void doFinalize(StorageDirectory sd) throws IOException {
File prevDir = sd.getPreviousDir();
if (!prevDir.exists()) return; // already discarded
final String dataDirPath = sd.getRoot().getCanonicalPath();
LOG.info(
"Finalizing upgrade for storage directory "
+ dataDirPath
+ ".\n cur LV = "
+ this.getLayoutVersion()
+ "; cur CTime = "
+ this.getCTime());
assert sd.getCurrentDir().exists() : "Current directory must exist.";
final File tmpDir = sd.getFinalizedTmp(); // finalized.tmp directory
final File bbwDir = new File(sd.getRoot(), Storage.STORAGE_1_BBW);
// 1. rename previous to finalized.tmp
rename(prevDir, tmpDir);
// 2. delete finalized.tmp dir in a separate thread
// Also delete the blocksBeingWritten from HDFS 1.x and earlier, if
// it exists.
new Daemon(
new Runnable() {
@Override
public void run() {
try {
deleteDir(tmpDir);
if (bbwDir.exists()) {
deleteDir(bbwDir);
}
} catch (IOException ex) {
LOG.error("Finalize upgrade for " + dataDirPath + " failed", ex);
}
LOG.info("Finalize upgrade for " + dataDirPath + " is complete");
}
@Override
public String toString() {
return "Finalize " + dataDirPath;
}
})
.start();
}
/*
* Finalize the block pool storage by deleting <BP>/previous directory
* that holds the snapshot.
*/
void doFinalize(File dnCurDir) throws IOException {
File bpRoot = getBpRoot(blockpoolID, dnCurDir);
StorageDirectory bpSd = new StorageDirectory(bpRoot);
// block pool level previous directory
File prevDir = bpSd.getPreviousDir();
if (!prevDir.exists()) {
return; // already finalized
}
final String dataDirPath = bpSd.getRoot().getCanonicalPath();
LOG.info(
"Finalizing upgrade for storage directory "
+ dataDirPath
+ ".\n cur LV = "
+ this.getLayoutVersion()
+ "; cur CTime = "
+ this.getCTime());
assert bpSd.getCurrentDir().exists() : "Current directory must exist.";
// rename previous to finalized.tmp
final File tmpDir = bpSd.getFinalizedTmp();
rename(prevDir, tmpDir);
// delete finalized.tmp dir in a separate thread
new Daemon(
new Runnable() {
@Override
public void run() {
try {
deleteDir(tmpDir);
} catch (IOException ex) {
LOG.error("Finalize upgrade for " + dataDirPath + " failed.", ex);
}
LOG.info("Finalize upgrade for " + dataDirPath + " is complete.");
}
@Override
public String toString() {
return "Finalize " + dataDirPath;
}
})
.start();
}
/**
* Rolling back to a snapshot in previous directory by moving it to current directory. Rollback
* procedure: <br>
* If previous directory exists:
*
* <ol>
* <li>Rename current to removed.tmp
* <li>Rename previous to current
* <li>Remove removed.tmp
* </ol>
*
* Do nothing, if previous directory does not exist.
*/
void doRollback(StorageDirectory sd, NamespaceInfo nsInfo) throws IOException {
File prevDir = sd.getPreviousDir();
// regular startup if previous dir does not exist
if (!prevDir.exists()) return;
DataStorage prevInfo = new DataStorage();
prevInfo.readPreviousVersionProperties(sd);
// We allow rollback to a state, which is either consistent with
// the namespace state or can be further upgraded to it.
if (!(prevInfo.getLayoutVersion() >= HdfsConstants.LAYOUT_VERSION
&& prevInfo.getCTime() <= nsInfo.getCTime())) // cannot rollback
throw new InconsistentFSStateException(
sd.getRoot(),
"Cannot rollback to a newer state.\nDatanode previous state: LV = "
+ prevInfo.getLayoutVersion()
+ " CTime = "
+ prevInfo.getCTime()
+ " is newer than the namespace state: LV = "
+ nsInfo.getLayoutVersion()
+ " CTime = "
+ nsInfo.getCTime());
LOG.info(
"Rolling back storage directory "
+ sd.getRoot()
+ ".\n target LV = "
+ nsInfo.getLayoutVersion()
+ "; target CTime = "
+ nsInfo.getCTime());
File tmpDir = sd.getRemovedTmp();
assert !tmpDir.exists() : "removed.tmp directory must not exist.";
// rename current to tmp
File curDir = sd.getCurrentDir();
assert curDir.exists() : "Current directory must exist.";
rename(curDir, tmpDir);
// rename previous to current
rename(prevDir, curDir);
// delete tmp dir
deleteDir(tmpDir);
LOG.info("Rollback of " + sd.getRoot() + " is complete");
}
