/** * Upgrade to any release after 0.22 (0.22 included) release e.g. 0.22 => 0.23 Upgrade procedure * is as follows: * * <ol> * <li>If <SD>/current/<bpid>/previous exists then delete it * <li>Rename <SD>/current/<bpid>/current to <SD>/current/bpid/current/previous.tmp * <li>Create new <SD>current/<bpid>/current directory * <ol> * <li>Hard links for block files are created from previous.tmp to current * <li>Save new version file in current directory * </ol> * <li>Rename previous.tmp to previous * </ol> * * @param bpSd storage directory <SD>/current/<bpid> * @param nsInfo Namespace Info from the namenode * @throws IOException on error */ void doUpgrade(DataNode datanode, StorageDirectory bpSd, NamespaceInfo nsInfo) throws IOException { // Upgrading is applicable only to release with federation or after if (!DataNodeLayoutVersion.supports(LayoutVersion.Feature.FEDERATION, layoutVersion)) { return; } LOG.info( "Upgrading block pool storage directory " + bpSd.getRoot() + ".\n old LV = " + this.getLayoutVersion() + "; old CTime = " + this.getCTime() + ".\n new LV = " + HdfsConstants.DATANODE_LAYOUT_VERSION + "; new CTime = " + nsInfo.getCTime()); // get <SD>/previous directory String dnRoot = getDataNodeStorageRoot(bpSd.getRoot().getCanonicalPath()); StorageDirectory dnSdStorage = new StorageDirectory(new File(dnRoot)); File dnPrevDir = dnSdStorage.getPreviousDir(); // If <SD>/previous directory exists delete it if (dnPrevDir.exists()) { deleteDir(dnPrevDir); } File bpCurDir = bpSd.getCurrentDir(); File bpPrevDir = bpSd.getPreviousDir(); assert bpCurDir.exists() : "BP level current directory must exist."; cleanupDetachDir(new File(bpCurDir, DataStorage.STORAGE_DIR_DETACHED)); // 1. Delete <SD>/current/<bpid>/previous dir before upgrading if (bpPrevDir.exists()) { deleteDir(bpPrevDir); } File bpTmpDir = bpSd.getPreviousTmp(); assert !bpTmpDir.exists() : "previous.tmp directory must not exist."; // 2. Rename <SD>/current/<bpid>/current to // <SD>/current/<bpid>/previous.tmp rename(bpCurDir, bpTmpDir); // 3. Create new <SD>/current with block files hardlinks and VERSION linkAllBlocks(datanode, bpTmpDir, bpCurDir); this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION; assert this.namespaceID == nsInfo.getNamespaceID() : "Data-node and name-node layout versions must be the same."; this.cTime = nsInfo.getCTime(); writeProperties(bpSd); // 4.rename <SD>/current/<bpid>/previous.tmp to // <SD>/current/<bpid>/previous rename(bpTmpDir, bpPrevDir); LOG.info("Upgrade of block pool " + blockpoolID + " at " + bpSd.getRoot() + " is complete"); }
/** Delete all files and directories in the trash directories. */ public void clearTrash() { for (StorageDirectory sd : storageDirs) { File trashRoot = getTrashRootDir(sd); Preconditions.checkState(!(trashRoot.exists() && sd.getPreviousDir().exists())); FileUtil.fullyDelete(trashRoot); LOG.info("Cleared trash for storage directory " + sd); } }
/** * Upgrade -- Move current storage into a backup directory, and hardlink all its blocks into the * new current directory. * * <p>Upgrade from pre-0.22 to 0.22 or later release e.g. 0.19/0.20/ => 0.22/0.23 * * <ul> * <li>If <SD>/previous exists then delete it * <li>Rename <SD>/current to <SD>/previous.tmp * <li>Create new <SD>/current/<bpid>/current directory * <li> * <ul> * <li>Hard links for block files are created from <SD>/previous.tmp to * <SD>/current/<bpid>/current * <li>Saves new version file in <SD>/current/<bpid>/current directory * </ul> * <li>Rename <SD>/previous.tmp to <SD>/previous * </ul> * * There should be only ONE namenode in the cluster for first time upgrade to 0.22 * * @param sd storage directory * @throws IOException on error */ void doUpgrade(StorageDirectory sd, NamespaceInfo nsInfo) throws IOException { if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { clusterID = nsInfo.getClusterID(); layoutVersion = nsInfo.getLayoutVersion(); writeProperties(sd); return; } LOG.info( "Upgrading storage directory " + sd.getRoot() + ".\n old LV = " + this.getLayoutVersion() + "; old CTime = " + this.getCTime() + ".\n new LV = " + nsInfo.getLayoutVersion() + "; new CTime = " + nsInfo.getCTime()); File curDir = sd.getCurrentDir(); File prevDir = sd.getPreviousDir(); File bbwDir = new File(sd.getRoot(), Storage.STORAGE_1_BBW); assert curDir.exists() : "Data node current directory must exist."; // Cleanup directory "detach" cleanupDetachDir(new File(curDir, STORAGE_DIR_DETACHED)); // 1. delete <SD>/previous dir before upgrading if (prevDir.exists()) deleteDir(prevDir); // get previous.tmp directory, <SD>/previous.tmp File tmpDir = sd.getPreviousTmp(); assert !tmpDir.exists() : "Data node previous.tmp directory must not exist."; // 2. Rename <SD>/current to <SD>/previous.tmp rename(curDir, tmpDir); // 3. Format BP and hard link blocks from previous directory File curBpDir = BlockPoolSliceStorage.getBpRoot(nsInfo.getBlockPoolID(), curDir); BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage( nsInfo.getNamespaceID(), nsInfo.getBlockPoolID(), nsInfo.getCTime(), nsInfo.getClusterID()); bpStorage.format(curDir, nsInfo); linkAllBlocks(tmpDir, bbwDir, new File(curBpDir, STORAGE_DIR_CURRENT)); // 4. Write version file under <SD>/current layoutVersion = HdfsConstants.LAYOUT_VERSION; clusterID = nsInfo.getClusterID(); writeProperties(sd); // 5. Rename <SD>/previous.tmp to <SD>/previous rename(tmpDir, prevDir); LOG.info("Upgrade of " + sd.getRoot() + " is complete"); addBlockPoolStorage(nsInfo.getBlockPoolID(), bpStorage); }
void finalizeUpgrade(String bpID) throws IOException { for (StorageDirectory sd : storageDirs) { File prevDir = sd.getPreviousDir(); if (prevDir.exists()) { throw new RuntimeException("未实现"); } else { BlockPoolSliceStorage bpStorage = bpStorageMap.get(bpID); bpStorage.doFinalize(sd.getCurrentDir()); } } }
/* * Finalize the upgrade for a block pool */ void finalizeUpgrade(String bpID) throws IOException { // To handle finalizing a snapshot taken at datanode level while // upgrading to federation, if datanode level snapshot previous exists, // then finalize it. Else finalize the corresponding BP. for (StorageDirectory sd : storageDirs) { File prevDir = sd.getPreviousDir(); if (prevDir.exists()) { // data node level storage finalize doFinalize(sd); } else { // block pool storage finalize using specific bpID BlockPoolSliceStorage bpStorage = bpStorageMap.get(bpID); bpStorage.doFinalize(sd.getCurrentDir()); } } }
/* * Roll back to old snapshot at the block pool level * If previous directory exists: * <ol> * <li>Rename <SD>/current/<bpid>/current to removed.tmp</li> * <li>Rename * <SD>/current/<bpid>/previous to current</li> * <li>Remove removed.tmp</li> * </ol> * * Do nothing if previous directory does not exist. * @param bpSd Block pool storage directory at <SD>/current/<bpid> */ void doRollback(StorageDirectory bpSd, NamespaceInfo nsInfo) throws IOException { File prevDir = bpSd.getPreviousDir(); // regular startup if previous dir does not exist if (!prevDir.exists()) return; // read attributes out of the VERSION file of previous directory BlockPoolSliceStorage prevInfo = new BlockPoolSliceStorage(); prevInfo.readPreviousVersionProperties(bpSd); // We allow rollback to a state, which is either consistent with // the namespace state or can be further upgraded to it. // In another word, we can only roll back when ( storedLV >= software LV) // && ( DN.previousCTime <= NN.ctime) if (!(prevInfo.getLayoutVersion() >= HdfsConstants.DATANODE_LAYOUT_VERSION && prevInfo.getCTime() <= nsInfo.getCTime())) { // cannot rollback throw new InconsistentFSStateException( bpSd.getRoot(), "Cannot rollback to a newer state.\nDatanode previous state: LV = " + prevInfo.getLayoutVersion() + " CTime = " + prevInfo.getCTime() + " is newer than the namespace state: LV = " + HdfsConstants.DATANODE_LAYOUT_VERSION + " CTime = " + nsInfo.getCTime()); } LOG.info( "Rolling back storage directory " + bpSd.getRoot() + ".\n target LV = " + nsInfo.getLayoutVersion() + "; target CTime = " + nsInfo.getCTime()); File tmpDir = bpSd.getRemovedTmp(); assert !tmpDir.exists() : "removed.tmp directory must not exist."; // 1. rename current to tmp File curDir = bpSd.getCurrentDir(); assert curDir.exists() : "Current directory must exist."; rename(curDir, tmpDir); // 2. rename previous to current rename(prevDir, curDir); // 3. delete removed.tmp dir deleteDir(tmpDir); LOG.info("Rollback of " + bpSd.getRoot() + " is complete"); }
void doFinalize(File dnCurDir) throws IOException { File bpRoot = getBpRoot(blockpoolID, dnCurDir); StorageDirectory bpSd = new StorageDirectory(bpRoot); File prevDir = bpSd.getPreviousDir(); if (!prevDir.exists()) { return; // already finalized } final String dataDirPath = bpSd.getRoot().getCanonicalPath(); LOG.info( "Finalizing upgrade for storage directory " + dataDirPath + ".\n cur LV = " + this.getLayoutVersion() + "; cur CTime = " + this.getCTime()); assert bpSd.getCurrentDir().exists() : "Current directory must exist."; throw new RuntimeException("未实现"); }
/** * Finalize procedure deletes an existing snapshot. * * <ol> * <li>Rename previous to finalized.tmp directory * <li>Fully delete the finalized.tmp directory * </ol> * * Do nothing, if previous directory does not exist */ void doFinalize(StorageDirectory sd) throws IOException { File prevDir = sd.getPreviousDir(); if (!prevDir.exists()) return; // already discarded final String dataDirPath = sd.getRoot().getCanonicalPath(); LOG.info( "Finalizing upgrade for storage directory " + dataDirPath + ".\n cur LV = " + this.getLayoutVersion() + "; cur CTime = " + this.getCTime()); assert sd.getCurrentDir().exists() : "Current directory must exist."; final File tmpDir = sd.getFinalizedTmp(); // finalized.tmp directory final File bbwDir = new File(sd.getRoot(), Storage.STORAGE_1_BBW); // 1. rename previous to finalized.tmp rename(prevDir, tmpDir); // 2. delete finalized.tmp dir in a separate thread // Also delete the blocksBeingWritten from HDFS 1.x and earlier, if // it exists. new Daemon( new Runnable() { @Override public void run() { try { deleteDir(tmpDir); if (bbwDir.exists()) { deleteDir(bbwDir); } } catch (IOException ex) { LOG.error("Finalize upgrade for " + dataDirPath + " failed", ex); } LOG.info("Finalize upgrade for " + dataDirPath + " is complete"); } @Override public String toString() { return "Finalize " + dataDirPath; } }) .start(); }
/* * Finalize the block pool storage by deleting <BP>/previous directory * that holds the snapshot. */ void doFinalize(File dnCurDir) throws IOException { File bpRoot = getBpRoot(blockpoolID, dnCurDir); StorageDirectory bpSd = new StorageDirectory(bpRoot); // block pool level previous directory File prevDir = bpSd.getPreviousDir(); if (!prevDir.exists()) { return; // already finalized } final String dataDirPath = bpSd.getRoot().getCanonicalPath(); LOG.info( "Finalizing upgrade for storage directory " + dataDirPath + ".\n cur LV = " + this.getLayoutVersion() + "; cur CTime = " + this.getCTime()); assert bpSd.getCurrentDir().exists() : "Current directory must exist."; // rename previous to finalized.tmp final File tmpDir = bpSd.getFinalizedTmp(); rename(prevDir, tmpDir); // delete finalized.tmp dir in a separate thread new Daemon( new Runnable() { @Override public void run() { try { deleteDir(tmpDir); } catch (IOException ex) { LOG.error("Finalize upgrade for " + dataDirPath + " failed.", ex); } LOG.info("Finalize upgrade for " + dataDirPath + " is complete."); } @Override public String toString() { return "Finalize " + dataDirPath; } }) .start(); }
/** * Rolling back to a snapshot in previous directory by moving it to current directory. Rollback * procedure: <br> * If previous directory exists: * * <ol> * <li>Rename current to removed.tmp * <li>Rename previous to current * <li>Remove removed.tmp * </ol> * * Do nothing, if previous directory does not exist. */ void doRollback(StorageDirectory sd, NamespaceInfo nsInfo) throws IOException { File prevDir = sd.getPreviousDir(); // regular startup if previous dir does not exist if (!prevDir.exists()) return; DataStorage prevInfo = new DataStorage(); prevInfo.readPreviousVersionProperties(sd); // We allow rollback to a state, which is either consistent with // the namespace state or can be further upgraded to it. if (!(prevInfo.getLayoutVersion() >= HdfsConstants.LAYOUT_VERSION && prevInfo.getCTime() <= nsInfo.getCTime())) // cannot rollback throw new InconsistentFSStateException( sd.getRoot(), "Cannot rollback to a newer state.\nDatanode previous state: LV = " + prevInfo.getLayoutVersion() + " CTime = " + prevInfo.getCTime() + " is newer than the namespace state: LV = " + nsInfo.getLayoutVersion() + " CTime = " + nsInfo.getCTime()); LOG.info( "Rolling back storage directory " + sd.getRoot() + ".\n target LV = " + nsInfo.getLayoutVersion() + "; target CTime = " + nsInfo.getCTime()); File tmpDir = sd.getRemovedTmp(); assert !tmpDir.exists() : "removed.tmp directory must not exist."; // rename current to tmp File curDir = sd.getCurrentDir(); assert curDir.exists() : "Current directory must exist."; rename(curDir, tmpDir); // rename previous to current rename(prevDir, curDir); // delete tmp dir deleteDir(tmpDir); LOG.info("Rollback of " + sd.getRoot() + " is complete"); }
private void doTransition( DataNode datanode, StorageDirectory sd, NamespaceInfo nsInfo, StartupOption startOpt) throws IOException { if (startOpt == StartupOption.ROLLBACK && sd.getPreviousDir().exists()) { throw new RuntimeException("ROLLBACK 未实现"); } else { int restored = restoreBlockFilesFromTrash(getTrashRootDir(sd)); LOG.info("从垃圾桶恢复" + restored + "块文件."); } readProperties(sd); checkVersionUpgradable(this.layoutVersion); assert this.layoutVersion >= HdfsConstants.DATANODE_LAYOUT_VERSION : "Future version is not allowed"; if (getNamespaceID() != nsInfo.getNamespaceID()) { throw new IOException( "Incompatible namespaceIDs in " + sd.getRoot().getCanonicalPath() + ": namenode namespaceID = " + nsInfo.getNamespaceID() + "; datanode namespaceID = " + getNamespaceID()); } if (!blockpoolID.equals(nsInfo.getBlockPoolID())) { throw new IOException( "Incompatible blockpoolIDs in " + sd.getRoot().getCanonicalPath() + ": namenode blockpoolID = " + nsInfo.getBlockPoolID() + "; datanode blockpoolID = " + blockpoolID); } if (this.layoutVersion == HdfsConstants.DATANODE_LAYOUT_VERSION && this.cTime == nsInfo.getCTime()) { return; // 正常启动 } throw new RuntimeException("upgrade 未实现"); }
/** * Analyze whether a transition of the BP state is required and perform it if necessary. <br> * Rollback if previousLV >= LAYOUT_VERSION && prevCTime <= namenode.cTime. Upgrade if this.LV > * LAYOUT_VERSION || this.cTime < namenode.cTime Regular startup if this.LV = LAYOUT_VERSION && * this.cTime = namenode.cTime * * @param sd storage directory <SD>/current/<bpid> * @param nsInfo namespace info * @param startOpt startup option * @throws IOException */ private void doTransition( DataNode datanode, StorageDirectory sd, NamespaceInfo nsInfo, StartupOption startOpt) throws IOException { if (startOpt == StartupOption.ROLLBACK && sd.getPreviousDir().exists()) { Preconditions.checkState( !getTrashRootDir(sd).exists(), sd.getPreviousDir() + " and " + getTrashRootDir(sd) + " should not " + " both be present."); doRollback(sd, nsInfo); // rollback if applicable } else if (startOpt == StartupOption.ROLLBACK && !sd.getPreviousDir().exists()) { // Restore all the files in the trash. The restored files are retained // during rolling upgrade rollback. They are deleted during rolling // upgrade downgrade. int restored = restoreBlockFilesFromTrash(getTrashRootDir(sd)); LOG.info("Restored " + restored + " block files from trash."); } readProperties(sd); checkVersionUpgradable(this.layoutVersion); assert this.layoutVersion >= HdfsConstants.DATANODE_LAYOUT_VERSION : "Future version is not allowed"; if (getNamespaceID() != nsInfo.getNamespaceID()) { throw new IOException( "Incompatible namespaceIDs in " + sd.getRoot().getCanonicalPath() + ": namenode namespaceID = " + nsInfo.getNamespaceID() + "; datanode namespaceID = " + getNamespaceID()); } if (!blockpoolID.equals(nsInfo.getBlockPoolID())) { throw new IOException( "Incompatible blockpoolIDs in " + sd.getRoot().getCanonicalPath() + ": namenode blockpoolID = " + nsInfo.getBlockPoolID() + "; datanode blockpoolID = " + blockpoolID); } if (this.layoutVersion == HdfsConstants.DATANODE_LAYOUT_VERSION && this.cTime == nsInfo.getCTime()) { return; // regular startup } if (this.layoutVersion > HdfsConstants.DATANODE_LAYOUT_VERSION) { int restored = restoreBlockFilesFromTrash(getTrashRootDir(sd)); LOG.info( "Restored " + restored + " block files from trash " + "before the layout upgrade. These blocks will be moved to " + "the previous directory during the upgrade"); } if (this.layoutVersion > HdfsConstants.DATANODE_LAYOUT_VERSION || this.cTime < nsInfo.getCTime()) { doUpgrade(datanode, sd, nsInfo); // upgrade return; } // layoutVersion == LAYOUT_VERSION && this.cTime > nsInfo.cTime // must shutdown throw new IOException( "Datanode state: LV = " + this.getLayoutVersion() + " CTime = " + this.getCTime() + " is newer than the namespace state: LV = " + nsInfo.getLayoutVersion() + " CTime = " + nsInfo.getCTime()); }