/* * Read ClusterID, StorageID, StorageType, CTime from * DataStorage VERSION file and verify them. */ @Override protected void setFieldsFromProperties(Properties props, StorageDirectory sd) throws IOException { setLayoutVersion(props, sd); setcTime(props, sd); setStorageType(props, sd); setClusterId(props, layoutVersion, sd); // Read NamespaceID in version before federation if (!LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { setNamespaceID(props, sd); } // valid storage id, storage id may be empty String ssid = props.getProperty("storageID"); if (ssid == null) { throw new InconsistentFSStateException( sd.getRoot(), "file " + STORAGE_FILE_VERSION + " is invalid."); } String sid = getStorageID(); if (!(sid.equals("") || ssid.equals("") || sid.equals(ssid))) { throw new InconsistentFSStateException(sd.getRoot(), "has incompatible storage Id."); } if (sid.equals("")) { // update id only if it was empty setStorageID(ssid); } }
/** * Analize which and whether a transition of the fs state is required and perform it if necessary. * * <p>Rollback if previousLV >= LAYOUT_VERSION && prevCTime <= namenode.cTime Upgrade if this.LV > * LAYOUT_VERSION || this.cTime < namenode.cTime Regular startup if this.LV = LAYOUT_VERSION && * this.cTime = namenode.cTime * * @param datanode Datanode to which this storage belongs to * @param sd storage directory * @param nsInfo namespace info * @param startOpt startup option * @throws IOException */ private void doTransition( DataNode datanode, StorageDirectory sd, NamespaceInfo nsInfo, StartupOption startOpt) throws IOException { if (startOpt == StartupOption.ROLLBACK) { doRollback(sd, nsInfo); // rollback if applicable } readProperties(sd); checkVersionUpgradable(this.layoutVersion); assert this.layoutVersion >= HdfsConstants.LAYOUT_VERSION : "Future version is not allowed"; boolean federationSupported = LayoutVersion.supports(Feature.FEDERATION, layoutVersion); // For pre-federation version - validate the namespaceID if (!federationSupported && getNamespaceID() != nsInfo.getNamespaceID()) { throw new IOException( "Incompatible namespaceIDs in " + sd.getRoot().getCanonicalPath() + ": namenode namespaceID = " + nsInfo.getNamespaceID() + "; datanode namespaceID = " + getNamespaceID()); } // For version that supports federation, validate clusterID if (federationSupported && !getClusterID().equals(nsInfo.getClusterID())) { throw new IOException( "Incompatible clusterIDs in " + sd.getRoot().getCanonicalPath() + ": namenode clusterID = " + nsInfo.getClusterID() + "; datanode clusterID = " + getClusterID()); } // regular start up if (this.layoutVersion == HdfsConstants.LAYOUT_VERSION && this.cTime == nsInfo.getCTime()) return; // regular startup // verify necessity of a distributed upgrade UpgradeManagerDatanode um = datanode.getUpgradeManagerDatanode(nsInfo.getBlockPoolID()); verifyDistributedUpgradeProgress(um, nsInfo); // do upgrade if (this.layoutVersion > HdfsConstants.LAYOUT_VERSION || this.cTime < nsInfo.getCTime()) { doUpgrade(sd, nsInfo); // upgrade return; } // layoutVersion == LAYOUT_VERSION && this.cTime > nsInfo.cTime // must shutdown throw new IOException( "Datanode state: LV = " + this.getLayoutVersion() + " CTime = " + this.getCTime() + " is newer than the namespace state: LV = " + nsInfo.getLayoutVersion() + " CTime = " + nsInfo.getCTime()); }
/** * Upgrade -- Move current storage into a backup directory, and hardlink all its blocks into the * new current directory. * * <p>Upgrade from pre-0.22 to 0.22 or later release e.g. 0.19/0.20/ => 0.22/0.23 * * <ul> * <li>If <SD>/previous exists then delete it * <li>Rename <SD>/current to <SD>/previous.tmp * <li>Create new <SD>/current/<bpid>/current directory * <li> * <ul> * <li>Hard links for block files are created from <SD>/previous.tmp to * <SD>/current/<bpid>/current * <li>Saves new version file in <SD>/current/<bpid>/current directory * </ul> * <li>Rename <SD>/previous.tmp to <SD>/previous * </ul> * * There should be only ONE namenode in the cluster for first time upgrade to 0.22 * * @param sd storage directory * @throws IOException on error */ void doUpgrade(StorageDirectory sd, NamespaceInfo nsInfo) throws IOException { if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { clusterID = nsInfo.getClusterID(); layoutVersion = nsInfo.getLayoutVersion(); writeProperties(sd); return; } LOG.info( "Upgrading storage directory " + sd.getRoot() + ".\n old LV = " + this.getLayoutVersion() + "; old CTime = " + this.getCTime() + ".\n new LV = " + nsInfo.getLayoutVersion() + "; new CTime = " + nsInfo.getCTime()); File curDir = sd.getCurrentDir(); File prevDir = sd.getPreviousDir(); File bbwDir = new File(sd.getRoot(), Storage.STORAGE_1_BBW); assert curDir.exists() : "Data node current directory must exist."; // Cleanup directory "detach" cleanupDetachDir(new File(curDir, STORAGE_DIR_DETACHED)); // 1. delete <SD>/previous dir before upgrading if (prevDir.exists()) deleteDir(prevDir); // get previous.tmp directory, <SD>/previous.tmp File tmpDir = sd.getPreviousTmp(); assert !tmpDir.exists() : "Data node previous.tmp directory must not exist."; // 2. Rename <SD>/current to <SD>/previous.tmp rename(curDir, tmpDir); // 3. Format BP and hard link blocks from previous directory File curBpDir = BlockPoolSliceStorage.getBpRoot(nsInfo.getBlockPoolID(), curDir); BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage( nsInfo.getNamespaceID(), nsInfo.getBlockPoolID(), nsInfo.getCTime(), nsInfo.getClusterID()); bpStorage.format(curDir, nsInfo); linkAllBlocks(tmpDir, bbwDir, new File(curBpDir, STORAGE_DIR_CURRENT)); // 4. Write version file under <SD>/current layoutVersion = HdfsConstants.LAYOUT_VERSION; clusterID = nsInfo.getClusterID(); writeProperties(sd); // 5. Rename <SD>/previous.tmp to <SD>/previous rename(tmpDir, prevDir); LOG.info("Upgrade of " + sd.getRoot() + " is complete"); addBlockPoolStorage(nsInfo.getBlockPoolID(), bpStorage); }
/** * Upgrade to any release after 0.22 (0.22 included) release e.g. 0.22 => 0.23 Upgrade procedure * is as follows: * * <ol> * <li>If <SD>/current/<bpid>/previous exists then delete it * <li>Rename <SD>/current/<bpid>/current to <SD>/current/bpid/current/previous.tmp * <li>Create new <SD>current/<bpid>/current directory * <ol> * <li>Hard links for block files are created from previous.tmp to current * <li>Save new version file in current directory * </ol> * <li>Rename previous.tmp to previous * </ol> * * @param bpSd storage directory <SD>/current/<bpid> * @param nsInfo Namespace Info from the namenode * @throws IOException on error */ void doUpgrade(DataNode datanode, StorageDirectory bpSd, NamespaceInfo nsInfo) throws IOException { // Upgrading is applicable only to release with federation or after if (!DataNodeLayoutVersion.supports(LayoutVersion.Feature.FEDERATION, layoutVersion)) { return; } LOG.info( "Upgrading block pool storage directory " + bpSd.getRoot() + ".\n old LV = " + this.getLayoutVersion() + "; old CTime = " + this.getCTime() + ".\n new LV = " + HdfsConstants.DATANODE_LAYOUT_VERSION + "; new CTime = " + nsInfo.getCTime()); // get <SD>/previous directory String dnRoot = getDataNodeStorageRoot(bpSd.getRoot().getCanonicalPath()); StorageDirectory dnSdStorage = new StorageDirectory(new File(dnRoot)); File dnPrevDir = dnSdStorage.getPreviousDir(); // If <SD>/previous directory exists delete it if (dnPrevDir.exists()) { deleteDir(dnPrevDir); } File bpCurDir = bpSd.getCurrentDir(); File bpPrevDir = bpSd.getPreviousDir(); assert bpCurDir.exists() : "BP level current directory must exist."; cleanupDetachDir(new File(bpCurDir, DataStorage.STORAGE_DIR_DETACHED)); // 1. Delete <SD>/current/<bpid>/previous dir before upgrading if (bpPrevDir.exists()) { deleteDir(bpPrevDir); } File bpTmpDir = bpSd.getPreviousTmp(); assert !bpTmpDir.exists() : "previous.tmp directory must not exist."; // 2. Rename <SD>/current/<bpid>/current to // <SD>/current/<bpid>/previous.tmp rename(bpCurDir, bpTmpDir); // 3. Create new <SD>/current with block files hardlinks and VERSION linkAllBlocks(datanode, bpTmpDir, bpCurDir); this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION; assert this.namespaceID == nsInfo.getNamespaceID() : "Data-node and name-node layout versions must be the same."; this.cTime = nsInfo.getCTime(); writeProperties(bpSd); // 4.rename <SD>/current/<bpid>/previous.tmp to // <SD>/current/<bpid>/previous rename(bpTmpDir, bpPrevDir); LOG.info("Upgrade of block pool " + blockpoolID + " at " + bpSd.getRoot() + " is complete"); }
/* * Roll back to old snapshot at the block pool level * If previous directory exists: * <ol> * <li>Rename <SD>/current/<bpid>/current to removed.tmp</li> * <li>Rename * <SD>/current/<bpid>/previous to current</li> * <li>Remove removed.tmp</li> * </ol> * * Do nothing if previous directory does not exist. * @param bpSd Block pool storage directory at <SD>/current/<bpid> */ void doRollback(StorageDirectory bpSd, NamespaceInfo nsInfo) throws IOException { File prevDir = bpSd.getPreviousDir(); // regular startup if previous dir does not exist if (!prevDir.exists()) return; // read attributes out of the VERSION file of previous directory BlockPoolSliceStorage prevInfo = new BlockPoolSliceStorage(); prevInfo.readPreviousVersionProperties(bpSd); // We allow rollback to a state, which is either consistent with // the namespace state or can be further upgraded to it. // In another word, we can only roll back when ( storedLV >= software LV) // && ( DN.previousCTime <= NN.ctime) if (!(prevInfo.getLayoutVersion() >= HdfsConstants.DATANODE_LAYOUT_VERSION && prevInfo.getCTime() <= nsInfo.getCTime())) { // cannot rollback throw new InconsistentFSStateException( bpSd.getRoot(), "Cannot rollback to a newer state.\nDatanode previous state: LV = " + prevInfo.getLayoutVersion() + " CTime = " + prevInfo.getCTime() + " is newer than the namespace state: LV = " + HdfsConstants.DATANODE_LAYOUT_VERSION + " CTime = " + nsInfo.getCTime()); } LOG.info( "Rolling back storage directory " + bpSd.getRoot() + ".\n target LV = " + nsInfo.getLayoutVersion() + "; target CTime = " + nsInfo.getCTime()); File tmpDir = bpSd.getRemovedTmp(); assert !tmpDir.exists() : "removed.tmp directory must not exist."; // 1. rename current to tmp File curDir = bpSd.getCurrentDir(); assert curDir.exists() : "Current directory must exist."; rename(curDir, tmpDir); // 2. rename previous to current rename(prevDir, curDir); // 3. delete removed.tmp dir deleteDir(tmpDir); LOG.info("Rollback of " + bpSd.getRoot() + " is complete"); }
@Override protected void setFieldsFromProperties(Properties props, StorageDirectory sd) throws IOException { setLayoutVersion(props, sd); setNamespaceID(props, sd); setcTime(props, sd); String sbpid = props.getProperty("blockpoolID"); setBlockPoolID(sd.getRoot(), sbpid); }
private void setFieldsFromProperties( Properties props, StorageDirectory sd, boolean overrideLayoutVersion, int toLayoutVersion) throws IOException { if (overrideLayoutVersion) { this.layoutVersion = toLayoutVersion; } else { setLayoutVersion(props, sd); } setcTime(props, sd); checkStorageType(props, sd); setClusterId(props, layoutVersion, sd); if (!DataNodeLayoutVersion.supports(LayoutVersion.Feature.FEDERATION, layoutVersion)) { setNamespaceID(props, sd); } String ssid = props.getProperty("storageID"); if (ssid == null) { throw new InconsistentFSStateException( sd.getRoot(), "file " + STORAGE_FILE_VERSION + " is invalid."); } String sid = sd.getStorageUuid(); if (!(sid == null || sid.equals("") || ssid.equals("") || sid.equals(ssid))) { throw new InconsistentFSStateException(sd.getRoot(), "具有不兼容的存储标识。"); } if (sid == null) { // update id only if it was null sd.setStorageUuid(ssid); } if (props.getProperty("datanodeUuid") != null) { String dnUuid = props.getProperty("datanodeUuid"); if (getDatanodeUuid() == null) { setDatanodeUuid(dnUuid); } else if (getDatanodeUuid().compareTo(dnUuid) != 0) { throw new InconsistentFSStateException( sd.getRoot(), "Root " + sd.getRoot() + ": DatanodeUuid=" + dnUuid + ", does not match " + getDatanodeUuid() + " from other" + " StorageDirectory."); } } }
private void doTransition( DataNode datanode, StorageDirectory sd, NamespaceInfo nsInfo, StartupOption startOpt) throws IOException { if (startOpt == StartupOption.ROLLBACK) { throw new RuntimeException("未实现"); } readProperties(sd); checkVersionUpgradable(this.layoutVersion); // 版本号为负数,越小越高,文件里面的layoutVersion必须比hadoop版本大或者等于 assert this.layoutVersion >= HdfsConstants.DATANODE_LAYOUT_VERSION : "不允许将来的版本"; boolean federationSupported = DataNodeLayoutVersion.supports(LayoutVersion.Feature.FEDERATION, layoutVersion); if (!federationSupported && getNamespaceID() != nsInfo.getNamespaceID()) { throw new IOException( "Incompatible namespaceIDs in " + sd.getRoot().getCanonicalPath() + ": namenode namespaceID = " + nsInfo.getNamespaceID() + "; datanode namespaceID = " + getNamespaceID()); } if (federationSupported && !getClusterID().equals(nsInfo.getClusterID())) { throw new IOException( "Incompatible clusterIDs in " + sd.getRoot().getCanonicalPath() + ": namenode clusterID = " + nsInfo.getClusterID() + "; datanode clusterID = " + getClusterID()); } boolean haveValidStorageId = DataNodeLayoutVersion.supports( LayoutVersion.Feature.ADD_DATANODE_AND_STORAGE_UUIDS, layoutVersion) && DatanodeStorage.isValidStorageId(sd.getStorageUuid()); if (this.layoutVersion == HdfsConstants.DATANODE_LAYOUT_VERSION) { createStorageID(sd, !haveValidStorageId); return; } throw new IOException( "BUG: The stored LV = " + this.getLayoutVersion() + " is newer than the supported LV = " + HdfsConstants.DATANODE_LAYOUT_VERSION); }
/** * Finalize procedure deletes an existing snapshot. * * <ol> * <li>Rename previous to finalized.tmp directory * <li>Fully delete the finalized.tmp directory * </ol> * * Do nothing, if previous directory does not exist */ void doFinalize(StorageDirectory sd) throws IOException { File prevDir = sd.getPreviousDir(); if (!prevDir.exists()) return; // already discarded final String dataDirPath = sd.getRoot().getCanonicalPath(); LOG.info( "Finalizing upgrade for storage directory " + dataDirPath + ".\n cur LV = " + this.getLayoutVersion() + "; cur CTime = " + this.getCTime()); assert sd.getCurrentDir().exists() : "Current directory must exist."; final File tmpDir = sd.getFinalizedTmp(); // finalized.tmp directory final File bbwDir = new File(sd.getRoot(), Storage.STORAGE_1_BBW); // 1. rename previous to finalized.tmp rename(prevDir, tmpDir); // 2. delete finalized.tmp dir in a separate thread // Also delete the blocksBeingWritten from HDFS 1.x and earlier, if // it exists. new Daemon( new Runnable() { @Override public void run() { try { deleteDir(tmpDir); if (bbwDir.exists()) { deleteDir(bbwDir); } } catch (IOException ex) { LOG.error("Finalize upgrade for " + dataDirPath + " failed", ex); } LOG.info("Finalize upgrade for " + dataDirPath + " is complete"); } @Override public String toString() { return "Finalize " + dataDirPath; } }) .start(); }
/** * Remove block pool level storage directory. * * @param absPathToRemove the absolute path of the root for the block pool level storage to * remove. */ void remove(File absPathToRemove) { Preconditions.checkArgument(absPathToRemove.isAbsolute()); LOG.info("Removing block level storage: " + absPathToRemove); for (Iterator<StorageDirectory> it = this.storageDirs.iterator(); it.hasNext(); ) { StorageDirectory sd = it.next(); if (sd.getRoot().getAbsoluteFile().equals(absPathToRemove)) { it.remove(); break; } } }
/** * Rolling back to a snapshot in previous directory by moving it to current directory. Rollback * procedure: <br> * If previous directory exists: * * <ol> * <li>Rename current to removed.tmp * <li>Rename previous to current * <li>Remove removed.tmp * </ol> * * Do nothing, if previous directory does not exist. */ void doRollback(StorageDirectory sd, NamespaceInfo nsInfo) throws IOException { File prevDir = sd.getPreviousDir(); // regular startup if previous dir does not exist if (!prevDir.exists()) return; DataStorage prevInfo = new DataStorage(); prevInfo.readPreviousVersionProperties(sd); // We allow rollback to a state, which is either consistent with // the namespace state or can be further upgraded to it. if (!(prevInfo.getLayoutVersion() >= HdfsConstants.LAYOUT_VERSION && prevInfo.getCTime() <= nsInfo.getCTime())) // cannot rollback throw new InconsistentFSStateException( sd.getRoot(), "Cannot rollback to a newer state.\nDatanode previous state: LV = " + prevInfo.getLayoutVersion() + " CTime = " + prevInfo.getCTime() + " is newer than the namespace state: LV = " + nsInfo.getLayoutVersion() + " CTime = " + nsInfo.getCTime()); LOG.info( "Rolling back storage directory " + sd.getRoot() + ".\n target LV = " + nsInfo.getLayoutVersion() + "; target CTime = " + nsInfo.getCTime()); File tmpDir = sd.getRemovedTmp(); assert !tmpDir.exists() : "removed.tmp directory must not exist."; // rename current to tmp File curDir = sd.getCurrentDir(); assert curDir.exists() : "Current directory must exist."; rename(curDir, tmpDir); // rename previous to current rename(prevDir, curDir); // delete tmp dir deleteDir(tmpDir); LOG.info("Rollback of " + sd.getRoot() + " is complete"); }
/** * Analyze checkpoint directories. Create directories if they do not exist. Recover from an * unsuccessful checkpoint is necessary. * * @param dataDirs * @param editsDirs * @throws IOException */ void recoverCreate(Collection<File> dataDirs, Collection<File> editsDirs) throws IOException { Collection<File> tempDataDirs = new ArrayList<File>(dataDirs); Collection<File> tempEditsDirs = new ArrayList<File>(editsDirs); this.storageDirs = new ArrayList<StorageDirectory>(); setStorageDirectories(tempDataDirs, tempEditsDirs); for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext(); ) { StorageDirectory sd = it.next(); boolean isAccessible = true; try { // create directories if don't exist yet if (!sd.getRoot().mkdirs()) { // do nothing, directory is already created } } catch (SecurityException se) { isAccessible = false; } if (!isAccessible) throw new InconsistentFSStateException( sd.getRoot(), "cannot access checkpoint directory."); StorageState curState; try { curState = sd.analyzeStorage(HdfsConstants.StartupOption.REGULAR); // sd is locked but not opened switch (curState) { case NON_EXISTENT: // fail if any of the configured checkpoint dirs are inaccessible throw new InconsistentFSStateException( sd.getRoot(), "checkpoint directory does not exist or is not accessible."); case NOT_FORMATTED: break; // it's ok since initially there is no current and VERSION case NORMAL: break; default: // recovery is possible sd.doRecover(curState); } } catch (IOException ioe) { sd.unlock(); throw ioe; } } }
private void doTransition( DataNode datanode, StorageDirectory sd, NamespaceInfo nsInfo, StartupOption startOpt) throws IOException { if (startOpt == StartupOption.ROLLBACK && sd.getPreviousDir().exists()) { throw new RuntimeException("ROLLBACK 未实现"); } else { int restored = restoreBlockFilesFromTrash(getTrashRootDir(sd)); LOG.info("从垃圾桶恢复" + restored + "块文件."); } readProperties(sd); checkVersionUpgradable(this.layoutVersion); assert this.layoutVersion >= HdfsConstants.DATANODE_LAYOUT_VERSION : "Future version is not allowed"; if (getNamespaceID() != nsInfo.getNamespaceID()) { throw new IOException( "Incompatible namespaceIDs in " + sd.getRoot().getCanonicalPath() + ": namenode namespaceID = " + nsInfo.getNamespaceID() + "; datanode namespaceID = " + getNamespaceID()); } if (!blockpoolID.equals(nsInfo.getBlockPoolID())) { throw new IOException( "Incompatible blockpoolIDs in " + sd.getRoot().getCanonicalPath() + ": namenode blockpoolID = " + nsInfo.getBlockPoolID() + "; datanode blockpoolID = " + blockpoolID); } if (this.layoutVersion == HdfsConstants.DATANODE_LAYOUT_VERSION && this.cTime == nsInfo.getCTime()) { return; // 正常启动 } throw new RuntimeException("upgrade 未实现"); }
public synchronized boolean createStorageID(StorageDirectory sd, boolean regenerateStorageIds) { final String oldStorageID = sd.getStorageUuid(); if (oldStorageID == null || regenerateStorageIds) { sd.setStorageUuid(DatanodeStorage.generateUuid()); LOG.info( "Generated new storageID " + sd.getStorageUuid() + " for directory " + sd.getRoot() + (oldStorageID == null ? "" : (" to replace " + oldStorageID))); return true; } return false; }
@Override public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException { File oldF = new File(sd.getRoot(), "storage"); if (!oldF.exists()) return false; // check the layout version inside the storage file // Lock and Read old storage file RandomAccessFile oldFile = new RandomAccessFile(oldF, "rws"); FileLock oldLock = oldFile.getChannel().tryLock(); try { oldFile.seek(0); int oldVersion = oldFile.readInt(); if (oldVersion < LAST_PRE_UPGRADE_LAYOUT_VERSION) return false; } finally { oldLock.release(); oldFile.close(); } return true; }
void doFinalize(File dnCurDir) throws IOException { File bpRoot = getBpRoot(blockpoolID, dnCurDir); StorageDirectory bpSd = new StorageDirectory(bpRoot); File prevDir = bpSd.getPreviousDir(); if (!prevDir.exists()) { return; // already finalized } final String dataDirPath = bpSd.getRoot().getCanonicalPath(); LOG.info( "Finalizing upgrade for storage directory " + dataDirPath + ".\n cur LV = " + this.getLayoutVersion() + "; cur CTime = " + this.getCTime()); assert bpSd.getCurrentDir().exists() : "Current directory must exist."; throw new RuntimeException("未实现"); }
/* * Finalize the block pool storage by deleting <BP>/previous directory * that holds the snapshot. */ void doFinalize(File dnCurDir) throws IOException { File bpRoot = getBpRoot(blockpoolID, dnCurDir); StorageDirectory bpSd = new StorageDirectory(bpRoot); // block pool level previous directory File prevDir = bpSd.getPreviousDir(); if (!prevDir.exists()) { return; // already finalized } final String dataDirPath = bpSd.getRoot().getCanonicalPath(); LOG.info( "Finalizing upgrade for storage directory " + dataDirPath + ".\n cur LV = " + this.getLayoutVersion() + "; cur CTime = " + this.getCTime()); assert bpSd.getCurrentDir().exists() : "Current directory must exist."; // rename previous to finalized.tmp final File tmpDir = bpSd.getFinalizedTmp(); rename(prevDir, tmpDir); // delete finalized.tmp dir in a separate thread new Daemon( new Runnable() { @Override public void run() { try { deleteDir(tmpDir); } catch (IOException ex) { LOG.error("Finalize upgrade for " + dataDirPath + " failed.", ex); } LOG.info("Finalize upgrade for " + dataDirPath + " is complete."); } @Override public String toString() { return "Finalize " + dataDirPath; } }) .start(); }
private File getTrashRootDir(StorageDirectory sd) { return new File(sd.getRoot(), TRASH_ROOT_DIR); }
/** * Analyze whether a transition of the BP state is required and perform it if necessary. <br> * Rollback if previousLV >= LAYOUT_VERSION && prevCTime <= namenode.cTime. Upgrade if this.LV > * LAYOUT_VERSION || this.cTime < namenode.cTime Regular startup if this.LV = LAYOUT_VERSION && * this.cTime = namenode.cTime * * @param sd storage directory <SD>/current/<bpid> * @param nsInfo namespace info * @param startOpt startup option * @throws IOException */ private void doTransition( DataNode datanode, StorageDirectory sd, NamespaceInfo nsInfo, StartupOption startOpt) throws IOException { if (startOpt == StartupOption.ROLLBACK && sd.getPreviousDir().exists()) { Preconditions.checkState( !getTrashRootDir(sd).exists(), sd.getPreviousDir() + " and " + getTrashRootDir(sd) + " should not " + " both be present."); doRollback(sd, nsInfo); // rollback if applicable } else if (startOpt == StartupOption.ROLLBACK && !sd.getPreviousDir().exists()) { // Restore all the files in the trash. The restored files are retained // during rolling upgrade rollback. They are deleted during rolling // upgrade downgrade. int restored = restoreBlockFilesFromTrash(getTrashRootDir(sd)); LOG.info("Restored " + restored + " block files from trash."); } readProperties(sd); checkVersionUpgradable(this.layoutVersion); assert this.layoutVersion >= HdfsConstants.DATANODE_LAYOUT_VERSION : "Future version is not allowed"; if (getNamespaceID() != nsInfo.getNamespaceID()) { throw new IOException( "Incompatible namespaceIDs in " + sd.getRoot().getCanonicalPath() + ": namenode namespaceID = " + nsInfo.getNamespaceID() + "; datanode namespaceID = " + getNamespaceID()); } if (!blockpoolID.equals(nsInfo.getBlockPoolID())) { throw new IOException( "Incompatible blockpoolIDs in " + sd.getRoot().getCanonicalPath() + ": namenode blockpoolID = " + nsInfo.getBlockPoolID() + "; datanode blockpoolID = " + blockpoolID); } if (this.layoutVersion == HdfsConstants.DATANODE_LAYOUT_VERSION && this.cTime == nsInfo.getCTime()) { return; // regular startup } if (this.layoutVersion > HdfsConstants.DATANODE_LAYOUT_VERSION) { int restored = restoreBlockFilesFromTrash(getTrashRootDir(sd)); LOG.info( "Restored " + restored + " block files from trash " + "before the layout upgrade. These blocks will be moved to " + "the previous directory during the upgrade"); } if (this.layoutVersion > HdfsConstants.DATANODE_LAYOUT_VERSION || this.cTime < nsInfo.getCTime()) { doUpgrade(datanode, sd, nsInfo); // upgrade return; } // layoutVersion == LAYOUT_VERSION && this.cTime > nsInfo.cTime // must shutdown throw new IOException( "Datanode state: LV = " + this.getLayoutVersion() + " CTime = " + this.getCTime() + " is newer than the namespace state: LV = " + nsInfo.getLayoutVersion() + " CTime = " + nsInfo.getCTime()); }