/** * Delete the specified snapshot * * @param snapshot * @throws SnapshotDoesNotExistException If the specified snapshot does not exist. * @throws IOException For filesystem IOExceptions */ public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException { // check to see if it is completed if (!isSnapshotCompleted(snapshot)) { throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot)); } String snapshotName = snapshot.getName(); // first create the snapshot description and check to see if it exists FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); // Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with // just the "name" and it does not contains the "real" snapshot information snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); // call coproc pre hook MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preDeleteSnapshot(snapshot); } LOG.debug("Deleting snapshot: " + snapshotName); // delete the existing snapshot if (!fs.delete(snapshotDir, true)) { throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir); } // call coproc post hook if (cpHost != null) { cpHost.postDeleteSnapshot(snapshot); } }
/** * @param snapshot descriptor of the snapshot to take * @param masterServices master services provider */ public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices masterServices) { super(masterServices, EventType.C_M_SNAPSHOT_TABLE); assert snapshot != null : "SnapshotDescription must not be nul1"; assert masterServices != null : "MasterServices must not be nul1"; this.master = masterServices; this.snapshot = snapshot; this.snapshotTable = TableName.valueOf(snapshot.getTable()); this.conf = this.master.getConfiguration(); this.fs = this.master.getMasterFileSystem().getFileSystem(); this.rootDir = this.master.getMasterFileSystem().getRootDir(); this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); this.monitor = new ForeignExceptionDispatcher(snapshot.getName()); this.snapshotManifest = SnapshotManifest.create(conf, fs, workingDir, snapshot, monitor); this.tableLockManager = master.getTableLockManager(); this.tableLock = this.tableLockManager.writeLock(snapshotTable, EventType.C_M_SNAPSHOT_TABLE.toString()); // prepare the verify this.verifier = new MasterSnapshotVerifier(masterServices, snapshot, rootDir); // update the running tasks this.status = TaskMonitor.get() .createStatus("Taking " + snapshot.getType() + " snapshot on table: " + snapshotTable); }
/** * Take a snapshot using the specified handler. On failure the snapshot temporary working * directory is removed. NOTE: prepareToTakeSnapshot() called before this one takes care of the * rejecting the snapshot request if the table is busy with another snapshot/restore operation. * * @param snapshot the snapshot description * @param handler the snapshot handler */ private synchronized void snapshotTable( SnapshotDescription snapshot, final TakeSnapshotHandler handler) throws HBaseSnapshotException { try { handler.prepare(); this.executorService.submit(handler); this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler); } catch (Exception e) { // cleanup the working directory by trying to delete it from the fs. Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); try { if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) { LOG.error( "Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } } catch (IOException e1) { LOG.error( "Couldn't delete working directory (" + workingDir + " for snapshot:" + ClientSnapshotDescriptionUtils.toString(snapshot)); } // fail the snapshot throw new SnapshotCreationException( "Could not build snapshot handler", e, ProtobufUtil.createSnapshotDesc(snapshot)); } }
/** * Create a snapshot timer for the master which notifies the monitor when an error occurs * * @param snapshot snapshot to monitor * @param conf configuration to use when getting the max snapshot life * @param monitor monitor to notify when the snapshot life expires * @return the timer to use update to signal the start and end of the snapshot */ private TimeoutExceptionInjector getMasterTimerAndBindToMonitor( SnapshotDescription snapshot, Configuration conf, ForeignExceptionListener monitor) { long maxTime = SnapshotDescriptionUtils.getMaxMasterTimeout( conf, snapshot.getType(), SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME); return new TimeoutExceptionInjector(monitor, maxTime); }
/** * Cleans up any snapshots in the snapshot/.tmp directory that were left from failed snapshot * attempts. * * @throws IOException if we can't reach the filesystem */ void resetTempDir() throws IOException { // cleanup any existing snapshots. Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir); if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) { if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) { LOG.warn("Couldn't delete working snapshot directory: " + tmpdir); } } }
/** * Gets the list of all completed snapshots. * * @param snapshotDir snapshot directory * @return list of SnapshotDescriptions * @throws IOException File system exception */ private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException { List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>(); // first create the snapshot root path and check to see if it exists FileSystem fs = master.getMasterFileSystem().getFileSystem(); if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); // if there are no snapshots, return an empty list if (!fs.exists(snapshotDir)) { return snapshotDescs; } // ignore all the snapshots in progress FileStatus[] snapshots = fs.listStatus( snapshotDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs)); MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost(); // loop through all the completed snapshots for (FileStatus snapshot : snapshots) { Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE); // if the snapshot is bad if (!fs.exists(info)) { LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist"); continue; } FSDataInputStream in = null; try { in = fs.open(info); SnapshotDescription desc = SnapshotDescription.parseFrom(in); if (cpHost != null) { try { cpHost.preListSnapshot(desc); } catch (AccessDeniedException e) { LOG.warn( "Current user does not have access to " + desc.getName() + " snapshot. " + "Either you should be owner of this snapshot or admin user."); // Skip this and try for next snapshot continue; } } snapshotDescs.add(desc); // call coproc post hook if (cpHost != null) { cpHost.postListSnapshot(desc); } } catch (IOException e) { LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e); } finally { if (in != null) { in.close(); } } } return snapshotDescs; }
/** * Check to see if the snapshot is one of the currently completed snapshots Returns true if the * snapshot exists in the "completed snapshots folder". * * @param snapshot expected snapshot to check * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is * not stored * @throws IOException if the filesystem throws an unexpected exception, * @throws IllegalArgumentException if snapshot name is invalid. */ private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException { try { final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); FileSystem fs = master.getMasterFileSystem().getFileSystem(); // check to see if the snapshot already exists return fs.exists(snapshotDir); } catch (IllegalArgumentException iae) { throw new UnknownSnapshotException("Unexpected exception thrown", iae); } }
/** * Restore or Clone the specified snapshot * * @param reqSnapshot * @param nonceGroup unique value to prevent duplicated RPC * @param nonce unique value to prevent duplicated RPC * @throws IOException */ public long restoreOrCloneSnapshot( SnapshotDescription reqSnapshot, final long nonceGroup, final long nonce) throws IOException { FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir); // check if the snapshot exists if (!fs.exists(snapshotDir)) { LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist."); throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(reqSnapshot)); } // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot // information. SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs, snapshotDir, snapshot); HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor(); TableName tableName = TableName.valueOf(reqSnapshot.getTable()); // stop tracking "abandoned" handlers cleanupSentinels(); // Verify snapshot validity SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest); // Execute the restore/clone operation long procId; if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) { procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceGroup, nonce); } else { procId = cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceGroup, nonce); } return procId; }
/** * Check to make sure that we are OK to run the passed snapshot. Checks to make sure that we * aren't already running a snapshot or restore on the requested table. * * @param snapshot description of the snapshot we want to start * @throws HBaseSnapshotException if the filesystem could not be prepared to start the snapshot */ private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot) throws HBaseSnapshotException { FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); TableName snapshotTable = TableName.valueOf(snapshot.getTable()); // make sure we aren't already running a snapshot if (isTakingSnapshot(snapshot)) { SnapshotSentinel handler = this.snapshotHandlers.get(snapshotTable); throw new SnapshotCreationException( "Rejected taking " + ClientSnapshotDescriptionUtils.toString(snapshot) + " because we are already running another snapshot " + (handler != null ? ("on the same table " + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot())) : "with the same name"), ProtobufUtil.createSnapshotDesc(snapshot)); } // make sure we aren't running a restore on the same table if (isRestoringTable(snapshotTable)) { throw new SnapshotCreationException( "Rejected taking " + ClientSnapshotDescriptionUtils.toString(snapshot) + " because we are already have a restore in progress on the same snapshot."); } try { // delete the working directory, since we aren't running the snapshot. Likely leftovers // from a failed attempt. fs.delete(workingDir, true); // recreate the working directory for the snapshot if (!fs.mkdirs(workingDir)) { throw new SnapshotCreationException( "Couldn't create working directory (" + workingDir + ") for snapshot", ProtobufUtil.createSnapshotDesc(snapshot)); } } catch (HBaseSnapshotException e) { throw e; } catch (IOException e) { throw new SnapshotCreationException( "Exception while checking to see if snapshot could be started.", e, ProtobufUtil.createSnapshotDesc(snapshot)); } }
/** * Gets the list of all completed snapshots. * * @return list of SnapshotDescriptions * @throws IOException File system exception */ public List<SnapshotDescription> getCompletedSnapshots() throws IOException { return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir)); }
/** * Called at startup, to verify if snapshot operation is supported, and to avoid starting the * master if there're snapshots present but the cleaners needed are missing. Otherwise we can end * up with snapshot data loss. * * @param conf The {@link Configuration} object to use * @param mfs The MasterFileSystem to use * @throws IOException in case of file-system operation failure * @throws UnsupportedOperationException in case cleaners are missing and there're snapshot in the * system */ private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs) throws IOException, UnsupportedOperationException { // Verify if snapshot is disabled by the user String enabled = conf.get(HBASE_SNAPSHOT_ENABLED); boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false); boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled); // Extract cleaners from conf Set<String> hfileCleaners = new HashSet<String>(); String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS); if (cleaners != null) Collections.addAll(hfileCleaners, cleaners); Set<String> logCleaners = new HashSet<String>(); cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS); if (cleaners != null) Collections.addAll(logCleaners, cleaners); // check if an older version of snapshot directory was present Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME); FileSystem fs = mfs.getFileSystem(); List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir)); if (ss != null && !ss.isEmpty()) { LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir); LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME); } // If the user has enabled the snapshot, we force the cleaners to be present // otherwise we still need to check if cleaners are enabled or not and verify // that there're no snapshot in the .snapshot folder. if (snapshotEnabled) { // Inject snapshot cleaners, if snapshot.enable is true hfileCleaners.add(SnapshotHFileCleaner.class.getName()); hfileCleaners.add(HFileLinkCleaner.class.getName()); // Set cleaners conf conf.setStrings( HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, hfileCleaners.toArray(new String[hfileCleaners.size()])); conf.setStrings( HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, logCleaners.toArray(new String[logCleaners.size()])); } else { // Verify if cleaners are present snapshotEnabled = hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) && hfileCleaners.contains(HFileLinkCleaner.class.getName()); // Warn if the cleaners are enabled but the snapshot.enabled property is false/not set. if (snapshotEnabled) { LOG.warn( "Snapshot log and hfile cleaners are present in the configuration, " + "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " + (userDisabled ? "is set to 'false'." : "is not set.")); } } // Mark snapshot feature as enabled if cleaners are present and user has not disabled it. this.isSnapshotSupported = snapshotEnabled && !userDisabled; // If cleaners are not enabled, verify that there're no snapshot in the .snapshot folder // otherwise we end up with snapshot data loss. if (!snapshotEnabled) { LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners."); Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir()); if (fs.exists(snapshotDir)) { FileStatus[] snapshots = FSUtils.listStatus( fs, snapshotDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs)); if (snapshots != null) { LOG.error("Snapshots are present, but cleaners are not enabled."); checkSnapshotSupport(); } } } }
/** * Execute the core common portions of taking a snapshot. The {@link #snapshotRegions(List)} call * should get implemented for each snapshot flavor. */ @Override @edu.umd.cs.findbugs.annotations.SuppressWarnings( value = "REC_CATCH_EXCEPTION", justification = "Intentional") public void process() { String msg = "Running " + snapshot.getType() + " table snapshot " + snapshot.getName() + " " + eventType + " on table " + snapshotTable; LOG.info(msg); status.setStatus(msg); try { // If regions move after this meta scan, the region specific snapshot should fail, triggering // an external exception that gets captured here. // write down the snapshot info in the working directory SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs); snapshotManifest.addTableDescriptor(this.htd); monitor.rethrowException(); List<Pair<HRegionInfo, ServerName>> regionsAndLocations; if (TableName.META_TABLE_NAME.equals(snapshotTable)) { regionsAndLocations = new MetaTableLocator().getMetaRegionsAndLocations(server.getZooKeeper()); } else { regionsAndLocations = MetaTableAccessor.getTableRegionsAndLocations( server.getConnection(), snapshotTable, false); } // run the snapshot snapshotRegions(regionsAndLocations); monitor.rethrowException(); // extract each pair to separate lists Set<String> serverNames = new HashSet<String>(); for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) { if (p != null && p.getFirst() != null && p.getSecond() != null) { HRegionInfo hri = p.getFirst(); if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) continue; serverNames.add(p.getSecond().toString()); } } // flush the in-memory state, and write the single manifest status.setStatus("Consolidate snapshot: " + snapshot.getName()); snapshotManifest.consolidate(); // verify the snapshot is valid status.setStatus("Verifying snapshot: " + snapshot.getName()); verifier.verifySnapshot(this.workingDir, serverNames); // complete the snapshot, atomically moving from tmp to .snapshot dir. completeSnapshot(this.snapshotDir, this.workingDir, this.fs); msg = "Snapshot " + snapshot.getName() + " of table " + snapshotTable + " completed"; status.markComplete(msg); LOG.info(msg); metricsSnapshot.addSnapshot(status.getCompletionTimestamp() - status.getStartTime()); } catch (Exception e) { // FindBugs: REC_CATCH_EXCEPTION status.abort( "Failed to complete snapshot " + snapshot.getName() + " on table " + snapshotTable + " because " + e.getMessage()); String reason = "Failed taking snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " due to exception:" + e.getMessage(); LOG.error(reason, e); ForeignException ee = new ForeignException(reason, e); monitor.receive(ee); // need to mark this completed to close off and allow cleanup to happen. cancel(reason); } finally { LOG.debug("Launching cleanup of working dir:" + workingDir); try { // if the working dir is still present, the snapshot has failed. it is present we delete // it. if (fs.exists(workingDir) && !this.fs.delete(workingDir, true)) { LOG.error("Couldn't delete snapshot working directory:" + workingDir); } } catch (IOException e) { LOG.error("Couldn't delete snapshot working directory:" + workingDir); } releaseTableLock(); } }