Пример #1
0
  /**
   * Check to make sure that we are OK to run the passed snapshot. Checks to make sure that we
   * aren't already running a snapshot or restore on the requested table.
   *
   * @param snapshot description of the snapshot we want to start
   * @throws HBaseSnapshotException if the filesystem could not be prepared to start the snapshot
   */
  private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
      throws HBaseSnapshotException {
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
    TableName snapshotTable = TableName.valueOf(snapshot.getTable());

    // make sure we aren't already running a snapshot
    if (isTakingSnapshot(snapshot)) {
      SnapshotSentinel handler = this.snapshotHandlers.get(snapshotTable);
      throw new SnapshotCreationException(
          "Rejected taking "
              + ClientSnapshotDescriptionUtils.toString(snapshot)
              + " because we are already running another snapshot "
              + (handler != null
                  ? ("on the same table "
                      + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()))
                  : "with the same name"),
          ProtobufUtil.createSnapshotDesc(snapshot));
    }

    // make sure we aren't running a restore on the same table
    if (isRestoringTable(snapshotTable)) {
      throw new SnapshotCreationException(
          "Rejected taking "
              + ClientSnapshotDescriptionUtils.toString(snapshot)
              + " because we are already have a restore in progress on the same snapshot.");
    }

    try {
      // delete the working directory, since we aren't running the snapshot. Likely leftovers
      // from a failed attempt.
      fs.delete(workingDir, true);

      // recreate the working directory for the snapshot
      if (!fs.mkdirs(workingDir)) {
        throw new SnapshotCreationException(
            "Couldn't create working directory (" + workingDir + ") for snapshot",
            ProtobufUtil.createSnapshotDesc(snapshot));
      }
    } catch (HBaseSnapshotException e) {
      throw e;
    } catch (IOException e) {
      throw new SnapshotCreationException(
          "Exception while checking to see if snapshot could be started.",
          e,
          ProtobufUtil.createSnapshotDesc(snapshot));
    }
  }
Пример #2
0
 /**
  * Take a snapshot using the specified handler. On failure the snapshot temporary working
  * directory is removed. NOTE: prepareToTakeSnapshot() called before this one takes care of the
  * rejecting the snapshot request if the table is busy with another snapshot/restore operation.
  *
  * @param snapshot the snapshot description
  * @param handler the snapshot handler
  */
 private synchronized void snapshotTable(
     SnapshotDescription snapshot, final TakeSnapshotHandler handler)
     throws HBaseSnapshotException {
   try {
     handler.prepare();
     this.executorService.submit(handler);
     this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
   } catch (Exception e) {
     // cleanup the working directory by trying to delete it from the fs.
     Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
     try {
       if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
         LOG.error(
             "Couldn't delete working directory ("
                 + workingDir
                 + " for snapshot:"
                 + ClientSnapshotDescriptionUtils.toString(snapshot));
       }
     } catch (IOException e1) {
       LOG.error(
           "Couldn't delete working directory ("
               + workingDir
               + " for snapshot:"
               + ClientSnapshotDescriptionUtils.toString(snapshot));
     }
     // fail the snapshot
     throw new SnapshotCreationException(
         "Could not build snapshot handler", e, ProtobufUtil.createSnapshotDesc(snapshot));
   }
 }
Пример #3
0
  /**
   * Check if the specified snapshot is done
   *
   * @param expected
   * @return true if snapshot is ready to be restored, false if it is still being taken.
   * @throws IOException IOException if error from HDFS or RPC
   * @throws UnknownSnapshotException if snapshot is invalid or does not exist.
   */
  public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
    // check the request to make sure it has a snapshot
    if (expected == null) {
      throw new UnknownSnapshotException(
          "No snapshot name passed in request, can't figure out which snapshot you want to check.");
    }

    String ssString = ClientSnapshotDescriptionUtils.toString(expected);

    // check to see if the sentinel exists,
    // and if the task is complete removes it from the in-progress snapshots map.
    SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);

    // stop tracking "abandoned" handlers
    cleanupSentinels();

    if (handler == null) {
      // If there's no handler in the in-progress map, it means one of the following:
      //   - someone has already requested the snapshot state
      //   - the requested snapshot was completed long time ago (cleanupSentinels() timeout)
      //   - the snapshot was never requested
      // In those cases returns to the user the "done state" if the snapshots exists on disk,
      // otherwise raise an exception saying that the snapshot is not running and doesn't exist.
      if (!isSnapshotCompleted(expected)) {
        throw new UnknownSnapshotException(
            "Snapshot "
                + ssString
                + " is not currently running or one of the known completed snapshots.");
      }
      // was done, return true;
      return true;
    }

    // pass on any failure we find in the sentinel
    try {
      handler.rethrowExceptionIfFailed();
    } catch (ForeignException e) {
      // Give some procedure info on an exception.
      String status;
      Procedure p = coordinator.getProcedure(expected.getName());
      if (p != null) {
        status = p.getStatus();
      } else {
        status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
      }
      throw new HBaseSnapshotException(
          "Snapshot " + ssString + " had an error.  " + status,
          e,
          ProtobufUtil.createSnapshotDesc(expected));
    }

    // check to see if we are done
    if (handler.isFinished()) {
      LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
      return true;
    } else if (LOG.isDebugEnabled()) {
      LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
    }
    return false;
  }
Пример #4
0
  /**
   * Delete the specified snapshot
   *
   * @param snapshot
   * @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
   * @throws IOException For filesystem IOExceptions
   */
  public void deleteSnapshot(SnapshotDescription snapshot)
      throws SnapshotDoesNotExistException, IOException {
    // check to see if it is completed
    if (!isSnapshotCompleted(snapshot)) {
      throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot));
    }

    String snapshotName = snapshot.getName();
    // first create the snapshot description and check to see if it exists
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
    // Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
    // just the "name" and it does not contains the "real" snapshot information
    snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

    // call coproc pre hook
    MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
    if (cpHost != null) {
      cpHost.preDeleteSnapshot(snapshot);
    }

    LOG.debug("Deleting snapshot: " + snapshotName);
    // delete the existing snapshot
    if (!fs.delete(snapshotDir, true)) {
      throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
    }

    // call coproc post hook
    if (cpHost != null) {
      cpHost.postDeleteSnapshot(snapshot);
    }
  }
Пример #5
0
 /**
  * Load the information in the SnapshotManifest. Called by SnapshotManifest.open()
  *
  * <p>If the format is v2 and there is no data-manifest, means that we are loading an in-progress
  * snapshot. Since we support rolling-upgrades, we loook for v1 and v2 regions format.
  */
 private void load() throws IOException {
   switch (getSnapshotFormat(desc)) {
     case SnapshotManifestV1.DESCRIPTOR_VERSION:
       {
         this.htd =
             FSTableDescriptors.getTableDescriptorFromFs(fs, workingDir).getHTableDescriptor();
         ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
         try {
           this.regionManifests =
               SnapshotManifestV1.loadRegionManifests(conf, tpool, fs, workingDir, desc);
         } finally {
           tpool.shutdown();
         }
         break;
       }
     case SnapshotManifestV2.DESCRIPTOR_VERSION:
       {
         SnapshotDataManifest dataManifest = readDataManifest();
         if (dataManifest != null) {
           htd = ProtobufUtil.convertToHTableDesc(dataManifest.getTableSchema());
           regionManifests = dataManifest.getRegionManifestsList();
         } else {
           // Compatibility, load the v1 regions
           // This happens only when the snapshot is in-progress and the cache wants to refresh.
           List<SnapshotRegionManifest> v1Regions, v2Regions;
           ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
           try {
             v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, fs, workingDir, desc);
             v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, fs, workingDir, desc);
           } catch (InvalidProtocolBufferException e) {
             throw new CorruptedSnapshotException(
                 "unable to parse region manifest " + e.getMessage(), e);
           } finally {
             tpool.shutdown();
           }
           if (v1Regions != null && v2Regions != null) {
             regionManifests =
                 new ArrayList<SnapshotRegionManifest>(v1Regions.size() + v2Regions.size());
             regionManifests.addAll(v1Regions);
             regionManifests.addAll(v2Regions);
           } else if (v1Regions != null) {
             regionManifests = v1Regions;
           } else /* if (v2Regions != null) */ {
             regionManifests = v2Regions;
           }
         }
         break;
       }
     default:
       throw new CorruptedSnapshotException(
           "Invalid Snapshot version: " + desc.getVersion(),
           ProtobufUtil.createSnapshotDesc(desc));
   }
 }
Пример #6
0
 private RegionVisitor createRegionVisitor(final SnapshotDescription desc) throws IOException {
   switch (getSnapshotFormat(desc)) {
     case SnapshotManifestV1.DESCRIPTOR_VERSION:
       return new SnapshotManifestV1.ManifestBuilder(conf, fs, workingDir);
     case SnapshotManifestV2.DESCRIPTOR_VERSION:
       return new SnapshotManifestV2.ManifestBuilder(conf, fs, workingDir);
     default:
       throw new CorruptedSnapshotException(
           "Invalid Snapshot version: " + desc.getVersion(),
           ProtobufUtil.createSnapshotDesc(desc));
   }
 }
Пример #7
0
  /**
   * Restore or Clone the specified snapshot
   *
   * @param reqSnapshot
   * @param nonceGroup unique value to prevent duplicated RPC
   * @param nonce unique value to prevent duplicated RPC
   * @throws IOException
   */
  public long restoreOrCloneSnapshot(
      SnapshotDescription reqSnapshot, final long nonceGroup, final long nonce) throws IOException {
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);

    // check if the snapshot exists
    if (!fs.exists(snapshotDir)) {
      LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
      throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(reqSnapshot));
    }

    // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
    // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot
    // information.
    SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
    SnapshotManifest manifest =
        SnapshotManifest.open(master.getConfiguration(), fs, snapshotDir, snapshot);
    HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
    TableName tableName = TableName.valueOf(reqSnapshot.getTable());

    // stop tracking "abandoned" handlers
    cleanupSentinels();

    // Verify snapshot validity
    SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);

    // Execute the restore/clone operation
    long procId;
    if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
      procId =
          restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceGroup, nonce);
    } else {
      procId =
          cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceGroup, nonce);
    }
    return procId;
  }
Пример #8
0
  /**
   * Take a snapshot based on the enabled/disabled state of the table.
   *
   * @param snapshot
   * @throws HBaseSnapshotException when a snapshot specific exception occurs.
   * @throws IOException when some sort of generic IO exception occurs.
   */
  public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
    // check to see if we already completed the snapshot
    if (isSnapshotCompleted(snapshot)) {
      throw new SnapshotExistsException(
          "Snapshot '" + snapshot.getName() + "' already stored on the filesystem.",
          ProtobufUtil.createSnapshotDesc(snapshot));
    }

    LOG.debug("No existing snapshot, attempting snapshot...");

    // stop tracking "abandoned" handlers
    cleanupSentinels();

    // check to see if the table exists
    HTableDescriptor desc = null;
    try {
      desc = master.getTableDescriptors().get(TableName.valueOf(snapshot.getTable()));
    } catch (FileNotFoundException e) {
      String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
      LOG.error(msg);
      throw new SnapshotCreationException(msg, e, ProtobufUtil.createSnapshotDesc(snapshot));
    } catch (IOException e) {
      throw new SnapshotCreationException(
          "Error while geting table description for table " + snapshot.getTable(),
          e,
          ProtobufUtil.createSnapshotDesc(snapshot));
    }
    if (desc == null) {
      throw new SnapshotCreationException(
          "Table '" + snapshot.getTable() + "' doesn't exist, can't take snapshot.",
          ProtobufUtil.createSnapshotDesc(snapshot));
    }
    SnapshotDescription.Builder builder = snapshot.toBuilder();
    // if not specified, set the snapshot format
    if (!snapshot.hasVersion()) {
      builder.setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION);
    }
    User user = RpcServer.getRequestUser();
    if (User.isHBaseSecurityEnabled(master.getConfiguration()) && user != null) {
      builder.setOwner(user.getShortName());
    }
    snapshot = builder.build();

    // call pre coproc hook
    MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
    if (cpHost != null) {
      cpHost.preSnapshot(snapshot, desc);
    }

    // if the table is enabled, then have the RS run actually the snapshot work
    TableName snapshotTable = TableName.valueOf(snapshot.getTable());
    if (master.getTableStateManager().isTableState(snapshotTable, TableState.State.ENABLED)) {
      LOG.debug("Table enabled, starting distributed snapshot.");
      snapshotEnabledTable(snapshot);
      LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
    }
    // For disabled table, snapshot is created by the master
    else if (master.getTableStateManager().isTableState(snapshotTable, TableState.State.DISABLED)) {
      LOG.debug("Table is disabled, running snapshot entirely on master.");
      snapshotDisabledTable(snapshot);
      LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
    } else {
      LOG.error(
          "Can't snapshot table '"
              + snapshot.getTable()
              + "', isn't open or closed, we don't know what to do!");
      TablePartiallyOpenException tpoe =
          new TablePartiallyOpenException(snapshot.getTable() + " isn't fully open.");
      throw new SnapshotCreationException(
          "Table is not entirely open or closed", tpoe, ProtobufUtil.createSnapshotDesc(snapshot));
    }

    // call post coproc hook
    if (cpHost != null) {
      cpHost.postSnapshot(snapshot, desc);
    }
  }