예제 #1
0
  /**
   * Delete the specified snapshot
   *
   * @param snapshot
   * @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
   * @throws IOException For filesystem IOExceptions
   */
  public void deleteSnapshot(SnapshotDescription snapshot)
      throws SnapshotDoesNotExistException, IOException {
    // check to see if it is completed
    if (!isSnapshotCompleted(snapshot)) {
      throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot));
    }

    String snapshotName = snapshot.getName();
    // first create the snapshot description and check to see if it exists
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
    // Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
    // just the "name" and it does not contains the "real" snapshot information
    snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

    // call coproc pre hook
    MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
    if (cpHost != null) {
      cpHost.preDeleteSnapshot(snapshot);
    }

    LOG.debug("Deleting snapshot: " + snapshotName);
    // delete the existing snapshot
    if (!fs.delete(snapshotDir, true)) {
      throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
    }

    // call coproc post hook
    if (cpHost != null) {
      cpHost.postDeleteSnapshot(snapshot);
    }
  }
예제 #2
0
 protected void verifyRoundRobinDistribution(HTable ht, int expectedRegions) throws IOException {
   MasterServices services = TEST_UTIL.getMiniHBaseCluster().getMaster();
   AssignmentManager am = services.getAssignmentManager();
   Map<HRegionInfo, HServerAddress> regions = ht.getRegionsInfo();
   for (HRegionInfo regionInfo : regions.keySet()) {
     try {
       am.waitForAssignment(regionInfo);
     } catch (InterruptedException e) {
       LOG.info("Interrupted waiting for region to be assigned during " + "create table call", e);
       Thread.currentThread().interrupt();
       return;
     }
   }
   int numRS = ht.getCurrentNrHRS();
   regions = ht.getRegionsInfo();
   Map<HServerAddress, List<HRegionInfo>> server2Regions =
       new HashMap<HServerAddress, List<HRegionInfo>>();
   for (Map.Entry<HRegionInfo, HServerAddress> entry : regions.entrySet()) {
     HServerAddress server = entry.getValue();
     List<HRegionInfo> regs = server2Regions.get(server);
     if (regs == null) {
       regs = new ArrayList<HRegionInfo>();
       server2Regions.put(server, regs);
     }
     regs.add(entry.getKey());
   }
   float average = (float) expectedRegions / numRS;
   int min = (int) Math.floor(average);
   int max = (int) Math.ceil(average);
   for (List<HRegionInfo> regionList : server2Regions.values()) {
     assertTrue(regionList.size() == min || regionList.size() == max);
   }
 }
예제 #3
0
 private long getRegionSize(HRegionInfo hri) {
   ServerName sn =
       masterServices.getAssignmentManager().getRegionStates().getRegionServerOfRegion(hri);
   RegionLoad regionLoad =
       masterServices.getServerManager().getLoad(sn).getRegionsLoad().get(hri.getRegionName());
   return regionLoad.getStorefileSizeMB();
 }
예제 #4
0
  /**
   * Gets the list of all completed snapshots.
   *
   * @param snapshotDir snapshot directory
   * @return list of SnapshotDescriptions
   * @throws IOException File system exception
   */
  private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
    List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
    // first create the snapshot root path and check to see if it exists
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);

    // if there are no snapshots, return an empty list
    if (!fs.exists(snapshotDir)) {
      return snapshotDescs;
    }

    // ignore all the snapshots in progress
    FileStatus[] snapshots =
        fs.listStatus(
            snapshotDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
    MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
    // loop through all the completed snapshots
    for (FileStatus snapshot : snapshots) {
      Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
      // if the snapshot is bad
      if (!fs.exists(info)) {
        LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
        continue;
      }
      FSDataInputStream in = null;
      try {
        in = fs.open(info);
        SnapshotDescription desc = SnapshotDescription.parseFrom(in);
        if (cpHost != null) {
          try {
            cpHost.preListSnapshot(desc);
          } catch (AccessDeniedException e) {
            LOG.warn(
                "Current user does not have access to "
                    + desc.getName()
                    + " snapshot. "
                    + "Either you should be owner of this snapshot or admin user.");
            // Skip this and try for next snapshot
            continue;
          }
        }
        snapshotDescs.add(desc);

        // call coproc post hook
        if (cpHost != null) {
          cpHost.postListSnapshot(desc);
        }
      } catch (IOException e) {
        LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
      } finally {
        if (in != null) {
          in.close();
        }
      }
    }
    return snapshotDescs;
  }
예제 #5
0
 /**
  * Cleans up any snapshots in the snapshot/.tmp directory that were left from failed snapshot
  * attempts.
  *
  * @throws IOException if we can't reach the filesystem
  */
 void resetTempDir() throws IOException {
   // cleanup any existing snapshots.
   Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
   if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
     if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
       LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
     }
   }
 }
예제 #6
0
  /**
   * Fully specify all necessary components of a snapshot manager. Exposed for testing.
   *
   * @param master services for the master where the manager is running
   * @param coordinator procedure coordinator instance. exposed for testing.
   * @param pool HBase ExecutorServcie instance, exposed for testing.
   */
  public SnapshotManager(
      final MasterServices master,
      final MetricsMaster metricsMaster,
      ProcedureCoordinator coordinator,
      ExecutorService pool)
      throws IOException, UnsupportedOperationException {
    this.master = master;

    this.rootDir = master.getMasterFileSystem().getRootDir();
    checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());

    this.coordinator = coordinator;
    this.executorService = pool;
    resetTempDir();
  }
예제 #7
0
  /**
   * Clone the specified snapshot. The clone will fail if the destination table has a snapshot or
   * restore in progress.
   *
   * @param reqSnapshot Snapshot Descriptor from request
   * @param tableName table to clone
   * @param snapshot Snapshot Descriptor
   * @param snapshotTableDesc Table Descriptor
   * @param nonceGroup unique value to prevent duplicated RPC
   * @param nonce unique value to prevent duplicated RPC
   * @return procId the ID of the clone snapshot procedure
   * @throws IOException
   */
  private long cloneSnapshot(
      final SnapshotDescription reqSnapshot,
      final TableName tableName,
      final SnapshotDescription snapshot,
      final HTableDescriptor snapshotTableDesc,
      final long nonceGroup,
      final long nonce)
      throws IOException {
    MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
    HTableDescriptor htd = new HTableDescriptor(tableName, snapshotTableDesc);
    if (cpHost != null) {
      cpHost.preCloneSnapshot(reqSnapshot, htd);
    }
    long procId;
    try {
      procId = cloneSnapshot(snapshot, htd, nonceGroup, nonce);
    } catch (IOException e) {
      LOG.error(
          "Exception occurred while cloning the snapshot "
              + snapshot.getName()
              + " as table "
              + tableName.getNameAsString(),
          e);
      throw e;
    }
    LOG.info("Clone snapshot=" + snapshot.getName() + " as table=" + tableName);

    if (cpHost != null) {
      cpHost.postCloneSnapshot(reqSnapshot, htd);
    }
    return procId;
  }
예제 #8
0
  /**
   * @param snapshot descriptor of the snapshot to take
   * @param masterServices master services provider
   */
  public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices masterServices) {
    super(masterServices, EventType.C_M_SNAPSHOT_TABLE);
    assert snapshot != null : "SnapshotDescription must not be nul1";
    assert masterServices != null : "MasterServices must not be nul1";

    this.master = masterServices;
    this.snapshot = snapshot;
    this.snapshotTable = TableName.valueOf(snapshot.getTable());
    this.conf = this.master.getConfiguration();
    this.fs = this.master.getMasterFileSystem().getFileSystem();
    this.rootDir = this.master.getMasterFileSystem().getRootDir();
    this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
    this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
    this.monitor = new ForeignExceptionDispatcher(snapshot.getName());
    this.snapshotManifest = SnapshotManifest.create(conf, fs, workingDir, snapshot, monitor);

    this.tableLockManager = master.getTableLockManager();
    this.tableLock =
        this.tableLockManager.writeLock(snapshotTable, EventType.C_M_SNAPSHOT_TABLE.toString());

    // prepare the verify
    this.verifier = new MasterSnapshotVerifier(masterServices, snapshot, rootDir);
    // update the running tasks
    this.status =
        TaskMonitor.get()
            .createStatus("Taking " + snapshot.getType() + " snapshot on table: " + snapshotTable);
  }
예제 #9
0
 /**
  * Modify table is async so wait on completion of the table operation in master.
  *
  * @param tableName
  * @param htd
  * @throws IOException
  */
 private void modifyTable(final byte[] tableName, final HTableDescriptor htd) throws IOException {
   MasterServices services = TEST_UTIL.getMiniHBaseCluster().getMaster();
   ExecutorService executor = services.getExecutorService();
   AtomicBoolean done = new AtomicBoolean(false);
   executor.registerListener(EventType.C_M_MODIFY_TABLE, new DoneListener(done));
   this.admin.modifyTable(tableName, htd);
   while (!done.get()) {
     synchronized (done) {
       try {
         done.wait(1000);
       } catch (InterruptedException e) {
         e.printStackTrace();
       }
     }
   }
   executor.unregisterListener(EventType.C_M_MODIFY_TABLE);
 }
예제 #10
0
  /**
   * Restore the specified snapshot. The restore will fail if the destination table has a snapshot
   * or restore in progress.
   *
   * @param reqSnapshot Snapshot Descriptor from request
   * @param tableName table to restore
   * @param snapshot Snapshot Descriptor
   * @param snapshotTableDesc Table Descriptor
   * @param nonceGroup unique value to prevent duplicated RPC
   * @param nonce unique value to prevent duplicated RPC
   * @return procId the ID of the restore snapshot procedure
   * @throws IOException
   */
  private long restoreSnapshot(
      final SnapshotDescription reqSnapshot,
      final TableName tableName,
      final SnapshotDescription snapshot,
      final HTableDescriptor snapshotTableDesc,
      final long nonceGroup,
      final long nonce)
      throws IOException {
    MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();

    if (master
        .getTableStateManager()
        .isTableState(TableName.valueOf(snapshot.getTable()), TableState.State.ENABLED)) {
      throw new UnsupportedOperationException(
          "Table '"
              + TableName.valueOf(snapshot.getTable())
              + "' must be disabled in order to "
              + "perform a restore operation.");
    }

    // call Coprocessor pre hook
    if (cpHost != null) {
      cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
    }

    long procId;
    try {
      procId = restoreSnapshot(snapshot, snapshotTableDesc, nonceGroup, nonce);
    } catch (IOException e) {
      LOG.error(
          "Exception occurred while restoring the snapshot "
              + snapshot.getName()
              + " as table "
              + tableName.getNameAsString(),
          e);
      throw e;
    }
    LOG.info("Restore snapshot=" + snapshot.getName() + " as table=" + tableName);

    if (cpHost != null) {
      cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
    }

    return procId;
  }
예제 #11
0
  @Override
  public void initialize(MasterServices master, MetricsMaster metricsMaster)
      throws KeeperException, IOException, UnsupportedOperationException {
    this.master = master;

    this.rootDir = master.getMasterFileSystem().getRootDir();
    checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());

    // get the configuration for the coordinator
    Configuration conf = master.getConfiguration();
    long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
    long timeoutMillis =
        Math.max(
            conf.getLong(
                SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_KEY,
                SnapshotDescriptionUtils.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT),
            conf.getLong(
                SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS,
                SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME));
    int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);

    // setup the default procedure coordinator
    String name = master.getServerName().toString();
    ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
    ProcedureCoordinatorRpcs comms =
        new ZKProcedureCoordinatorRpcs(
            master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);

    this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
    this.executorService = master.getExecutorService();
    resetTempDir();
  }
예제 #12
0
 /**
  * Check to see if the snapshot is one of the currently completed snapshots Returns true if the
  * snapshot exists in the "completed snapshots folder".
  *
  * @param snapshot expected snapshot to check
  * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
  *     not stored
  * @throws IOException if the filesystem throws an unexpected exception,
  * @throws IllegalArgumentException if snapshot name is invalid.
  */
 private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
   try {
     final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
     FileSystem fs = master.getMasterFileSystem().getFileSystem();
     // check to see if the snapshot already exists
     return fs.exists(snapshotDir);
   } catch (IllegalArgumentException iae) {
     throw new UnknownSnapshotException("Unexpected exception thrown", iae);
   }
 }
예제 #13
0
 /** Remove the procedures that are marked as finished */
 private synchronized void cleanupCompletedRestoreInMap() {
   ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
   Iterator<Map.Entry<TableName, Long>> it = restoreTableToProcIdMap.entrySet().iterator();
   while (it.hasNext()) {
     Map.Entry<TableName, Long> entry = it.next();
     Long procId = entry.getValue();
     if (procExec.isRunning() && procExec.isFinished(procId)) {
       it.remove();
     }
   }
 }
예제 #14
0
  /**
   * Restore the specified snapshot. The restore will fail if the destination table has a snapshot
   * or restore in progress.
   *
   * @param snapshot Snapshot Descriptor
   * @param hTableDescriptor Table Descriptor
   * @param nonceGroup unique value to prevent duplicated RPC
   * @param nonce unique value to prevent duplicated RPC
   * @return procId the ID of the restore snapshot procedure
   */
  private synchronized long restoreSnapshot(
      final SnapshotDescription snapshot,
      final HTableDescriptor hTableDescriptor,
      final long nonceGroup,
      final long nonce)
      throws HBaseSnapshotException {
    TableName tableName = hTableDescriptor.getTableName();

    // make sure we aren't running a snapshot on the same table
    if (isTakingSnapshot(tableName)) {
      throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
    }

    // make sure we aren't running a restore on the same table
    if (isRestoringTable(tableName)) {
      throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
    }

    try {
      long procId =
          master
              .getMasterProcedureExecutor()
              .submitProcedure(
                  new RestoreSnapshotProcedure(
                      master.getMasterProcedureExecutor().getEnvironment(),
                      hTableDescriptor,
                      snapshot),
                  nonceGroup,
                  nonce);
      this.restoreTableToProcIdMap.put(tableName, procId);
      return procId;
    } catch (Exception e) {
      String msg =
          "Couldn't restore the snapshot="
              + ClientSnapshotDescriptionUtils.toString(snapshot)
              + " on table="
              + tableName;
      LOG.error(msg, e);
      throw new RestoreSnapshotException(msg, e);
    }
  }
예제 #15
0
 /**
  * Verify if the restore of the specified table is in progress.
  *
  * @param tableName table under restore
  * @return <tt>true</tt> if there is a restore in progress of the specified table.
  */
 private synchronized boolean isRestoringTable(final TableName tableName) {
   Long procId = this.restoreTableToProcIdMap.get(tableName);
   if (procId == null) {
     return false;
   }
   ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
   if (procExec.isRunning() && !procExec.isFinished(procId)) {
     return true;
   } else {
     this.restoreTableToProcIdMap.remove(tableName);
     return false;
   }
 }
예제 #16
0
  @Override
  public void cancel(String why) {
    if (finished) return;

    this.finished = true;
    LOG.info(
        "Stop taking snapshot="
            + ClientSnapshotDescriptionUtils.toString(snapshot)
            + " because: "
            + why);
    CancellationException ce = new CancellationException(why);
    monitor.receive(new ForeignException(master.getServerName().toString(), ce));
  }
예제 #17
0
  /**
   * Restore or Clone the specified snapshot
   *
   * @param reqSnapshot
   * @param nonceGroup unique value to prevent duplicated RPC
   * @param nonce unique value to prevent duplicated RPC
   * @throws IOException
   */
  public long restoreOrCloneSnapshot(
      SnapshotDescription reqSnapshot, final long nonceGroup, final long nonce) throws IOException {
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);

    // check if the snapshot exists
    if (!fs.exists(snapshotDir)) {
      LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
      throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(reqSnapshot));
    }

    // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
    // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot
    // information.
    SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
    SnapshotManifest manifest =
        SnapshotManifest.open(master.getConfiguration(), fs, snapshotDir, snapshot);
    HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
    TableName tableName = TableName.valueOf(reqSnapshot.getTable());

    // stop tracking "abandoned" handlers
    cleanupSentinels();

    // Verify snapshot validity
    SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);

    // Execute the restore/clone operation
    long procId;
    if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
      procId =
          restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceGroup, nonce);
    } else {
      procId =
          cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceGroup, nonce);
    }
    return procId;
  }
예제 #18
0
  /**
   * Check to make sure that we are OK to run the passed snapshot. Checks to make sure that we
   * aren't already running a snapshot or restore on the requested table.
   *
   * @param snapshot description of the snapshot we want to start
   * @throws HBaseSnapshotException if the filesystem could not be prepared to start the snapshot
   */
  private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
      throws HBaseSnapshotException {
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
    TableName snapshotTable = TableName.valueOf(snapshot.getTable());

    // make sure we aren't already running a snapshot
    if (isTakingSnapshot(snapshot)) {
      SnapshotSentinel handler = this.snapshotHandlers.get(snapshotTable);
      throw new SnapshotCreationException(
          "Rejected taking "
              + ClientSnapshotDescriptionUtils.toString(snapshot)
              + " because we are already running another snapshot "
              + (handler != null
                  ? ("on the same table "
                      + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()))
                  : "with the same name"),
          ProtobufUtil.createSnapshotDesc(snapshot));
    }

    // make sure we aren't running a restore on the same table
    if (isRestoringTable(snapshotTable)) {
      throw new SnapshotCreationException(
          "Rejected taking "
              + ClientSnapshotDescriptionUtils.toString(snapshot)
              + " because we are already have a restore in progress on the same snapshot.");
    }

    try {
      // delete the working directory, since we aren't running the snapshot. Likely leftovers
      // from a failed attempt.
      fs.delete(workingDir, true);

      // recreate the working directory for the snapshot
      if (!fs.mkdirs(workingDir)) {
        throw new SnapshotCreationException(
            "Couldn't create working directory (" + workingDir + ") for snapshot",
            ProtobufUtil.createSnapshotDesc(snapshot));
      }
    } catch (HBaseSnapshotException e) {
      throw e;
    } catch (IOException e) {
      throw new SnapshotCreationException(
          "Exception while checking to see if snapshot could be started.",
          e,
          ProtobufUtil.createSnapshotDesc(snapshot));
    }
  }
예제 #19
0
  /**
   * Computes next most "urgent" normalization action on the table. Action may be either a split, or
   * a merge, or no action.
   *
   * @param table table to normalize
   * @param types desired types of NormalizationPlan
   * @return normalization plan to execute
   */
  @Override
  public List<NormalizationPlan> computePlanForTable(TableName table, List<PlanType> types)
      throws HBaseIOException {
    if (table == null || table.isSystemTable()) {
      LOG.debug("Normalization of system table " + table + " isn't allowed");
      return null;
    }

    List<NormalizationPlan> plans = new ArrayList<NormalizationPlan>();
    List<HRegionInfo> tableRegions =
        masterServices.getAssignmentManager().getRegionStates().getRegionsOfTable(table);

    // TODO: should we make min number of regions a config param?
    if (tableRegions == null || tableRegions.size() < MIN_REGION_COUNT) {
      int nrRegions = tableRegions == null ? 0 : tableRegions.size();
      LOG.debug(
          "Table "
              + table
              + " has "
              + nrRegions
              + " regions, required min number"
              + " of regions for normalizer to run is "
              + MIN_REGION_COUNT
              + ", not running normalizer");
      return null;
    }

    LOG.debug(
        "Computing normalization plan for table: "
            + table
            + ", number of regions: "
            + tableRegions.size());

    long totalSizeMb = 0;

    for (int i = 0; i < tableRegions.size(); i++) {
      HRegionInfo hri = tableRegions.get(i);
      long regionSize = getRegionSize(hri);
      totalSizeMb += regionSize;
    }

    double avgRegionSize = totalSizeMb / (double) tableRegions.size();

    LOG.debug("Table " + table + ", total aggregated regions size: " + totalSizeMb);
    LOG.debug("Table " + table + ", average region size: " + avgRegionSize);

    int candidateIdx = 0;
    while (candidateIdx < tableRegions.size()) {
      HRegionInfo hri = tableRegions.get(candidateIdx);
      long regionSize = getRegionSize(hri);
      // if the region is > 2 times larger than average, we split it, split
      // is more high priority normalization action than merge.
      if (types.contains(PlanType.SPLIT) && regionSize > 2 * avgRegionSize) {
        LOG.debug(
            "Table "
                + table
                + ", large region "
                + hri.getRegionNameAsString()
                + " has size "
                + regionSize
                + ", more than twice avg size, splitting");
        plans.add(new SplitNormalizationPlan(hri, null));
      } else {
        if (candidateIdx == tableRegions.size() - 1) {
          break;
        }
        HRegionInfo hri2 = tableRegions.get(candidateIdx + 1);
        long regionSize2 = getRegionSize(hri2);
        if (types.contains(PlanType.MERGE) && regionSize + regionSize2 < avgRegionSize) {
          LOG.debug(
              "Table "
                  + table
                  + ", small region size: "
                  + regionSize
                  + " plus its neighbor size: "
                  + regionSize2
                  + ", less than the avg size "
                  + avgRegionSize
                  + ", merging them");
          plans.add(new MergeNormalizationPlan(hri, hri2));
          candidateIdx++;
        }
      }
      candidateIdx++;
    }
    if (plans.isEmpty()) {
      LOG.debug("No normalization needed, regions look good for table: " + table);
      return null;
    }
    Collections.sort(plans, planComparator);
    return plans;
  }
예제 #20
0
  /**
   * Take a snapshot based on the enabled/disabled state of the table.
   *
   * @param snapshot
   * @throws HBaseSnapshotException when a snapshot specific exception occurs.
   * @throws IOException when some sort of generic IO exception occurs.
   */
  public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
    // check to see if we already completed the snapshot
    if (isSnapshotCompleted(snapshot)) {
      throw new SnapshotExistsException(
          "Snapshot '" + snapshot.getName() + "' already stored on the filesystem.",
          ProtobufUtil.createSnapshotDesc(snapshot));
    }

    LOG.debug("No existing snapshot, attempting snapshot...");

    // stop tracking "abandoned" handlers
    cleanupSentinels();

    // check to see if the table exists
    HTableDescriptor desc = null;
    try {
      desc = master.getTableDescriptors().get(TableName.valueOf(snapshot.getTable()));
    } catch (FileNotFoundException e) {
      String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
      LOG.error(msg);
      throw new SnapshotCreationException(msg, e, ProtobufUtil.createSnapshotDesc(snapshot));
    } catch (IOException e) {
      throw new SnapshotCreationException(
          "Error while geting table description for table " + snapshot.getTable(),
          e,
          ProtobufUtil.createSnapshotDesc(snapshot));
    }
    if (desc == null) {
      throw new SnapshotCreationException(
          "Table '" + snapshot.getTable() + "' doesn't exist, can't take snapshot.",
          ProtobufUtil.createSnapshotDesc(snapshot));
    }
    SnapshotDescription.Builder builder = snapshot.toBuilder();
    // if not specified, set the snapshot format
    if (!snapshot.hasVersion()) {
      builder.setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION);
    }
    User user = RpcServer.getRequestUser();
    if (User.isHBaseSecurityEnabled(master.getConfiguration()) && user != null) {
      builder.setOwner(user.getShortName());
    }
    snapshot = builder.build();

    // call pre coproc hook
    MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
    if (cpHost != null) {
      cpHost.preSnapshot(snapshot, desc);
    }

    // if the table is enabled, then have the RS run actually the snapshot work
    TableName snapshotTable = TableName.valueOf(snapshot.getTable());
    if (master.getTableStateManager().isTableState(snapshotTable, TableState.State.ENABLED)) {
      LOG.debug("Table enabled, starting distributed snapshot.");
      snapshotEnabledTable(snapshot);
      LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
    }
    // For disabled table, snapshot is created by the master
    else if (master.getTableStateManager().isTableState(snapshotTable, TableState.State.DISABLED)) {
      LOG.debug("Table is disabled, running snapshot entirely on master.");
      snapshotDisabledTable(snapshot);
      LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
    } else {
      LOG.error(
          "Can't snapshot table '"
              + snapshot.getTable()
              + "', isn't open or closed, we don't know what to do!");
      TablePartiallyOpenException tpoe =
          new TablePartiallyOpenException(snapshot.getTable() + " isn't fully open.");
      throw new SnapshotCreationException(
          "Table is not entirely open or closed", tpoe, ProtobufUtil.createSnapshotDesc(snapshot));
    }

    // call post coproc hook
    if (cpHost != null) {
      cpHost.postSnapshot(snapshot, desc);
    }
  }