Example #1
0
 /**
  * Creates or force updates an unassigned node to the OFFLINE state for the specified region.
  *
  * <p>Attempts to create the node but if it exists will force it to transition to and OFFLINE
  * state.
  *
  * <p>Sets a watcher on the unassigned region node if the method is successful.
  *
  * <p>This method should be used when assigning a region.
  *
  * @param zkw zk reference
  * @param region region to be created as offline
  * @param serverName server event originates from
  * @throws KeeperException if unexpected zookeeper exception
  * @throws KeeperException.NodeExistsException if node already exists
  */
 public static boolean createOrForceNodeOffline(
     ZooKeeperWatcher zkw, HRegionInfo region, String serverName) throws KeeperException {
   LOG.debug(
       zkw.prefix(
           "Creating (or updating) unassigned node for "
               + region.getEncodedName()
               + " with OFFLINE state"));
   RegionTransitionData data =
       new RegionTransitionData(EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName);
   synchronized (zkw.getNodes()) {
     String node = getNodeName(zkw, region.getEncodedName());
     zkw.sync(node);
     zkw.getNodes().add(node);
     int version = ZKUtil.checkExists(zkw, node);
     if (version == -1) {
       ZKUtil.createAndWatch(zkw, node, data.getBytes());
     } else {
       if (!ZKUtil.setData(zkw, node, data.getBytes(), version)) {
         return false;
       } else {
         // We successfully forced to OFFLINE, reset watch and handle if
         // the state changed in between our set and the watch
         RegionTransitionData curData = ZKAssign.getData(zkw, region.getEncodedName());
         if (curData.getEventType() != data.getEventType()) {
           // state changed, need to process
           return false;
         }
       }
     }
   }
   return true;
 }
  /**
   * Create a new {@link Reference} as copy of the source one.
   *
   * <p>
   *
   * <blockquote>
   *
   * <pre>
   * The source table looks like:
   *    1234/abc      (original file)
   *    5678/abc.1234 (reference file)
   *
   * After the clone operation looks like:
   *   wxyz/table=1234-abc
   *   stuv/table=1234-abc.wxyz
   *
   * NOTE that the region name in the clone changes (md5 of regioninfo)
   * and the reference should reflect that change.
   * </pre>
   *
   * </blockquote>
   *
   * @param familyDir destination directory for the store file
   * @param regionInfo destination region info for the table
   * @param hfileName reference file name
   */
  private void restoreReferenceFile(
      final Path familyDir, final HRegionInfo regionInfo, final String hfileName)
      throws IOException {
    // Extract the referred information (hfile name and parent region)
    String tableName = snapshotDesc.getTable();
    Path refPath =
        StoreFile.getReferredToFile(
            new Path(
                new Path(new Path(tableName, regionInfo.getEncodedName()), familyDir.getName()),
                hfileName));
    String snapshotRegionName = refPath.getParent().getParent().getName();
    String fileName = refPath.getName();

    // The new reference should have the cloned region name as parent, if it is a clone.
    String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
    if (clonedRegionName == null) clonedRegionName = snapshotRegionName;

    // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
    String refLink = fileName;
    if (!HFileLink.isHFileLink(fileName)) {
      refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName);
    }
    Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);

    // Create the new reference
    Path linkPath =
        new Path(
            familyDir,
            HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName));
    InputStream in = new HFileLink(conf, linkPath).open(fs);
    OutputStream out = fs.create(outPath);
    IOUtils.copyBytes(in, out, conf);
  }
Example #3
0
 public static void createNodeOffline(
     ZooKeeperWatcher zkw, HRegionInfo region, String serverName, final EventType event)
     throws KeeperException, KeeperException.NodeExistsException {
   LOG.debug(
       zkw.prefix(
           "Creating unassigned node for " + region.getEncodedName() + " in OFFLINE state"));
   RegionTransitionData data = new RegionTransitionData(event, region.getRegionName(), serverName);
   synchronized (zkw.getNodes()) {
     String node = getNodeName(zkw, region.getEncodedName());
     zkw.getNodes().add(node);
     ZKUtil.createAndWatch(zkw, node, data.getBytes());
   }
 }
Example #4
0
 /**
  * Forces an existing unassigned node to the OFFLINE state for the specified region.
  *
  * <p>Does not create a new node. If a node does not already exist for this region, a {@link
  * NoNodeException} will be thrown.
  *
  * <p>Sets a watcher on the unassigned region node if the method is successful.
  *
  * <p>This method should only be used during recovery of regionserver failure.
  *
  * @param zkw zk reference
  * @param region region to be forced as offline
  * @param serverName server event originates from
  * @throws KeeperException if unexpected zookeeper exception
  * @throws KeeperException.NoNodeException if node does not exist
  */
 public static void forceNodeOffline(ZooKeeperWatcher zkw, HRegionInfo region, String serverName)
     throws KeeperException, KeeperException.NoNodeException {
   LOG.debug(
       zkw.prefix(
           "Forcing existing unassigned node for "
               + region.getEncodedName()
               + " to OFFLINE state"));
   RegionTransitionData data =
       new RegionTransitionData(EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName);
   synchronized (zkw.getNodes()) {
     String node = getNodeName(zkw, region.getEncodedName());
     zkw.getNodes().add(node);
     ZKUtil.setData(zkw, node, data.getBytes());
   }
 }
Example #5
0
  /**
   * Write out an info file under the stored region directory. Useful recovering mangled regions. If
   * the regionInfo already exists on-disk, then we fast exit.
   */
  void checkRegionInfoOnFilesystem() throws IOException {
    // Compose the content of the file so we can compare to length in filesystem. If not same,
    // rewrite it (it may have been written in the old format using Writables instead of pb). The
    // pb version is much shorter -- we write now w/o the toString version -- so checking length
    // only should be sufficient. I don't want to read the file every time to check if it pb
    // serialized.
    byte[] content = getRegionInfoFileContent(regionInfo);
    try {
      Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);

      FileStatus status = fs.getFileStatus(regionInfoFile);
      if (status != null && status.getLen() == content.length) {
        // Then assume the content good and move on.
        // NOTE: that the length is not sufficient to define the the content matches.
        return;
      }

      LOG.info("Rewriting .regioninfo file at: " + regionInfoFile);
      if (!fs.delete(regionInfoFile, false)) {
        throw new IOException("Unable to remove existing " + regionInfoFile);
      }
    } catch (FileNotFoundException e) {
      LOG.warn(REGION_INFO_FILE + " file not found for region: " + regionInfo.getEncodedName());
    }

    // Write HRI to a file in case we need to recover hbase:meta
    writeRegionInfoOnFilesystem(content, true);
  }
  public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
      throws IOException {
    // archive family store files
    Path tableDir = new Path(rootdir, region.getTableNameAsString());
    HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);

    // delete the family folder
    Path familyDir = new Path(tableDir,
      new Path(region.getEncodedName(), Bytes.toString(familyName)));
    if (fs.delete(familyDir, true) == false) {
      throw new IOException("Could not delete family "
          + Bytes.toString(familyName) + " from FileSystem for region "
          + region.getRegionNameAsString() + "(" + region.getEncodedName()
          + ")");
    }
  }
  /**
   * Test if the region can be closed properly
   *
   * @throws IOException
   * @throws NodeExistsException
   * @throws KeeperException
   * @throws org.apache.hadoop.hbase.exceptions.DeserializationException
   */
  @Test
  public void testCloseRegion()
      throws IOException, NodeExistsException, KeeperException, DeserializationException {
    final Server server = new MockServer(HTU);
    final RegionServerServices rss = HTU.createMockRegionServerService();

    HTableDescriptor htd = TEST_HTD;
    HRegionInfo hri = TEST_HRI;

    // open a region first so that it can be closed later
    OpenRegion(server, rss, htd, hri);

    // close the region
    // Create it CLOSING, which is what Master set before sending CLOSE RPC
    int versionOfClosingNode =
        ZKAssign.createNodeClosing(server.getZooKeeper(), hri, server.getServerName());

    // The CloseRegionHandler will validate the expected version
    // Given it is set to correct versionOfClosingNode,
    // CloseRegionHandlerit should be RS_ZK_REGION_CLOSED
    CloseRegionHandler handler =
        new CloseRegionHandler(server, rss, hri, false, true, versionOfClosingNode);
    handler.process();
    // Handler should have transitioned it to RS_ZK_REGION_CLOSED
    RegionTransition rt =
        RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(), hri.getEncodedName()));
    assertTrue(rt.getEventType().equals(EventType.RS_ZK_REGION_CLOSED));
  }
Example #8
0
  /*
   * Should we check all assignments or just not in RIT?
   */
  public static void waitUntilAssigned(HBaseAdmin admin, HRegionInfo region)
      throws IOException, InterruptedException {
    long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
    long expiration = timeout + System.currentTimeMillis();
    while (System.currentTimeMillis() < expiration) {
      try {
        Map<String, RegionState> rits = admin.getClusterStatus().getRegionsInTransition();

        if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
          // yay! no longer RIT
          return;
        }
        // still in rit
        LOG.info("Region still in transition, waiting for " + "it to become assigned: " + region);
      } catch (IOException e) {
        LOG.warn("Exception when waiting for region to become assigned," + " retrying", e);
      }
      Thread.sleep(1000);
    }
    throw new IOException(
        "Region "
            + region
            + " failed to move out of "
            + "transition within timeout "
            + timeout
            + "ms");
  }
Example #9
0
  /**
   * Creates a new unassigned node in the CLOSING state for the specified region.
   *
   * <p>Does not transition nodes from any states. If a node already exists for this region, a
   * {@link NodeExistsException} will be thrown.
   *
   * <p>If creation is successful, returns the version number of the CLOSING node created.
   *
   * <p>Does not set any watches.
   *
   * <p>This method should only be used by a RegionServer when initiating a close of a region after
   * receiving a CLOSE RPC from the Master.
   *
   * @param zkw zk reference
   * @param region region to be created as closing
   * @param serverName server event originates from
   * @return version of node after transition, -1 if unsuccessful transition
   * @throws KeeperException if unexpected zookeeper exception
   * @throws KeeperException.NodeExistsException if node already exists
   */
  public static int createNodeClosing(ZooKeeperWatcher zkw, HRegionInfo region, String serverName)
      throws KeeperException, KeeperException.NodeExistsException {
    LOG.debug(
        zkw.prefix(
            "Creating unassigned node for " + region.getEncodedName() + " in a CLOSING state"));

    RegionTransitionData data =
        new RegionTransitionData(
            EventType.RS_ZK_REGION_CLOSING, region.getRegionName(), serverName);

    synchronized (zkw.getNodes()) {
      String node = getNodeName(zkw, region.getEncodedName());
      zkw.getNodes().add(node);
      return ZKUtil.createAndWatch(zkw, node, data.getBytes());
    }
  }
Example #10
0
 @Override
 public void postMove(
     ObserverContext<MasterCoprocessorEnvironment> ctx,
     HRegionInfo region,
     ServerName srcServer,
     ServerName destServer)
     throws IOException {
   if (balancer != null && balancer.isTableColocated(region.getTable())) {
     AssignmentManager am = ctx.getEnvironment().getMasterServices().getAssignmentManager();
     RegionStates regionStates = am.getRegionStates();
     String tableName = region.getTable().getNameAsString();
     String correspondingTable =
         region.getTable().getNameAsString().startsWith(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX)
             ? MetaDataUtil.getUserTableName(tableName)
             : MetaDataUtil.getLocalIndexTableName(tableName);
     List<HRegionInfo> regions =
         regionStates.getRegionsOfTable(TableName.valueOf(correspondingTable));
     for (HRegionInfo hri : regions) {
       if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) == 0 && destServer != null) {
         balancer.regionOnline(hri, destServer);
         am.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, destServer));
         am.unassign(hri);
       }
     }
   }
   super.postMove(ctx, region, srcServer, destServer);
 }
Example #11
0
 /**
  * A region is offline, won't be in transition any more. Its state should be the specified
  * expected state, which can only be Split/Merged/Offline/null(=Offline)/SplittingNew/MergingNew.
  */
 public void regionOffline(final HRegionInfo hri, final State expectedState) {
   Preconditions.checkArgument(
       expectedState == null || RegionState.isUnassignable(expectedState),
       "Offlined region should not be " + expectedState);
   if (isRegionInState(hri, State.SPLITTING_NEW, State.MERGING_NEW)) {
     // Remove it from all region maps
     deleteRegion(hri);
     return;
   }
   State newState = expectedState == null ? State.OFFLINE : expectedState;
   updateRegionState(hri, newState);
   String encodedName = hri.getEncodedName();
   synchronized (this) {
     regionsInTransition.remove(encodedName);
     ServerName oldServerName = regionAssignments.remove(hri);
     if (oldServerName != null && serverHoldings.containsKey(oldServerName)) {
       if (newState == State.MERGED
           || newState == State.SPLIT
           || hri.isMetaRegion()
           || tableStateManager.isTableState(
               hri.getTable(), TableState.State.DISABLED, TableState.State.DISABLING)) {
         // Offline the region only if it's merged/split, or the table is disabled/disabling.
         // Otherwise, offline it from this server only when it is online on a different server.
         LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
         removeFromServerHoldings(oldServerName, hri);
         removeFromReplicaMapping(hri);
       } else {
         // Need to remember it so that we can offline it from this
         // server when it is online on a different server.
         oldAssignments.put(encodedName, oldServerName);
       }
     }
   }
 }
Example #12
0
  /**
   * A region is online, won't be in transition any more. We can't confirm it is really online on
   * specified region server because it hasn't been put in region server's online region list yet.
   */
  public void regionOnline(final HRegionInfo hri, final ServerName serverName, long openSeqNum) {
    String encodedName = hri.getEncodedName();
    if (!serverManager.isServerOnline(serverName)) {
      // This is possible if the region server dies before master gets a
      // chance to handle ZK event in time. At this time, if the dead server
      // is already processed by SSH, we should ignore this event.
      // If not processed yet, ignore and let SSH deal with it.
      LOG.warn("Ignored, " + encodedName + " was opened on a dead server: " + serverName);
      return;
    }
    updateRegionState(hri, State.OPEN, serverName, openSeqNum);

    synchronized (this) {
      regionsInTransition.remove(encodedName);
      ServerName oldServerName = regionAssignments.put(hri, serverName);
      if (!serverName.equals(oldServerName)) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Onlined " + hri.getShortNameToLog() + " on " + serverName);
        }
        addToServerHoldings(serverName, hri);
        addToReplicaMapping(hri);
        if (oldServerName == null) {
          oldServerName = oldAssignments.remove(encodedName);
        }
        if (oldServerName != null
            && !oldServerName.equals(serverName)
            && serverHoldings.containsKey(oldServerName)) {
          LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
          removeFromServerHoldings(oldServerName, hri);
        }
      }
    }
  }
  /**
   * Restore the on-disk table to a specified snapshot state.
   *
   * @return the set of regions touched by the restore operation
   */
  public RestoreMetaChanges restoreHdfsRegions() throws IOException {
    LOG.debug("starting restore");
    Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
    if (snapshotRegionNames == null) {
      LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
      return null;
    }

    RestoreMetaChanges metaChanges = new RestoreMetaChanges();

    // Identify which region are still available and which not.
    // NOTE: we rely upon the region name as: "table name, start key, end key"
    List<HRegionInfo> tableRegions = getTableRegions();
    if (tableRegions != null) {
      monitor.rethrowException();
      for (HRegionInfo regionInfo : tableRegions) {
        String regionName = regionInfo.getEncodedName();
        if (snapshotRegionNames.contains(regionName)) {
          LOG.info("region to restore: " + regionName);
          snapshotRegionNames.remove(regionName);
          metaChanges.addRegionToRestore(regionInfo);
        } else {
          LOG.info("region to remove: " + regionName);
          metaChanges.addRegionToRemove(regionInfo);
        }
      }

      // Restore regions using the snapshot data
      monitor.rethrowException();
      restoreHdfsRegions(metaChanges.getRegionsToRestore());

      // Remove regions from the current table
      monitor.rethrowException();
      removeHdfsRegions(metaChanges.getRegionsToRemove());
    }

    // Regions to Add: present in the snapshot but not in the current table
    if (snapshotRegionNames.size() > 0) {
      List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();

      monitor.rethrowException();
      for (String regionName : snapshotRegionNames) {
        LOG.info("region to add: " + regionName);
        Path regionDir = new Path(snapshotDir, regionName);
        regionsToAdd.add(HRegion.loadDotRegionInfoFileContent(fs, regionDir));
      }

      // Create new regions cloning from the snapshot
      monitor.rethrowException();
      HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
      metaChanges.setNewRegions(clonedRegions);
    }

    // Restore WALs
    monitor.rethrowException();
    restoreWALs();

    return metaChanges;
  }
 public static void closeRegion(HBaseTestingUtility HTU, HRegionServer rs, HRegionInfo hri)
     throws Exception {
   AdminProtos.CloseRegionRequest crr =
       RequestConverter.buildCloseRegionRequest(rs.getServerName(), hri.getEncodedName());
   AdminProtos.CloseRegionResponse responseClose = rs.rpcServices.closeRegion(null, crr);
   Assert.assertTrue(responseClose.getClosed());
   checkRegionIsClosed(HTU, rs, hri);
 }
Example #15
0
  private static Map<String, HRegionInfo> getRegionsByEncodedName(Set<HRegionInfo> regions) {
    Map<String, HRegionInfo> regionsByEncodedName = Maps.newHashMapWithExpectedSize(regions.size());
    for (HRegionInfo region : regions) {
      regionsByEncodedName.put(region.getEncodedName(), region);
    }

    return regionsByEncodedName;
  }
Example #16
0
 /** Returns the Hdfs size of the given region in bytes. */
 public long getHdfsSize(HRegionInfo info) throws IOException {
   Path tableDir =
       HTableDescriptor.getTableDir(
           FSUtils.getRootDir(hbaseConf_), Bytes.toBytes(hbaseTableName_));
   FileSystem fs = tableDir.getFileSystem(hbaseConf_);
   Path regionDir = tableDir.suffix("/" + info.getEncodedName());
   return fs.getContentSummary(regionDir).getLength();
 }
 private void OpenRegion(
     Server server, RegionServerServices rss, HTableDescriptor htd, HRegionInfo hri)
     throws IOException, NodeExistsException, KeeperException, DeserializationException {
   // Create it OFFLINE node, which is what Master set before sending OPEN RPC
   ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName());
   OpenRegionHandler openHandler = new OpenRegionHandler(server, rss, hri, htd);
   rss.getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE);
   openHandler.process();
   // This parse is not used?
   RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(), hri.getEncodedName()));
   // delete the node, which is what Master do after the region is opened
   ZKAssign.deleteNode(
       server.getZooKeeper(),
       hri.getEncodedName(),
       EventType.RS_ZK_REGION_OPENED,
       server.getServerName());
 }
Example #18
0
  @Override
  public void process() throws IOException {
    boolean transitionToFailedOpen = false;
    boolean openSuccessful = false;
    try {
      final String name = regionInfo.getRegionNameAsString();
      if (this.server.isStopped() || this.rsServices.isStopping()) {
        return;
      }
      final String encodedName = regionInfo.getEncodedName();

      // Check that this region is not already online
      HRegion region = this.rsServices.getFromOnlineRegions(encodedName);

      // Open region.  After a successful open, failures in subsequent
      // processing needs to do a close as part of cleanup.
      region = openRegion();
      if (region == null) {
        tryTransitionToFailedOpen(regionInfo);
        transitionToFailedOpen = true;
        return;
      }
      boolean failed = true;
      if (tickleOpening("post_region_open")) {
        if (updateMeta(region)) {
          failed = false;
        }
      }
      if (failed || this.server.isStopped() || this.rsServices.isStopping()) {
        cleanupFailedOpen(region);
        tryTransitionToFailedOpen(regionInfo);
        transitionToFailedOpen = true;
        return;
      }

      if (!transitionToOpened(region)) {
        // If we fail to transition to opened, it's because of one of two cases:
        //    (a) we lost our ZK lease
        // OR (b) someone else opened the region before us
        // In either case, we don't need to transition to FAILED_OPEN state.
        // In case (a), the Master will process us as a dead server. In case
        // (b) the region is already being handled elsewhere anyway.
        cleanupFailedOpen(region);
        transitionToFailedOpen = true;
        return;
      }
      // Successful region open, and add it to OnlineRegions
      this.rsServices.addToOnlineRegions(region);
      openSuccessful = true;
      // Done!  Successful region open
      LOG.debug("Opened " + name + " on server:" + this.server.getServerName());
    } finally {
      this.rsServices.removeFromRegionsInTransition(this.regionInfo);
      if (!openSuccessful && !transitionToFailedOpen) {
        tryTransitionToFailedOpen(regionInfo);
      }
    }
  }
 public void printLocalityAndDispersionForCurrentPlan(
     Map<String, Map<String, Float>> regionLocalityMap) throws IOException {
   SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot();
   FavoredNodesPlan assignmentPlan = snapshot.getExistingAssignmentPlan();
   Set<TableName> tables = snapshot.getTableSet();
   Map<TableName, List<HRegionInfo>> tableToRegionsMap = snapshot.getTableToRegionMap();
   for (TableName table : tables) {
     float[] locality = new float[3];
     if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
       continue;
     }
     List<HRegionInfo> regions = tableToRegionsMap.get(table);
     for (HRegionInfo currentRegion : regions) {
       Map<String, Float> regionLocality = regionLocalityMap.get(currentRegion.getEncodedName());
       if (regionLocality == null) {
         continue;
       }
       List<ServerName> servers = assignmentPlan.getFavoredNodes(currentRegion);
       if (servers != null) {
         int i = 0;
         for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) {
           ServerName server = servers.get(p.ordinal());
           Float currentLocality = 0f;
           if (servers != null) {
             currentLocality = regionLocality.get(server.getHostname());
             if (currentLocality == null) {
               currentLocality = 0f;
             }
             locality[i] += currentLocality;
           }
           i++;
         }
       }
     }
     for (int i = 0; i < locality.length; i++) {
       String copy = null;
       if (i == 0) {
         copy = "primary";
       } else if (i == 1) {
         copy = "secondary";
       } else if (i == 2) {
         copy = "tertiary";
       }
       float avgLocality = 100 * locality[i] / regions.size();
       LOG.info(
           "For Table: "
               + table
               + " ; #Total Regions: "
               + regions.size()
               + " ; The average locality for "
               + copy
               + " is "
               + avgLocality
               + " %");
     }
     printDispersionScores(table, snapshot, regions.size(), null, false);
   }
 }
Example #20
0
 /**
  * Write out a merge reference under the given merges directory. Package local so it doesnt leak
  * out of regionserver.
  *
  * @param mergedRegion {@link HRegionInfo} of the merged region
  * @param familyName Column Family Name
  * @param f File to create reference.
  * @param mergedDir
  * @return Path to created reference.
  * @throws IOException
  */
 Path mergeStoreFile(
     final HRegionInfo mergedRegion,
     final String familyName,
     final StoreFile f,
     final Path mergedDir)
     throws IOException {
   Path referenceDir = new Path(new Path(mergedDir, mergedRegion.getEncodedName()), familyName);
   // A whole reference to the store file.
   Reference r = Reference.createTopReference(regionInfo.getStartKey());
   // Add the referred-to regions name as a dot separated suffix.
   // See REF_NAME_REGEX regex above. The referred-to regions name is
   // up in the path of the passed in <code>f</code> -- parentdir is family,
   // then the directory above is the region name.
   String mergingRegionName = regionInfo.getEncodedName();
   // Write reference with same file id only with the other region name as
   // suffix and into the new region location (under same family).
   Path p = new Path(referenceDir, f.getPath().getName() + "." + mergingRegionName);
   return r.write(fs, p);
 }
Example #21
0
 /**
  * Commit a merged region, moving it from the merges temporary directory to the proper location in
  * the filesystem.
  *
  * @param mergedRegionInfo merged region {@link HRegionInfo}
  * @throws IOException
  */
 void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
   Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName());
   Path mergedRegionTmpDir = this.getMergesDir(mergedRegionInfo);
   // Move the tmp dir in the expected location
   if (mergedRegionTmpDir != null && fs.exists(mergedRegionTmpDir)) {
     if (!fs.rename(mergedRegionTmpDir, regionDir)) {
       throw new IOException("Unable to rename " + mergedRegionTmpDir + " to " + regionDir);
     }
   }
 }
Example #22
0
 /**
  * Creates an unassigned node in the OFFLINE state for the specified region.
  *
  * <p>Runs asynchronously. Depends on no pre-existing znode.
  *
  * <p>Sets a watcher on the unassigned region node.
  *
  * @param zkw zk reference
  * @param region region to be created as offline
  * @param serverName server event originates from
  * @param cb
  * @param ctx
  * @throws KeeperException if unexpected zookeeper exception
  * @throws KeeperException.NodeExistsException if node already exists
  */
 public static void asyncCreateNodeOffline(
     ZooKeeperWatcher zkw,
     HRegionInfo region,
     String serverName,
     final AsyncCallback.StringCallback cb,
     final Object ctx)
     throws KeeperException {
   LOG.debug(
       zkw.prefix(
           "Async create of unassigned node for "
               + region.getEncodedName()
               + " with OFFLINE state"));
   RegionTransitionData data =
       new RegionTransitionData(EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName);
   synchronized (zkw.getNodes()) {
     String node = getNodeName(zkw, region.getEncodedName());
     zkw.getNodes().add(node);
     ZKUtil.asyncCreate(zkw, node, data.getBytes(), cb, ctx);
   }
 }
 /**
  * Flush the given region in the mini cluster. Since no master, we cannot use HBaseAdmin.flush()
  */
 public static void flushRegion(HBaseTestingUtility HTU, HRegionInfo regionInfo)
     throws IOException {
   for (RegionServerThread rst : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
     HRegion region = rst.getRegionServer().getRegionByEncodedName(regionInfo.getEncodedName());
     if (region != null) {
       region.flushcache();
       return;
     }
   }
   throw new IOException("Region to flush cannot be found");
 }
Example #24
0
 private RegionState putRegionState(RegionState regionState) {
   HRegionInfo hri = regionState.getRegion();
   String encodedName = hri.getEncodedName();
   TableName table = hri.getTable();
   RegionState oldState = regionStates.put(encodedName, regionState);
   Map<String, RegionState> map = regionStatesTableIndex.get(table);
   if (map == null) {
     map = new HashMap<String, RegionState>();
     regionStatesTableIndex.put(table, map);
   }
   map.put(encodedName, regionState);
   return oldState;
 }
Example #25
0
 /**
  * Costruct a copy of another HRegionInfo
  *
  * @param other
  */
 public HRegionInfo(HRegionInfo other) {
   super();
   this.endKey = other.getEndKey();
   this.offLine = other.isOffline();
   this.regionId = other.getRegionId();
   this.regionName = other.getRegionName();
   this.split = other.isSplit();
   this.startKey = other.getStartKey();
   this.hashCode = other.hashCode();
   this.encodedName = other.getEncodedName();
   this.tableName = other.tableName;
   this.replicaId = other.replicaId;
 }
Example #26
0
  /**
   * Remove the region family from disk, archiving the store files.
   *
   * @param familyName Column Family Name
   * @throws IOException if an error occours during the archiving
   */
  public void deleteFamily(final String familyName) throws IOException {
    // archive family store files
    HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, Bytes.toBytes(familyName));

    // delete the family folder
    Path familyDir = getStoreDir(familyName);
    if (fs.exists(familyDir) && !deleteDir(familyDir))
      throw new IOException(
          "Could not delete family "
              + familyName
              + " from FileSystem for region "
              + regionInfo.getRegionNameAsString()
              + "("
              + regionInfo.getEncodedName()
              + ")");
  }
Example #27
0
 /** Remove a region from all state maps. */
 @VisibleForTesting
 public synchronized void deleteRegion(final HRegionInfo hri) {
   String encodedName = hri.getEncodedName();
   regionsInTransition.remove(encodedName);
   regionStates.remove(encodedName);
   TableName table = hri.getTable();
   Map<String, RegionState> indexMap = regionStatesTableIndex.get(table);
   indexMap.remove(encodedName);
   if (indexMap.size() == 0) regionStatesTableIndex.remove(table);
   lastAssignments.remove(encodedName);
   ServerName sn = regionAssignments.remove(hri);
   if (sn != null) {
     Set<HRegionInfo> regions = serverHoldings.get(sn);
     regions.remove(hri);
   }
 }
Example #28
0
  /**
   * Commit a daughter region, moving it from the split temporary directory to the proper location
   * in the filesystem.
   *
   * @param regionInfo daughter {@link HRegionInfo}
   * @throws IOException
   */
  Path commitDaughterRegion(final HRegionInfo regionInfo) throws IOException {
    Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName());
    Path daughterTmpDir = this.getSplitsDir(regionInfo);
    if (fs.exists(daughterTmpDir)) {
      // Write HRI to a file in case we need to recover hbase:meta
      Path regionInfoFile = new Path(daughterTmpDir, REGION_INFO_FILE);
      byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
      writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);

      // Move the daughter temp dir to the table dir
      if (!rename(daughterTmpDir, regionDir)) {
        throw new IOException("Unable to rename " + daughterTmpDir + " to " + regionDir);
      }
    }
    return regionDir;
  }
 /**
  * Clone region directory content from the snapshot info.
  *
  * <p>Each region is encoded with the table name, so the cloned region will have a different
  * region name.
  *
  * <p>Instead of copying the hfiles a HFileLink is created.
  *
  * @param region {@link HRegion} cloned
  * @param snapshotRegionInfo
  */
 private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
     throws IOException {
   final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
   final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
   final String tableName = tableDesc.getNameAsString();
   SnapshotReferenceUtil.visitRegionStoreFiles(
       fs,
       snapshotRegionDir,
       new FSVisitor.StoreFileVisitor() {
         public void storeFile(final String region, final String family, final String hfile)
             throws IOException {
           LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
           Path familyDir = new Path(regionDir, family);
           restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
         }
       });
 }
Example #30
0
 /**
  * Add a region to RegionStates with the specified state. If the region is already in
  * RegionStates, this call has no effect, and the original state is returned.
  *
  * @param hri the region info to create a state for
  * @param newState the state to the region in set to
  * @param serverName the server the region is transitioning on
  * @param lastHost the last server that hosts the region
  * @return the current state
  */
 public synchronized RegionState createRegionState(
     final HRegionInfo hri, State newState, ServerName serverName, ServerName lastHost) {
   if (newState == null || (newState == State.OPEN && serverName == null)) {
     newState = State.OFFLINE;
   }
   if (hri.isOffline() && hri.isSplit()) {
     newState = State.SPLIT;
     serverName = null;
   }
   String encodedName = hri.getEncodedName();
   RegionState regionState = regionStates.get(encodedName);
   if (regionState != null) {
     LOG.warn(
         "Tried to create a state for a region already in RegionStates, "
             + "used existing: "
             + regionState
             + ", ignored new: "
             + newState);
   } else {
     regionState = new RegionState(hri, newState, serverName);
     putRegionState(regionState);
     if (newState == State.OPEN) {
       if (!serverName.equals(lastHost)) {
         LOG.warn(
             "Open region's last host "
                 + lastHost
                 + " should be the same as the current one "
                 + serverName
                 + ", ignored the last and used the current one");
         lastHost = serverName;
       }
       lastAssignments.put(encodedName, lastHost);
       regionAssignments.put(hri, lastHost);
     } else if (!isOneOfStates(regionState, State.MERGED, State.SPLIT, State.OFFLINE)) {
       regionsInTransition.put(encodedName, regionState);
     }
     if (lastHost != null && newState != State.SPLIT) {
       addToServerHoldings(lastHost, hri);
       if (newState != State.OPEN) {
         oldAssignments.put(encodedName, lastHost);
       }
     }
   }
   return regionState;
 }