Esempio n. 1
1
 /**
  * Parses an HRegionInfo instance from the passed in stream. Presumes the HRegionInfo was
  * serialized to the stream with {@link #toDelimitedByteArray()}
  *
  * @param in
  * @return An instance of HRegionInfo.
  * @throws IOException
  */
 public static HRegionInfo parseFrom(final DataInputStream in) throws IOException {
   // I need to be able to move back in the stream if this is not a pb serialization so I can
   // do the Writable decoding instead.
   int pblen = ProtobufUtil.lengthOfPBMagic();
   byte[] pbuf = new byte[pblen];
   if (in.markSupported()) { // read it with mark()
     in.mark(pblen);
   }
   int read =
       in.read(pbuf); // assumption: if Writable serialization, it should be longer than pblen.
   if (read != pblen) throw new IOException("read=" + read + ", wanted=" + pblen);
   if (ProtobufUtil.isPBMagicPrefix(pbuf)) {
     return convert(HBaseProtos.RegionInfo.parseDelimitedFrom(in));
   } else {
     // Presume Writables.  Need to reset the stream since it didn't start w/ pb.
     if (in.markSupported()) {
       in.reset();
       HRegionInfo hri = new HRegionInfo();
       hri.readFields(in);
       return hri;
     } else {
       // we cannot use BufferedInputStream, it consumes more than we read from the underlying IS
       ByteArrayInputStream bais = new ByteArrayInputStream(pbuf);
       SequenceInputStream sis = new SequenceInputStream(bais, in); // concatenate input streams
       HRegionInfo hri = new HRegionInfo();
       hri.readFields(new DataInputStream(sis));
       return hri;
     }
   }
 }
Esempio n. 2
0
 /**
  * A region is offline, won't be in transition any more. Its state should be the specified
  * expected state, which can only be Split/Merged/Offline/null(=Offline)/SplittingNew/MergingNew.
  */
 public void regionOffline(final HRegionInfo hri, final State expectedState) {
   Preconditions.checkArgument(
       expectedState == null || RegionState.isUnassignable(expectedState),
       "Offlined region should not be " + expectedState);
   if (isRegionInState(hri, State.SPLITTING_NEW, State.MERGING_NEW)) {
     // Remove it from all region maps
     deleteRegion(hri);
     return;
   }
   State newState = expectedState == null ? State.OFFLINE : expectedState;
   updateRegionState(hri, newState);
   String encodedName = hri.getEncodedName();
   synchronized (this) {
     regionsInTransition.remove(encodedName);
     ServerName oldServerName = regionAssignments.remove(hri);
     if (oldServerName != null && serverHoldings.containsKey(oldServerName)) {
       if (newState == State.MERGED
           || newState == State.SPLIT
           || hri.isMetaRegion()
           || tableStateManager.isTableState(
               hri.getTable(), TableState.State.DISABLED, TableState.State.DISABLING)) {
         // Offline the region only if it's merged/split, or the table is disabled/disabling.
         // Otherwise, offline it from this server only when it is online on a different server.
         LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
         removeFromServerHoldings(oldServerName, hri);
         removeFromReplicaMapping(hri);
       } else {
         // Need to remember it so that we can offline it from this
         // server when it is online on a different server.
         oldAssignments.put(encodedName, oldServerName);
       }
     }
   }
 }
Esempio n. 3
0
  /**
   * A region is online, won't be in transition any more. We can't confirm it is really online on
   * specified region server because it hasn't been put in region server's online region list yet.
   */
  public void regionOnline(final HRegionInfo hri, final ServerName serverName, long openSeqNum) {
    String encodedName = hri.getEncodedName();
    if (!serverManager.isServerOnline(serverName)) {
      // This is possible if the region server dies before master gets a
      // chance to handle ZK event in time. At this time, if the dead server
      // is already processed by SSH, we should ignore this event.
      // If not processed yet, ignore and let SSH deal with it.
      LOG.warn("Ignored, " + encodedName + " was opened on a dead server: " + serverName);
      return;
    }
    updateRegionState(hri, State.OPEN, serverName, openSeqNum);

    synchronized (this) {
      regionsInTransition.remove(encodedName);
      ServerName oldServerName = regionAssignments.put(hri, serverName);
      if (!serverName.equals(oldServerName)) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Onlined " + hri.getShortNameToLog() + " on " + serverName);
        }
        addToServerHoldings(serverName, hri);
        addToReplicaMapping(hri);
        if (oldServerName == null) {
          oldServerName = oldAssignments.remove(encodedName);
        }
        if (oldServerName != null
            && !oldServerName.equals(serverName)
            && serverHoldings.containsKey(oldServerName)) {
          LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
          removeFromServerHoldings(oldServerName, hri);
        }
      }
    }
  }
Esempio n. 4
0
    @Override
    protected void updateMeta(final byte[] oldRegion1, final byte[] oldRegion2, HRegion newRegion)
        throws IOException {
      byte[][] regionsToDelete = {oldRegion1, oldRegion2};
      for (int r = 0; r < regionsToDelete.length; r++) {
        Delete delete = new Delete(regionsToDelete[r]);
        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
        root.delete(delete, null, true);

        if (LOG.isDebugEnabled()) {
          LOG.debug("updated columns in row: " + Bytes.toStringBinary(regionsToDelete[r]));
        }
      }
      HRegionInfo newInfo = newRegion.getRegionInfo();
      newInfo.setOffline(true);
      Put put = new Put(newRegion.getRegionName());
      put.add(
          HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(newInfo));
      root.put(put);
      if (LOG.isDebugEnabled()) {
        LOG.debug("updated columns in row: " + Bytes.toStringBinary(newRegion.getRegionName()));
      }
    }
Esempio n. 5
0
 protected HashMap<TableName, TreeMap<ServerName, List<HRegionInfo>>> mockClusterServersWithTables(
     Map<ServerName, List<HRegionInfo>> clusterServers) {
   HashMap<TableName, TreeMap<ServerName, List<HRegionInfo>>> result = new HashMap<>();
   for (Map.Entry<ServerName, List<HRegionInfo>> entry : clusterServers.entrySet()) {
     ServerName sal = entry.getKey();
     List<HRegionInfo> regions = entry.getValue();
     for (HRegionInfo hri : regions) {
       TreeMap<ServerName, List<HRegionInfo>> servers = result.get(hri.getTable());
       if (servers == null) {
         servers = new TreeMap<ServerName, List<HRegionInfo>>();
         result.put(hri.getTable(), servers);
       }
       List<HRegionInfo> hrilist = servers.get(sal);
       if (hrilist == null) {
         hrilist = new ArrayList<HRegionInfo>();
         servers.put(sal, hrilist);
       }
       hrilist.add(hri);
     }
   }
   for (Map.Entry<TableName, TreeMap<ServerName, List<HRegionInfo>>> entry : result.entrySet()) {
     for (ServerName srn : clusterServers.keySet()) {
       if (!entry.getValue().containsKey(srn))
         entry.getValue().put(srn, new ArrayList<HRegionInfo>());
     }
   }
   return result;
 }
  /** Test that if we do a close while opening it stops the opening. */
  @Test(timeout = 60000)
  public void testCancelOpeningWithoutZK() throws Exception {
    // We close
    closeRegionNoZK();
    checkRegionIsClosed(HTU, getRS(), hri);

    // Let do the initial steps, without having a handler
    getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE);

    // That's a close without ZK.
    AdminProtos.CloseRegionRequest crr =
        RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName);
    try {
      getRS().rpcServices.closeRegion(null, crr);
      Assert.assertTrue(false);
    } catch (ServiceException expected) {
    }

    // The state in RIT should have changed to close
    Assert.assertEquals(
        Boolean.FALSE, getRS().getRegionsInTransitionInRS().get(hri.getEncodedNameAsBytes()));

    // Let's start the open handler
    HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable());

    getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd));

    // The open handler should have removed the region from RIT but kept the region closed
    checkRegionIsClosed(HTU, getRS(), hri);

    openRegion(HTU, getRS(), hri);
  }
Esempio n. 7
0
 /**
  * Convert a RegionInfo to a HRegionInfo
  *
  * @param proto the RegionInfo to convert
  * @return the converted HRegionInfho
  */
 public static HRegionInfo convert(final RegionInfo proto) {
   if (proto == null) return null;
   TableName tableName = ProtobufUtil.toTableName(proto.getTableName());
   if (tableName.equals(TableName.META_TABLE_NAME)) {
     return FIRST_META_REGIONINFO;
   }
   long regionId = proto.getRegionId();
   int replicaId = proto.hasReplicaId() ? proto.getReplicaId() : DEFAULT_REPLICA_ID;
   byte[] startKey = null;
   byte[] endKey = null;
   if (proto.hasStartKey()) {
     startKey = proto.getStartKey().toByteArray();
   }
   if (proto.hasEndKey()) {
     endKey = proto.getEndKey().toByteArray();
   }
   boolean split = false;
   if (proto.hasSplit()) {
     split = proto.getSplit();
   }
   HRegionInfo hri = new HRegionInfo(tableName, startKey, endKey, split, regionId, replicaId);
   if (proto.hasOffline()) {
     hri.setOffline(proto.getOffline());
   }
   return hri;
 }
  /**
   * Create a new {@link Reference} as copy of the source one.
   *
   * <p>
   *
   * <blockquote>
   *
   * <pre>
   * The source table looks like:
   *    1234/abc      (original file)
   *    5678/abc.1234 (reference file)
   *
   * After the clone operation looks like:
   *   wxyz/table=1234-abc
   *   stuv/table=1234-abc.wxyz
   *
   * NOTE that the region name in the clone changes (md5 of regioninfo)
   * and the reference should reflect that change.
   * </pre>
   *
   * </blockquote>
   *
   * @param familyDir destination directory for the store file
   * @param regionInfo destination region info for the table
   * @param hfileName reference file name
   */
  private void restoreReferenceFile(
      final Path familyDir, final HRegionInfo regionInfo, final String hfileName)
      throws IOException {
    // Extract the referred information (hfile name and parent region)
    String tableName = snapshotDesc.getTable();
    Path refPath =
        StoreFile.getReferredToFile(
            new Path(
                new Path(new Path(tableName, regionInfo.getEncodedName()), familyDir.getName()),
                hfileName));
    String snapshotRegionName = refPath.getParent().getParent().getName();
    String fileName = refPath.getName();

    // The new reference should have the cloned region name as parent, if it is a clone.
    String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
    if (clonedRegionName == null) clonedRegionName = snapshotRegionName;

    // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
    String refLink = fileName;
    if (!HFileLink.isHFileLink(fileName)) {
      refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName);
    }
    Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);

    // Create the new reference
    Path linkPath =
        new Path(
            familyDir,
            HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName));
    InputStream in = new HFileLink(conf, linkPath).open(fs);
    OutputStream out = fs.create(outPath);
    IOUtils.copyBytes(in, out, conf);
  }
Esempio n. 9
0
 private HRegionInfo nextRegion() throws IOException {
   try {
     Result results = getMetaRow();
     if (results == null) {
       return null;
     }
     byte[] regionInfoValue =
         results.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
     if (regionInfoValue == null || regionInfoValue.length == 0) {
       throw new NoSuchElementException(
           "meta region entry missing "
               + Bytes.toString(HConstants.CATALOG_FAMILY)
               + ":"
               + Bytes.toString(HConstants.REGIONINFO_QUALIFIER));
     }
     HRegionInfo region = Writables.getHRegionInfo(regionInfoValue);
     if (!Bytes.equals(region.getTableName(), this.tableName)) {
       return null;
     }
     return region;
   } catch (IOException e) {
     e = RemoteExceptionHandler.checkIOException(e);
     LOG.error("meta scanner error", e);
     metaScanner.close();
     throw e;
   }
 }
Esempio n. 10
0
 /**
  * Creates or force updates an unassigned node to the OFFLINE state for the specified region.
  *
  * <p>Attempts to create the node but if it exists will force it to transition to and OFFLINE
  * state.
  *
  * <p>Sets a watcher on the unassigned region node if the method is successful.
  *
  * <p>This method should be used when assigning a region.
  *
  * @param zkw zk reference
  * @param region region to be created as offline
  * @param serverName server event originates from
  * @throws KeeperException if unexpected zookeeper exception
  * @throws KeeperException.NodeExistsException if node already exists
  */
 public static boolean createOrForceNodeOffline(
     ZooKeeperWatcher zkw, HRegionInfo region, String serverName) throws KeeperException {
   LOG.debug(
       zkw.prefix(
           "Creating (or updating) unassigned node for "
               + region.getEncodedName()
               + " with OFFLINE state"));
   RegionTransitionData data =
       new RegionTransitionData(EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName);
   synchronized (zkw.getNodes()) {
     String node = getNodeName(zkw, region.getEncodedName());
     zkw.sync(node);
     zkw.getNodes().add(node);
     int version = ZKUtil.checkExists(zkw, node);
     if (version == -1) {
       ZKUtil.createAndWatch(zkw, node, data.getBytes());
     } else {
       if (!ZKUtil.setData(zkw, node, data.getBytes(), version)) {
         return false;
       } else {
         // We successfully forced to OFFLINE, reset watch and handle if
         // the state changed in between our set and the watch
         RegionTransitionData curData = ZKAssign.getData(zkw, region.getEncodedName());
         if (curData.getEventType() != data.getEventType()) {
           // state changed, need to process
           return false;
         }
       }
     }
   }
   return true;
 }
Esempio n. 11
0
 static ScanResponse doMetaScanResponse(
     final SortedMap<byte[], Pair<HRegionInfo, ServerName>> meta,
     final AtomicLong sequenceids,
     final ScanRequest request) {
   ScanResponse.Builder builder = ScanResponse.newBuilder();
   int max = request.getNumberOfRows();
   int count = 0;
   Map<byte[], Pair<HRegionInfo, ServerName>> tail =
       request.hasScan() ? meta.tailMap(request.getScan().getStartRow().toByteArray()) : meta;
   ClientProtos.Result.Builder resultBuilder = ClientProtos.Result.newBuilder();
   for (Map.Entry<byte[], Pair<HRegionInfo, ServerName>> e : tail.entrySet()) {
     // Can be 0 on open of a scanner -- i.e. rpc to setup scannerid only.
     if (max <= 0) break;
     if (++count > max) break;
     HRegionInfo hri = e.getValue().getFirst();
     ByteString row = HBaseZeroCopyByteString.wrap(hri.getRegionName());
     resultBuilder.clear();
     resultBuilder.addCell(getRegionInfo(row, hri));
     resultBuilder.addCell(getServer(row, e.getValue().getSecond()));
     resultBuilder.addCell(getStartCode(row));
     builder.addResults(resultBuilder.build());
     // Set more to false if we are on the last region in table.
     if (hri.getEndKey().length <= 0) builder.setMoreResults(false);
     else builder.setMoreResults(true);
   }
   // If no scannerid, set one.
   builder.setScannerId(
       request.hasScannerId() ? request.getScannerId() : sequenceids.incrementAndGet());
   return builder.build();
 }
Esempio n. 12
0
  @Override
  public boolean prepare() throws IOException {
    if (!this.parent.isSplittable()) return false;
    // Split key can be null if this region is unsplittable; i.e. has refs.
    if (this.splitrow == null) return false;
    HRegionInfo hri = this.parent.getRegionInfo();
    parent.prepareToSplit();
    // Check splitrow.
    byte[] startKey = hri.getStartKey();
    byte[] endKey = hri.getEndKey();
    if (Bytes.equals(startKey, splitrow) || !this.parent.getRegionInfo().containsRow(splitrow)) {
      LOG.info(
          "Split row is not inside region key range or is equal to "
              + "startkey: "
              + Bytes.toStringBinary(this.splitrow));
      return false;
    }
    long rid = getDaughterRegionIdTimestamp(hri);
    this.hri_a = new HRegionInfo(hri.getTable(), startKey, this.splitrow, false, rid);
    this.hri_b = new HRegionInfo(hri.getTable(), this.splitrow, endKey, false, rid);

    transition(SplitTransactionPhase.PREPARED);

    return true;
  }
  public void prepareMutationsForMerge(
      HRegionInfo mergedRegion,
      HRegionInfo regionA,
      HRegionInfo regionB,
      ServerName serverName,
      List<Mutation> mutations,
      int regionReplication)
      throws IOException {
    HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion);

    // Put for parent
    Put putOfMerged = MetaTableAccessor.makePutFromRegionInfo(copyOfMerged);
    putOfMerged.add(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER, regionA.toByteArray());
    putOfMerged.add(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER, regionB.toByteArray());
    mutations.add(putOfMerged);
    // Deletes for merging regions
    Delete deleteA = MetaTableAccessor.makeDeleteFromRegionInfo(regionA);
    Delete deleteB = MetaTableAccessor.makeDeleteFromRegionInfo(regionB);
    mutations.add(deleteA);
    mutations.add(deleteB);

    // Add empty locations for region replicas of the merged region so that number of replicas can
    // be cached whenever the primary region is looked up from meta
    for (int i = 1; i < regionReplication; i++) {
      addEmptyLocation(putOfMerged, i);
    }

    // The merged is a new region, openSeqNum = 1 is fine.
    addLocation(putOfMerged, serverName, 1);
  }
Esempio n. 14
0
    @Override
    public void run() {
      byte[] key = new byte[keySize];
      byte[] value = new byte[valueSize];
      Random rand = new Random(Thread.currentThread().getId());
      HLog hlog = region.getLog();

      try {
        long startTime = System.currentTimeMillis();
        for (int i = 0; i < numIterations; ++i) {
          Put put = setupPut(rand, key, value, numFamilies);
          long now = System.currentTimeMillis();
          WALEdit walEdit = new WALEdit();
          addFamilyMapToWALEdit(put.getFamilyCellMap(), walEdit);
          HRegionInfo hri = region.getRegionInfo();
          if (this.noSync) {
            hlog.appendNoSync(hri, hri.getTable(), walEdit, new ArrayList<UUID>(), now, htd);
          } else {
            hlog.append(hri, hri.getTable(), walEdit, now, htd);
          }
        }
        long totalTime = (System.currentTimeMillis() - startTime);
        logBenchmarkResult(Thread.currentThread().getName(), numIterations, totalTime);
      } catch (Exception e) {
        LOG.error(getClass().getSimpleName() + " Thread failed", e);
      }
    }
Esempio n. 15
0
 /**
  * @param r Region we're working on.
  * @return whether znode is successfully transitioned to OPENED state.
  * @throws IOException
  */
 private boolean transitionToOpened(final HRegion r) throws IOException {
   boolean result = false;
   HRegionInfo hri = r.getRegionInfo();
   final String name = hri.getRegionNameAsString();
   // Finally, Transition ZK node to OPENED
   try {
     if (ZKAssign.transitionNodeOpened(
             this.server.getZooKeeper(), hri, this.server.getServerName(), this.version)
         == -1) {
       LOG.warn(
           "Completed the OPEN of region "
               + name
               + " but when transitioning from "
               + " OPENING to OPENED got a version mismatch, someone else clashed "
               + "so now unassigning -- closing region on server: "
               + this.server.getServerName());
     } else {
       LOG.debug(
           "region transitioned to opened in zookeeper: "
               + r.getRegionInfo()
               + ", server: "
               + this.server.getServerName());
       result = true;
     }
   } catch (KeeperException e) {
     LOG.error(
         "Failed transitioning node " + name + " from OPENING to OPENED -- closing region", e);
   }
   return result;
 }
  /**
   * Test if the region can be closed properly
   *
   * @throws IOException
   * @throws NodeExistsException
   * @throws KeeperException
   * @throws org.apache.hadoop.hbase.exceptions.DeserializationException
   */
  @Test
  public void testCloseRegion()
      throws IOException, NodeExistsException, KeeperException, DeserializationException {
    final Server server = new MockServer(HTU);
    final RegionServerServices rss = HTU.createMockRegionServerService();

    HTableDescriptor htd = TEST_HTD;
    HRegionInfo hri = TEST_HRI;

    // open a region first so that it can be closed later
    OpenRegion(server, rss, htd, hri);

    // close the region
    // Create it CLOSING, which is what Master set before sending CLOSE RPC
    int versionOfClosingNode =
        ZKAssign.createNodeClosing(server.getZooKeeper(), hri, server.getServerName());

    // The CloseRegionHandler will validate the expected version
    // Given it is set to correct versionOfClosingNode,
    // CloseRegionHandlerit should be RS_ZK_REGION_CLOSED
    CloseRegionHandler handler =
        new CloseRegionHandler(server, rss, hri, false, true, versionOfClosingNode);
    handler.process();
    // Handler should have transitioned it to RS_ZK_REGION_CLOSED
    RegionTransition rt =
        RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(), hri.getEncodedName()));
    assertTrue(rt.getEventType().equals(EventType.RS_ZK_REGION_CLOSED));
  }
Esempio n. 17
0
 protected void waitRegionInTransition(final List<HRegionInfo> regions)
     throws IOException, CoordinatedStateException {
   AssignmentManager am = this.masterServices.getAssignmentManager();
   RegionStates states = am.getRegionStates();
   long waitTime = server.getConfiguration().getLong("hbase.master.wait.on.region", 5 * 60 * 1000);
   for (HRegionInfo region : regions) {
     long done = System.currentTimeMillis() + waitTime;
     while (System.currentTimeMillis() < done) {
       if (states.isRegionInState(region, State.FAILED_OPEN)) {
         am.regionOffline(region);
       }
       if (!states.isRegionInTransition(region)) break;
       try {
         Thread.sleep(waitingTimeForEvents);
       } catch (InterruptedException e) {
         LOG.warn("Interrupted while sleeping");
         throw (InterruptedIOException) new InterruptedIOException().initCause(e);
       }
       LOG.debug(
           "Waiting on region to clear regions in transition; "
               + am.getRegionStates().getRegionTransitionState(region));
     }
     if (states.isRegionInTransition(region)) {
       throw new IOException(
           "Waited hbase.master.wait.on.region ("
               + waitTime
               + "ms) for region to leave region "
               + region.getRegionNameAsString()
               + " in transitions");
     }
   }
 }
Esempio n. 18
0
    @Override
    public Void call() throws IOException {
      IOException toThrow = null;
      StatisticsCollectionRunTracker collectionTracker =
          StatisticsCollectionRunTracker.getInstance(config);
      final HRegionInfo regionInfo = region.getRegionInfo();
      try {
        // update the statistics table
        // Just verify if this if fine
        ArrayList<Mutation> mutations = new ArrayList<Mutation>();

        if (LOG.isDebugEnabled()) {
          LOG.debug(
              "Deleting the stats for the region "
                  + regionInfo.getRegionNameAsString()
                  + " as part of major compaction");
        }
        stats.deleteStats(region, tracker, family, mutations);
        if (LOG.isDebugEnabled()) {
          LOG.debug(
              "Adding new stats for the region "
                  + regionInfo.getRegionNameAsString()
                  + " as part of major compaction");
        }
        stats.addStats(tracker, family, mutations);
        if (LOG.isDebugEnabled()) {
          LOG.debug(
              "Committing new stats for the region "
                  + regionInfo.getRegionNameAsString()
                  + " as part of major compaction");
        }
        stats.commitStats(mutations);
      } catch (IOException e) {
        LOG.error("Failed to update statistics table!", e);
        toThrow = e;
      } finally {
        try {
          collectionTracker.removeCompactingRegion(regionInfo);
          stats.close(); // close the writer
          tracker.close(); // close the tracker
        } catch (IOException e) {
          if (toThrow == null) toThrow = e;
          LOG.error("Error while closing the stats table", e);
        } finally {
          // close the delegate scanner
          try {
            delegate.close();
          } catch (IOException e) {
            if (toThrow == null) toThrow = e;
            LOG.error("Error while closing the scanner", e);
          } finally {
            if (toThrow != null) {
              throw toThrow;
            }
          }
        }
      }
      return null;
    }
  /**
   * Restore the on-disk table to a specified snapshot state.
   *
   * @return the set of regions touched by the restore operation
   */
  public RestoreMetaChanges restoreHdfsRegions() throws IOException {
    LOG.debug("starting restore");
    Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
    if (snapshotRegionNames == null) {
      LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
      return null;
    }

    RestoreMetaChanges metaChanges = new RestoreMetaChanges();

    // Identify which region are still available and which not.
    // NOTE: we rely upon the region name as: "table name, start key, end key"
    List<HRegionInfo> tableRegions = getTableRegions();
    if (tableRegions != null) {
      monitor.rethrowException();
      for (HRegionInfo regionInfo : tableRegions) {
        String regionName = regionInfo.getEncodedName();
        if (snapshotRegionNames.contains(regionName)) {
          LOG.info("region to restore: " + regionName);
          snapshotRegionNames.remove(regionName);
          metaChanges.addRegionToRestore(regionInfo);
        } else {
          LOG.info("region to remove: " + regionName);
          metaChanges.addRegionToRemove(regionInfo);
        }
      }

      // Restore regions using the snapshot data
      monitor.rethrowException();
      restoreHdfsRegions(metaChanges.getRegionsToRestore());

      // Remove regions from the current table
      monitor.rethrowException();
      removeHdfsRegions(metaChanges.getRegionsToRemove());
    }

    // Regions to Add: present in the snapshot but not in the current table
    if (snapshotRegionNames.size() > 0) {
      List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();

      monitor.rethrowException();
      for (String regionName : snapshotRegionNames) {
        LOG.info("region to add: " + regionName);
        Path regionDir = new Path(snapshotDir, regionName);
        regionsToAdd.add(HRegion.loadDotRegionInfoFileContent(fs, regionDir));
      }

      // Create new regions cloning from the snapshot
      monitor.rethrowException();
      HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
      metaChanges.setNewRegions(clonedRegions);
    }

    // Restore WALs
    monitor.rethrowException();
    restoreWALs();

    return metaChanges;
  }
Esempio n. 20
0
  private static Map<String, HRegionInfo> getRegionsByEncodedName(Set<HRegionInfo> regions) {
    Map<String, HRegionInfo> regionsByEncodedName = Maps.newHashMapWithExpectedSize(regions.size());
    for (HRegionInfo region : regions) {
      regionsByEncodedName.put(region.getEncodedName(), region);
    }

    return regionsByEncodedName;
  }
 /**
  * Create a new {@link HRegionInfo} from the snapshot region info. Keep the same startKey, endKey,
  * regionId and split information but change the table name.
  *
  * @param snapshotRegionInfo Info for region to clone.
  * @return the new HRegion instance
  */
 public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
   return new HRegionInfo(
       tableDesc.getName(),
       snapshotRegionInfo.getStartKey(),
       snapshotRegionInfo.getEndKey(),
       snapshotRegionInfo.isSplit(),
       snapshotRegionInfo.getRegionId());
 }
Esempio n. 22
0
 /**
  * Updates the location of the specified region to be the specified server.
  *
  * <p>Connects to the specified server which should be hosting the specified catalog region name
  * to perform the edit.
  *
  * @param catalogTracker
  * @param regionInfo region to update location of
  * @param sn Server name
  * @throws IOException In particular could throw {@link java.net.ConnectException} if the server
  *     is down on other end.
  */
 private static void updateLocation(
     final CatalogTracker catalogTracker, HRegionInfo regionInfo, ServerName sn)
     throws IOException {
   Put put = new Put(regionInfo.getRegionName());
   addLocation(put, sn);
   putToCatalogTable(catalogTracker, put);
   LOG.info("Updated row " + regionInfo.getRegionNameAsString() + " with server=" + sn);
 }
Esempio n. 23
0
  @Override
  public void process() throws IOException {
    boolean transitionToFailedOpen = false;
    boolean openSuccessful = false;
    try {
      final String name = regionInfo.getRegionNameAsString();
      if (this.server.isStopped() || this.rsServices.isStopping()) {
        return;
      }
      final String encodedName = regionInfo.getEncodedName();

      // Check that this region is not already online
      HRegion region = this.rsServices.getFromOnlineRegions(encodedName);

      // Open region.  After a successful open, failures in subsequent
      // processing needs to do a close as part of cleanup.
      region = openRegion();
      if (region == null) {
        tryTransitionToFailedOpen(regionInfo);
        transitionToFailedOpen = true;
        return;
      }
      boolean failed = true;
      if (tickleOpening("post_region_open")) {
        if (updateMeta(region)) {
          failed = false;
        }
      }
      if (failed || this.server.isStopped() || this.rsServices.isStopping()) {
        cleanupFailedOpen(region);
        tryTransitionToFailedOpen(regionInfo);
        transitionToFailedOpen = true;
        return;
      }

      if (!transitionToOpened(region)) {
        // If we fail to transition to opened, it's because of one of two cases:
        //    (a) we lost our ZK lease
        // OR (b) someone else opened the region before us
        // In either case, we don't need to transition to FAILED_OPEN state.
        // In case (a), the Master will process us as a dead server. In case
        // (b) the region is already being handled elsewhere anyway.
        cleanupFailedOpen(region);
        transitionToFailedOpen = true;
        return;
      }
      // Successful region open, and add it to OnlineRegions
      this.rsServices.addToOnlineRegions(region);
      openSuccessful = true;
      // Done!  Successful region open
      LOG.debug("Opened " + name + " on server:" + this.server.getServerName());
    } finally {
      this.rsServices.removeFromRegionsInTransition(this.regionInfo);
      if (!openSuccessful && !transitionToFailedOpen) {
        tryTransitionToFailedOpen(regionInfo);
      }
    }
  }
 public void printLocalityAndDispersionForCurrentPlan(
     Map<String, Map<String, Float>> regionLocalityMap) throws IOException {
   SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot();
   FavoredNodesPlan assignmentPlan = snapshot.getExistingAssignmentPlan();
   Set<TableName> tables = snapshot.getTableSet();
   Map<TableName, List<HRegionInfo>> tableToRegionsMap = snapshot.getTableToRegionMap();
   for (TableName table : tables) {
     float[] locality = new float[3];
     if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
       continue;
     }
     List<HRegionInfo> regions = tableToRegionsMap.get(table);
     for (HRegionInfo currentRegion : regions) {
       Map<String, Float> regionLocality = regionLocalityMap.get(currentRegion.getEncodedName());
       if (regionLocality == null) {
         continue;
       }
       List<ServerName> servers = assignmentPlan.getFavoredNodes(currentRegion);
       if (servers != null) {
         int i = 0;
         for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) {
           ServerName server = servers.get(p.ordinal());
           Float currentLocality = 0f;
           if (servers != null) {
             currentLocality = regionLocality.get(server.getHostname());
             if (currentLocality == null) {
               currentLocality = 0f;
             }
             locality[i] += currentLocality;
           }
           i++;
         }
       }
     }
     for (int i = 0; i < locality.length; i++) {
       String copy = null;
       if (i == 0) {
         copy = "primary";
       } else if (i == 1) {
         copy = "secondary";
       } else if (i == 2) {
         copy = "tertiary";
       }
       float avgLocality = 100 * locality[i] / regions.size();
       LOG.info(
           "For Table: "
               + table
               + " ; #Total Regions: "
               + regions.size()
               + " ; The average locality for "
               + copy
               + " is "
               + avgLocality
               + " %");
     }
     printDispersionScores(table, snapshot, regions.size(), null, false);
   }
 }
Esempio n. 25
0
 /**
  * Deletes the specified regions from META.
  *
  * @param catalogTracker
  * @param regionsInfo list of regions to be deleted from META
  * @throws IOException
  */
 public static void deleteRegions(CatalogTracker catalogTracker, List<HRegionInfo> regionsInfo)
     throws IOException {
   List<Delete> deletes = new ArrayList<Delete>(regionsInfo.size());
   for (HRegionInfo hri : regionsInfo) {
     deletes.add(new Delete(hri.getRegionName()));
   }
   deleteFromMetaTable(catalogTracker, deletes);
   LOG.info("Deleted from META, regions: " + regionsInfo);
 }
Esempio n. 26
0
 /** Extract the region encoded name from the region manifest */
 static String getRegionNameFromManifest(final SnapshotRegionManifest manifest) {
   byte[] regionName =
       HRegionInfo.createRegionName(
           ProtobufUtil.toTableName(manifest.getRegionInfo().getTableName()),
           manifest.getRegionInfo().getStartKey().toByteArray(),
           manifest.getRegionInfo().getRegionId(),
           true);
   return HRegionInfo.encodeRegionName(regionName);
 }
Esempio n. 27
0
 protected static List<HRegionInfo> getRegionInfoList(ServerName serverName, String tableName)
     throws IOException {
   List<HRegionInfo> onlineRegions = new ArrayList<>();
   for (HRegionInfo onlineRegion : CommandAdapter.getOnlineRegions(null, admin, serverName)) {
     if (onlineRegion.getTableNameAsString().equals(tableName)) {
       onlineRegions.add(onlineRegion);
     }
   }
   return onlineRegions;
 }
Esempio n. 28
0
  @Test
  public void testContainsRange() {
    HTableDescriptor tableDesc = new HTableDescriptor("testtable");
    HRegionInfo hri = new HRegionInfo(tableDesc, Bytes.toBytes("a"), Bytes.toBytes("g"));
    // Single row range at start of region
    assertTrue(hri.containsRange(Bytes.toBytes("a"), Bytes.toBytes("a")));
    // Fully contained range
    assertTrue(hri.containsRange(Bytes.toBytes("b"), Bytes.toBytes("c")));
    // Range overlapping start of region
    assertTrue(hri.containsRange(Bytes.toBytes("a"), Bytes.toBytes("c")));
    // Fully contained single-row range
    assertTrue(hri.containsRange(Bytes.toBytes("c"), Bytes.toBytes("c")));
    // Range that overlaps end key and hence doesn't fit
    assertFalse(hri.containsRange(Bytes.toBytes("a"), Bytes.toBytes("g")));
    // Single row range on end key
    assertFalse(hri.containsRange(Bytes.toBytes("g"), Bytes.toBytes("g")));
    // Single row range entirely outside
    assertFalse(hri.containsRange(Bytes.toBytes("z"), Bytes.toBytes("z")));

    // Degenerate range
    try {
      hri.containsRange(Bytes.toBytes("z"), Bytes.toBytes("a"));
      fail("Invalid range did not throw IAE");
    } catch (IllegalArgumentException iae) {
    }
  }
Esempio n. 29
0
 /**
  * Gets the online regions of the specified table. This method looks at the in-memory state. It
  * does not go to <code>hbase:meta</code>. Only returns <em>online</em> regions. If a region on
  * this table has been closed during a disable, etc., it will be included in the returned list.
  * So, the returned list may not necessarily be ALL regions in this table, its all the ONLINE
  * regions in the table.
  *
  * @param tableName
  * @return Online regions from <code>tableName</code>
  */
 public synchronized List<HRegionInfo> getRegionsOfTable(TableName tableName) {
   List<HRegionInfo> tableRegions = new ArrayList<HRegionInfo>();
   // boundary needs to have table's name but regionID 0 so that it is sorted
   // before all table's regions.
   HRegionInfo boundary = new HRegionInfo(tableName, null, null, false, 0L);
   for (HRegionInfo hri : regionAssignments.tailMap(boundary).keySet()) {
     if (!hri.getTable().equals(tableName)) break;
     tableRegions.add(hri);
   }
   return tableRegions;
 }
 private PairOfSameType<HRegionInfo> requestMergeRegion(
     HMaster master, TableName tablename, int regionAnum, int regionBnum) throws Exception {
   List<Pair<HRegionInfo, ServerName>> tableRegions =
       MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename);
   HRegionInfo regionA = tableRegions.get(regionAnum).getFirst();
   HRegionInfo regionB = tableRegions.get(regionBnum).getFirst();
   TEST_UTIL
       .getHBaseAdmin()
       .mergeRegions(regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false);
   return new PairOfSameType<HRegionInfo>(regionA, regionB);
 }