Ejemplo n.º 1
0
    @Override
    public void run() {
      byte[] key = new byte[keySize];
      byte[] value = new byte[valueSize];
      Random rand = new Random(Thread.currentThread().getId());
      HLog hlog = region.getLog();

      try {
        long startTime = System.currentTimeMillis();
        for (int i = 0; i < numIterations; ++i) {
          Put put = setupPut(rand, key, value, numFamilies);
          long now = System.currentTimeMillis();
          WALEdit walEdit = new WALEdit();
          addFamilyMapToWALEdit(put.getFamilyCellMap(), walEdit);
          HRegionInfo hri = region.getRegionInfo();
          if (this.noSync) {
            hlog.appendNoSync(hri, hri.getTable(), walEdit, new ArrayList<UUID>(), now, htd);
          } else {
            hlog.append(hri, hri.getTable(), walEdit, now, htd);
          }
        }
        long totalTime = (System.currentTimeMillis() - startTime);
        logBenchmarkResult(Thread.currentThread().getName(), numIterations, totalTime);
      } catch (Exception e) {
        LOG.error(getClass().getSimpleName() + " Thread failed", e);
      }
    }
Ejemplo n.º 2
0
 @Override
 public void postMove(
     ObserverContext<MasterCoprocessorEnvironment> ctx,
     HRegionInfo region,
     ServerName srcServer,
     ServerName destServer)
     throws IOException {
   if (balancer != null && balancer.isTableColocated(region.getTable())) {
     AssignmentManager am = ctx.getEnvironment().getMasterServices().getAssignmentManager();
     RegionStates regionStates = am.getRegionStates();
     String tableName = region.getTable().getNameAsString();
     String correspondingTable =
         region.getTable().getNameAsString().startsWith(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX)
             ? MetaDataUtil.getUserTableName(tableName)
             : MetaDataUtil.getLocalIndexTableName(tableName);
     List<HRegionInfo> regions =
         regionStates.getRegionsOfTable(TableName.valueOf(correspondingTable));
     for (HRegionInfo hri : regions) {
       if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) == 0 && destServer != null) {
         balancer.regionOnline(hri, destServer);
         am.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, destServer));
         am.unassign(hri);
       }
     }
   }
   super.postMove(ctx, region, srcServer, destServer);
 }
Ejemplo n.º 3
0
 protected HashMap<TableName, TreeMap<ServerName, List<HRegionInfo>>> mockClusterServersWithTables(
     Map<ServerName, List<HRegionInfo>> clusterServers) {
   HashMap<TableName, TreeMap<ServerName, List<HRegionInfo>>> result = new HashMap<>();
   for (Map.Entry<ServerName, List<HRegionInfo>> entry : clusterServers.entrySet()) {
     ServerName sal = entry.getKey();
     List<HRegionInfo> regions = entry.getValue();
     for (HRegionInfo hri : regions) {
       TreeMap<ServerName, List<HRegionInfo>> servers = result.get(hri.getTable());
       if (servers == null) {
         servers = new TreeMap<ServerName, List<HRegionInfo>>();
         result.put(hri.getTable(), servers);
       }
       List<HRegionInfo> hrilist = servers.get(sal);
       if (hrilist == null) {
         hrilist = new ArrayList<HRegionInfo>();
         servers.put(sal, hrilist);
       }
       hrilist.add(hri);
     }
   }
   for (Map.Entry<TableName, TreeMap<ServerName, List<HRegionInfo>>> entry : result.entrySet()) {
     for (ServerName srn : clusterServers.keySet()) {
       if (!entry.getValue().containsKey(srn))
         entry.getValue().put(srn, new ArrayList<HRegionInfo>());
     }
   }
   return result;
 }
Ejemplo n.º 4
0
  @Override
  public boolean prepare() throws IOException {
    if (!this.parent.isSplittable()) return false;
    // Split key can be null if this region is unsplittable; i.e. has refs.
    if (this.splitrow == null) return false;
    HRegionInfo hri = this.parent.getRegionInfo();
    parent.prepareToSplit();
    // Check splitrow.
    byte[] startKey = hri.getStartKey();
    byte[] endKey = hri.getEndKey();
    if (Bytes.equals(startKey, splitrow) || !this.parent.getRegionInfo().containsRow(splitrow)) {
      LOG.info(
          "Split row is not inside region key range or is equal to "
              + "startkey: "
              + Bytes.toStringBinary(this.splitrow));
      return false;
    }
    long rid = getDaughterRegionIdTimestamp(hri);
    this.hri_a = new HRegionInfo(hri.getTable(), startKey, this.splitrow, false, rid);
    this.hri_b = new HRegionInfo(hri.getTable(), this.splitrow, endKey, false, rid);

    transition(SplitTransactionPhase.PREPARED);

    return true;
  }
Ejemplo n.º 5
0
 /**
  * A region is offline, won't be in transition any more. Its state should be the specified
  * expected state, which can only be Split/Merged/Offline/null(=Offline)/SplittingNew/MergingNew.
  */
 public void regionOffline(final HRegionInfo hri, final State expectedState) {
   Preconditions.checkArgument(
       expectedState == null || RegionState.isUnassignable(expectedState),
       "Offlined region should not be " + expectedState);
   if (isRegionInState(hri, State.SPLITTING_NEW, State.MERGING_NEW)) {
     // Remove it from all region maps
     deleteRegion(hri);
     return;
   }
   State newState = expectedState == null ? State.OFFLINE : expectedState;
   updateRegionState(hri, newState);
   String encodedName = hri.getEncodedName();
   synchronized (this) {
     regionsInTransition.remove(encodedName);
     ServerName oldServerName = regionAssignments.remove(hri);
     if (oldServerName != null && serverHoldings.containsKey(oldServerName)) {
       if (newState == State.MERGED
           || newState == State.SPLIT
           || hri.isMetaRegion()
           || tableStateManager.isTableState(
               hri.getTable(), TableState.State.DISABLED, TableState.State.DISABLING)) {
         // Offline the region only if it's merged/split, or the table is disabled/disabling.
         // Otherwise, offline it from this server only when it is online on a different server.
         LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
         removeFromServerHoldings(oldServerName, hri);
         removeFromReplicaMapping(hri);
       } else {
         // Need to remember it so that we can offline it from this
         // server when it is online on a different server.
         oldAssignments.put(encodedName, oldServerName);
       }
     }
   }
 }
Ejemplo n.º 6
0
  /** Test that if we do a close while opening it stops the opening. */
  @Test(timeout = 60000)
  public void testCancelOpeningWithoutZK() throws Exception {
    // We close
    closeRegionNoZK();
    checkRegionIsClosed(HTU, getRS(), hri);

    // Let do the initial steps, without having a handler
    getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE);

    // That's a close without ZK.
    AdminProtos.CloseRegionRequest crr =
        RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName);
    try {
      getRS().rpcServices.closeRegion(null, crr);
      Assert.assertTrue(false);
    } catch (ServiceException expected) {
    }

    // The state in RIT should have changed to close
    Assert.assertEquals(
        Boolean.FALSE, getRS().getRegionsInTransitionInRS().get(hri.getEncodedNameAsBytes()));

    // Let's start the open handler
    HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable());

    getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd));

    // The open handler should have removed the region from RIT but kept the region closed
    checkRegionIsClosed(HTU, getRS(), hri);

    openRegion(HTU, getRS(), hri);
  }
Ejemplo n.º 7
0
 private int getRegionReplication(HRegionInfo r) throws IOException {
   if (tableStateManager != null) {
     HTableDescriptor htd = server.getTableDescriptors().get(r.getTable());
     if (htd != null) {
       return htd.getRegionReplication();
     }
   }
   return 1;
 }
Ejemplo n.º 8
0
 /**
  * Gets the online regions of the specified table. This method looks at the in-memory state. It
  * does not go to <code>hbase:meta</code>. Only returns <em>online</em> regions. If a region on
  * this table has been closed during a disable, etc., it will be included in the returned list.
  * So, the returned list may not necessarily be ALL regions in this table, its all the ONLINE
  * regions in the table.
  *
  * @param tableName
  * @return Online regions from <code>tableName</code>
  */
 public synchronized List<HRegionInfo> getRegionsOfTable(TableName tableName) {
   List<HRegionInfo> tableRegions = new ArrayList<HRegionInfo>();
   // boundary needs to have table's name but regionID 0 so that it is sorted
   // before all table's regions.
   HRegionInfo boundary = new HRegionInfo(tableName, null, null, false, 0L);
   for (HRegionInfo hri : regionAssignments.tailMap(boundary).keySet()) {
     if (!hri.getTable().equals(tableName)) break;
     tableRegions.add(hri);
   }
   return tableRegions;
 }
Ejemplo n.º 9
0
 private RegionState putRegionState(RegionState regionState) {
   HRegionInfo hri = regionState.getRegion();
   String encodedName = hri.getEncodedName();
   TableName table = hri.getTable();
   RegionState oldState = regionStates.put(encodedName, regionState);
   Map<String, RegionState> map = regionStatesTableIndex.get(table);
   if (map == null) {
     map = new HashMap<String, RegionState>();
     regionStatesTableIndex.put(table, map);
   }
   map.put(encodedName, regionState);
   return oldState;
 }
Ejemplo n.º 10
0
 /**
  * A table is deleted. Remove its regions from all internal maps. We loop through all regions
  * assuming we don't delete tables too much.
  */
 public void tableDeleted(final TableName tableName) {
   Set<HRegionInfo> regionsToDelete = new HashSet<HRegionInfo>();
   synchronized (this) {
     for (RegionState state : regionStates.values()) {
       HRegionInfo region = state.getRegion();
       if (region.getTable().equals(tableName)) {
         regionsToDelete.add(region);
       }
     }
   }
   for (HRegionInfo region : regionsToDelete) {
     deleteRegion(region);
   }
 }
  private void addRegion(HRegionInfo regionInfo) {
    // Process the region name to region info map
    regionNameToRegionInfoMap.put(regionInfo.getRegionNameAsString(), regionInfo);

    // Process the table to region map
    TableName tableName = regionInfo.getTable();
    List<HRegionInfo> regionList = tableToRegionMap.get(tableName);
    if (regionList == null) {
      regionList = new ArrayList<HRegionInfo>();
    }
    // Add the current region info into the tableToRegionMap
    regionList.add(regionInfo);
    tableToRegionMap.put(tableName, regionList);
  }
Ejemplo n.º 12
0
 /** Remove a region from all state maps. */
 @VisibleForTesting
 public synchronized void deleteRegion(final HRegionInfo hri) {
   String encodedName = hri.getEncodedName();
   regionsInTransition.remove(encodedName);
   regionStates.remove(encodedName);
   TableName table = hri.getTable();
   Map<String, RegionState> indexMap = regionStatesTableIndex.get(table);
   indexMap.remove(encodedName);
   if (indexMap.size() == 0) regionStatesTableIndex.remove(table);
   lastAssignments.remove(encodedName);
   ServerName sn = regionAssignments.remove(hri);
   if (sn != null) {
     Set<HRegionInfo> regions = serverHoldings.get(sn);
     regions.remove(hri);
   }
 }
Ejemplo n.º 13
0
 /**
  * Convert a HRegionInfo to a RegionInfo
  *
  * @param info the HRegionInfo to convert
  * @return the converted RegionInfo
  */
 public static RegionInfo convert(final HRegionInfo info) {
   if (info == null) return null;
   RegionInfo.Builder builder = RegionInfo.newBuilder();
   builder.setTableName(ProtobufUtil.toProtoTableName(info.getTable()));
   builder.setRegionId(info.getRegionId());
   if (info.getStartKey() != null) {
     builder.setStartKey(ByteStringer.wrap(info.getStartKey()));
   }
   if (info.getEndKey() != null) {
     builder.setEndKey(ByteStringer.wrap(info.getEndKey()));
   }
   builder.setOffline(info.isOffline());
   builder.setSplit(info.isSplit());
   builder.setReplicaId(info.getReplicaId());
   return builder.build();
 }
Ejemplo n.º 14
0
  /**
   * This is an EXPENSIVE clone. Cloning though is the safest thing to do. Can't let out original
   * since it can change and at least the load balancer wants to iterate this exported list. We need
   * to synchronize on regions since all access to this.servers is under a lock on this.regions.
   *
   * @return A clone of current assignments by table.
   */
  protected Map<TableName, Map<ServerName, List<HRegionInfo>>> getAssignmentsByTable() {
    Map<TableName, Map<ServerName, List<HRegionInfo>>> result =
        new HashMap<TableName, Map<ServerName, List<HRegionInfo>>>();
    synchronized (this) {
      if (!server
          .getConfiguration()
          .getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, false)) {
        Map<ServerName, List<HRegionInfo>> svrToRegions =
            new HashMap<ServerName, List<HRegionInfo>>(serverHoldings.size());
        for (Map.Entry<ServerName, Set<HRegionInfo>> e : serverHoldings.entrySet()) {
          svrToRegions.put(e.getKey(), new ArrayList<HRegionInfo>(e.getValue()));
        }
        result.put(TableName.valueOf(HConstants.ENSEMBLE_TABLE_NAME), svrToRegions);
      } else {
        for (Map.Entry<ServerName, Set<HRegionInfo>> e : serverHoldings.entrySet()) {
          for (HRegionInfo hri : e.getValue()) {
            if (hri.isMetaRegion()) continue;
            TableName tablename = hri.getTable();
            Map<ServerName, List<HRegionInfo>> svrToRegions = result.get(tablename);
            if (svrToRegions == null) {
              svrToRegions = new HashMap<ServerName, List<HRegionInfo>>(serverHoldings.size());
              result.put(tablename, svrToRegions);
            }
            List<HRegionInfo> regions = svrToRegions.get(e.getKey());
            if (regions == null) {
              regions = new ArrayList<HRegionInfo>();
              svrToRegions.put(e.getKey(), regions);
            }
            regions.add(hri);
          }
        }
      }
    }

    Map<ServerName, ServerLoad> onlineSvrs = serverManager.getOnlineServers();
    // Take care of servers w/o assignments, and remove servers in draining mode
    List<ServerName> drainingServers = this.serverManager.getDrainingServersList();
    for (Map<ServerName, List<HRegionInfo>> map : result.values()) {
      for (ServerName svr : onlineSvrs.keySet()) {
        if (!map.containsKey(svr)) {
          map.put(svr, new ArrayList<HRegionInfo>());
        }
      }
      map.keySet().removeAll(drainingServers);
    }
    return result;
  }
Ejemplo n.º 15
0
 HRegion createRegion(final Path testdir, final WALFactory wals) throws IOException {
   // Make a region with start and end keys. Use 'aaa', to 'AAA'.  The load
   // region utility will add rows between 'aaa' and 'zzz'.
   HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
   HColumnDescriptor hcd = new HColumnDescriptor(CF);
   htd.addFamily(hcd);
   HRegionInfo hri = new HRegionInfo(htd.getTableName(), STARTROW, ENDROW);
   HRegion r =
       HBaseTestingUtility.createRegionAndWAL(hri, testdir, TEST_UTIL.getConfiguration(), htd);
   HBaseTestingUtility.closeRegionAndWAL(r);
   return HRegion.openHRegion(
       testdir,
       hri,
       htd,
       wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace()),
       TEST_UTIL.getConfiguration());
 }
Ejemplo n.º 16
0
      @Override
      public void run() {
        while (!isStopped()) {
          try {
            List<HRegionInfo> regions =
                MetaScanner.listAllRegions(TEST_UTIL.getConfiguration(), connection, false);

            // select a random region
            HRegionInfo parent = regions.get(random.nextInt(regions.size()));
            if (parent == null || !TABLENAME.equals(parent.getTable())) {
              continue;
            }

            long startKey = 0, endKey = Long.MAX_VALUE;
            byte[] start = parent.getStartKey();
            byte[] end = parent.getEndKey();
            if (!Bytes.equals(HConstants.EMPTY_START_ROW, parent.getStartKey())) {
              startKey = Bytes.toLong(parent.getStartKey());
            }
            if (!Bytes.equals(HConstants.EMPTY_END_ROW, parent.getEndKey())) {
              endKey = Bytes.toLong(parent.getEndKey());
            }
            if (startKey == endKey) {
              continue;
            }

            long midKey =
                BigDecimal.valueOf(startKey)
                    .add(BigDecimal.valueOf(endKey))
                    .divideToIntegralValue(BigDecimal.valueOf(2))
                    .longValue();

            HRegionInfo splita = new HRegionInfo(TABLENAME, start, Bytes.toBytes(midKey));
            HRegionInfo splitb = new HRegionInfo(TABLENAME, Bytes.toBytes(midKey), end);

            MetaTableAccessor.splitRegion(
                connection, parent, splita, splitb, ServerName.valueOf("fooserver", 1, 0));

            Threads.sleep(random.nextInt(200));
          } catch (Throwable e) {
            ex = e;
            Assert.fail(StringUtils.stringifyException(e));
          }
        }
      }
Ejemplo n.º 17
0
    @Override
    public void run() {
      byte[] key = new byte[keySize];
      byte[] value = new byte[valueSize];
      Random rand = new Random(Thread.currentThread().getId());
      WAL wal = region.getWAL();

      TraceScope threadScope = Trace.startSpan("WALPerfEval." + Thread.currentThread().getName());
      try {
        long startTime = System.currentTimeMillis();
        int lastSync = 0;
        for (int i = 0; i < numIterations; ++i) {
          assert Trace.currentSpan() == threadScope.getSpan() : "Span leak detected.";
          TraceScope loopScope = Trace.startSpan("runLoopIter" + i, loopSampler);
          try {
            long now = System.nanoTime();
            Put put = setupPut(rand, key, value, numFamilies);
            WALEdit walEdit = new WALEdit();
            addFamilyMapToWALEdit(put.getFamilyCellMap(), walEdit);
            HRegionInfo hri = region.getRegionInfo();
            final WALKey logkey = new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), now);
            wal.append(htd, hri, logkey, walEdit, region.getSequenceId(), true, null);
            if (!this.noSync) {
              if (++lastSync >= this.syncInterval) {
                wal.sync();
                lastSync = 0;
              }
            }
            latencyHistogram.update(System.nanoTime() - now);
          } finally {
            loopScope.close();
          }
        }
        long totalTime = (System.currentTimeMillis() - startTime);
        logBenchmarkResult(Thread.currentThread().getName(), numIterations, totalTime);
      } catch (Exception e) {
        LOG.error(getClass().getSimpleName() + " Thread failed", e);
      } finally {
        threadScope.close();
      }
    }
Ejemplo n.º 18
0
 /**
  * Get the region name for display. Optionally hide the start key.
  *
  * @param hri
  * @param conf
  * @return region name bytes
  */
 public static byte[] getRegionNameForDisplay(HRegionInfo hri, Configuration conf) {
   boolean displayKey = conf.getBoolean(DISPLAY_KEYS_KEY, true);
   if (displayKey || hri.getTable().equals(TableName.META_TABLE_NAME)) {
     return hri.getRegionName();
   } else {
     // create a modified regionname with the startkey replaced but preserving
     // the other parts including the encodedname.
     try {
       byte[][] regionNameParts = parseRegionName(hri.getRegionName());
       regionNameParts[1] = HIDDEN_START_KEY; // replace the real startkey
       int len = 0;
       // get the total length
       for (byte[] b : regionNameParts) {
         len += b.length;
       }
       byte[] encodedRegionName = Bytes.toBytes(encodeRegionName(hri.getRegionName()));
       len += encodedRegionName.length;
       // allocate some extra bytes for the delimiters and the last '.'
       byte[] modifiedName = new byte[len + regionNameParts.length + 1];
       int lengthSoFar = 0;
       int loopCount = 0;
       for (byte[] b : regionNameParts) {
         System.arraycopy(b, 0, modifiedName, lengthSoFar, b.length);
         lengthSoFar += b.length;
         if (loopCount++ == 2) modifiedName[lengthSoFar++] = REPLICA_ID_DELIMITER;
         else modifiedName[lengthSoFar++] = HConstants.DELIMITER;
       }
       // replace the last comma with '.'
       modifiedName[lengthSoFar - 1] = ENC_SEPARATOR;
       System.arraycopy(encodedRegionName, 0, modifiedName, lengthSoFar, encodedRegionName.length);
       lengthSoFar += encodedRegionName.length;
       modifiedName[lengthSoFar] = ENC_SEPARATOR;
       return modifiedName;
     } catch (IOException e) {
       // LOG.warn("Encountered exception " + e);
       throw new RuntimeException(e);
     }
   }
 }
Ejemplo n.º 19
0
 protected void wipeOutMeta() throws IOException {
   // Mess it up by blowing up meta.
   Admin admin = TEST_UTIL.getHBaseAdmin();
   Scan s = new Scan();
   Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
   ResultScanner scanner = meta.getScanner(s);
   List<Delete> dels = new ArrayList<Delete>();
   for (Result r : scanner) {
     HRegionInfo info = HRegionInfo.getHRegionInfo(r);
     if (info != null
         && !info.getTable()
             .getNamespaceAsString()
             .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
       Delete d = new Delete(r.getRow());
       dels.add(d);
       admin.unassign(r.getRow(), true);
     }
   }
   meta.delete(dels);
   scanner.close();
   meta.close();
 }
Ejemplo n.º 20
0
 /**
  * At cluster clean re/start, mark all user regions closed except those of tables that are
  * excluded, such as disabled/disabling/enabling tables. All user regions and their previous
  * locations are returned.
  */
 synchronized Map<HRegionInfo, ServerName> closeAllUserRegions(Set<TableName> excludedTables) {
   boolean noExcludeTables = excludedTables == null || excludedTables.isEmpty();
   Set<HRegionInfo> toBeClosed = new HashSet<HRegionInfo>(regionStates.size());
   for (RegionState state : regionStates.values()) {
     HRegionInfo hri = state.getRegion();
     if (state.isSplit() || hri.isSplit()) {
       continue;
     }
     TableName tableName = hri.getTable();
     if (!TableName.META_TABLE_NAME.equals(tableName)
         && (noExcludeTables || !excludedTables.contains(tableName))) {
       toBeClosed.add(hri);
     }
   }
   Map<HRegionInfo, ServerName> allUserRegions =
       new HashMap<HRegionInfo, ServerName>(toBeClosed.size());
   for (HRegionInfo hri : toBeClosed) {
     RegionState regionState = updateRegionState(hri, State.CLOSED);
     allUserRegions.put(hri, regionState.getServerName());
   }
   return allUserRegions;
 }
Ejemplo n.º 21
0
  public void addMobRegion(HRegionInfo regionInfo, HColumnDescriptor[] hcds) throws IOException {
    // 0. Get the ManifestBuilder/RegionVisitor
    RegionVisitor visitor = createRegionVisitor(desc);

    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot.");
    Object regionData = visitor.regionOpen(regionInfo);
    monitor.rethrowException();

    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for mob files");

    Path mobRegionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable());
    for (HColumnDescriptor hcd : hcds) {
      // 2.1. build the snapshot reference for the store if it's a mob store
      if (!hcd.isMobEnabled()) {
        continue;
      }
      Object familyData = visitor.familyOpen(regionData, hcd.getName());
      monitor.rethrowException();

      Path storePath = MobUtils.getMobFamilyPath(mobRegionPath, hcd.getNameAsString());
      List<StoreFileInfo> storeFiles = getStoreFiles(storePath);
      if (storeFiles == null) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("No mob files under family: " + hcd.getNameAsString());
        }
        continue;
      }

      addReferenceFiles(visitor, regionData, familyData, storeFiles, true);

      visitor.familyClose(regionData, familyData);
    }
    visitor.regionClose(regionData);
  }
  private Region initHRegion(HTableDescriptor htd, byte[] startKey, byte[] stopKey, int replicaId)
      throws IOException {
    Configuration conf = TEST_UTIL.getConfiguration();
    Path tableDir = FSUtils.getTableDir(testDir, htd.getTableName());

    HRegionInfo info = new HRegionInfo(htd.getTableName(), startKey, stopKey, false, 0, replicaId);

    HRegionFileSystem fs =
        new FailingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir, info);
    final Configuration walConf = new Configuration(conf);
    FSUtils.setRootDir(walConf, tableDir);
    final WALFactory wals = new WALFactory(walConf, null, "log_" + replicaId);
    HRegion region =
        new HRegion(
            fs,
            wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace()),
            conf,
            htd,
            null);

    region.initialize();

    return region;
  }
  /**
   * Get merged region info through the specified two regions
   *
   * @param a merging region A
   * @param b merging region B
   * @return the merged region info
   */
  public static HRegionInfo getMergedRegionInfo(final HRegionInfo a, final HRegionInfo b) {
    long rid = EnvironmentEdgeManager.currentTime();
    // Regionid is timestamp. Merged region's id can't be less than that of
    // merging regions else will insert at wrong location in hbase:meta
    if (rid < a.getRegionId() || rid < b.getRegionId()) {
      LOG.warn(
          "Clock skew; merging regions id are "
              + a.getRegionId()
              + " and "
              + b.getRegionId()
              + ", but current time here is "
              + rid);
      rid = Math.max(a.getRegionId(), b.getRegionId()) + 1;
    }

    byte[] startKey = null;
    byte[] endKey = null;
    // Choose the smaller as start key
    if (a.compareTo(b) <= 0) {
      startKey = a.getStartKey();
    } else {
      startKey = b.getStartKey();
    }
    // Choose the bigger as end key
    if (Bytes.equals(a.getEndKey(), HConstants.EMPTY_BYTE_ARRAY)
        || (!Bytes.equals(b.getEndKey(), HConstants.EMPTY_BYTE_ARRAY)
            && Bytes.compareTo(a.getEndKey(), b.getEndKey()) > 0)) {
      endKey = a.getEndKey();
    } else {
      endKey = b.getEndKey();
    }

    // Merged region is sorted between two merging regions in META
    HRegionInfo mergedRegionInfo = new HRegionInfo(a.getTable(), startKey, endKey, false, rid);
    return mergedRegionInfo;
  }
  // check each region whether the coprocessor upcalls are called or not.
  private void verifyMethodResult(
      Class<?> c, String methodName[], TableName tableName, Object value[]) throws IOException {
    try {
      for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) {
        if (!t.isAlive() || t.getRegionServer().isAborted() || t.getRegionServer().isStopping()) {
          continue;
        }
        for (HRegionInfo r : ProtobufUtil.getOnlineRegions(t.getRegionServer())) {
          if (!r.getTable().equals(tableName)) {
            continue;
          }
          RegionCoprocessorHost cph =
              t.getRegionServer().getOnlineRegion(r.getRegionName()).getCoprocessorHost();

          Coprocessor cp = cph.findCoprocessor(c.getName());
          assertNotNull(cp);
          for (int i = 0; i < methodName.length; ++i) {
            Method m = c.getMethod(methodName[i]);
            Object o = m.invoke(cp);
            assertTrue(
                "Result of "
                    + c.getName()
                    + "."
                    + methodName[i]
                    + " is expected to be "
                    + value[i].toString()
                    + ", while we get "
                    + o.toString(),
                o.equals(value[i]));
          }
        }
      }
    } catch (Exception e) {
      throw new IOException(e.toString());
    }
  }
  @SuppressWarnings("deprecation")
  @Test
  public void testCleanMergeReference() throws Exception {
    LOG.info("Starting testCleanMergeReference");
    admin.enableCatalogJanitor(false);
    try {
      final TableName tableName = TableName.valueOf("testCleanMergeReference");
      // Create table and load data.
      Table table = createTableAndLoadData(master, tableName);
      // Merge 1st and 2nd region
      mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1, INITIAL_REGION_NUM - 1);
      verifyRowCount(table, ROWSIZE);
      table.close();

      List<Pair<HRegionInfo, ServerName>> tableRegions =
          MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tableName);
      HRegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
      HTableDescriptor tableDescritor = master.getTableDescriptors().get(tableName);
      Result mergedRegionResult =
          MetaTableAccessor.getRegionResult(
              master.getConnection(), mergedRegionInfo.getRegionName());

      // contains merge reference in META
      assertTrue(
          mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER)
              != null);
      assertTrue(
          mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER)
              != null);

      // merging regions' directory are in the file system all the same
      PairOfSameType<HRegionInfo> p = MetaTableAccessor.getMergeRegions(mergedRegionResult);
      HRegionInfo regionA = p.getFirst();
      HRegionInfo regionB = p.getSecond();
      FileSystem fs = master.getMasterFileSystem().getFileSystem();
      Path rootDir = master.getMasterFileSystem().getRootDir();

      Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
      Path regionAdir = new Path(tabledir, regionA.getEncodedName());
      Path regionBdir = new Path(tabledir, regionB.getEncodedName());
      assertTrue(fs.exists(regionAdir));
      assertTrue(fs.exists(regionBdir));

      admin.compactRegion(mergedRegionInfo.getRegionName());
      // wait until merged region doesn't have reference file
      long timeout = System.currentTimeMillis() + waitTime;
      HRegionFileSystem hrfs =
          new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
      while (System.currentTimeMillis() < timeout) {
        if (!hrfs.hasReferences(tableDescritor)) {
          break;
        }
        Thread.sleep(50);
      }
      assertFalse(hrfs.hasReferences(tableDescritor));

      // run CatalogJanitor to clean merge references in hbase:meta and archive the
      // files of merging regions
      int cleaned = admin.runCatalogScan();
      assertTrue(cleaned > 0);
      assertFalse(fs.exists(regionAdir));
      assertFalse(fs.exists(regionBdir));

      mergedRegionResult =
          MetaTableAccessor.getRegionResult(
              master.getConnection(), mergedRegionInfo.getRegionName());
      assertFalse(
          mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER)
              != null);
      assertFalse(
          mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER)
              != null);

    } finally {
      admin.enableCatalogJanitor(true);
    }
  }
Ejemplo n.º 26
0
  /**
   * Creates a 'manifest' for the specified region, by reading directly from the disk. This is used
   * by the "offline snapshot" when the table is disabled.
   */
  public void addRegion(final Path tableDir, final HRegionInfo regionInfo) throws IOException {
    // 0. Get the ManifestBuilder/RegionVisitor
    RegionVisitor visitor = createRegionVisitor(desc);

    boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
    try {
      // Open the RegionFS
      HRegionFileSystem regionFs =
          HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, true);
      monitor.rethrowException();

      // 1. dump region meta info into the snapshot directory
      LOG.debug("Storing region-info for snapshot.");
      Object regionData = visitor.regionOpen(regionInfo);
      monitor.rethrowException();

      // 2. iterate through all the stores in the region
      LOG.debug("Creating references for hfiles");

      // This ensures that we have an atomic view of the directory as long as we have < ls limit
      // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files
      // in batches and may miss files being added/deleted. This could be more robust (iteratively
      // checking to see if we have all the files until we are sure), but the limit is currently
      // 1000 files/batch, far more than the number of store files under a single column family.
      Collection<String> familyNames = regionFs.getFamilies();
      if (familyNames != null) {
        for (String familyName : familyNames) {
          Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
          monitor.rethrowException();

          Collection<StoreFileInfo> storeFiles = null;
          if (isMobRegion) {
            Path regionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable());
            Path storePath = MobUtils.getMobFamilyPath(regionPath, familyName);
            storeFiles = getStoreFiles(storePath);
          } else {
            storeFiles = regionFs.getStoreFiles(familyName);
          }

          if (storeFiles == null) {
            if (LOG.isDebugEnabled()) {
              LOG.debug("No files under family: " + familyName);
            }
            continue;
          }

          // 2.1. build the snapshot reference for the store
          // iterate through all the store's files and create "references".
          addReferenceFiles(visitor, regionData, familyData, storeFiles, false);

          visitor.familyClose(regionData, familyData);
        }
      }
      visitor.regionClose(regionData);
    } catch (IOException e) {
      // the mob directory might not be created yet, so do nothing when it is a mob region
      if (!isMobRegion) {
        throw e;
      }
    }
  }