public void doAnAction() throws Exception {
      long iteration = numBulkLoads.getAndIncrement();
      Path dir = UTIL.getDataTestDirOnTestFS(String.format("bulkLoad_%08d", iteration));

      // create HFiles for different column families
      FileSystem fs = UTIL.getTestFileSystem();
      byte[] val = Bytes.toBytes(String.format("%010d", iteration));
      final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>(NUM_CFS);
      for (int i = 0; i < NUM_CFS; i++) {
        Path hfile = new Path(dir, family(i));
        byte[] fam = Bytes.toBytes(family(i));
        createHFile(fs, hfile, fam, QUAL, val, 1000);
        famPaths.add(new Pair<>(fam, hfile.toString()));
      }

      // bulk load HFiles
      final ClusterConnection conn = (ClusterConnection) UTIL.getAdmin().getConnection();
      RegionServerCallable<Void> callable =
          new RegionServerCallable<Void>(conn, tableName, Bytes.toBytes("aaa")) {
            @Override
            public Void call(int callTimeout) throws Exception {
              LOG.debug(
                  "Going to connect to server "
                      + getLocation()
                      + " for row "
                      + Bytes.toStringBinary(getRow()));
              byte[] regionName = getLocation().getRegionInfo().getRegionName();
              BulkLoadHFileRequest request =
                  RequestConverter.buildBulkLoadHFileRequest(famPaths, regionName, true);
              getStub().bulkLoadHFile(null, request);
              return null;
            }
          };
      RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(conf);
      RpcRetryingCaller<Void> caller = factory.<Void>newCaller();
      caller.callWithRetries(callable, Integer.MAX_VALUE);

      // Periodically do compaction to reduce the number of open file handles.
      if (numBulkLoads.get() % 5 == 0) {
        // 5 * 50 = 250 open file handles!
        callable =
            new RegionServerCallable<Void>(conn, tableName, Bytes.toBytes("aaa")) {
              @Override
              public Void call(int callTimeout) throws Exception {
                LOG.debug(
                    "compacting " + getLocation() + " for row " + Bytes.toStringBinary(getRow()));
                AdminProtos.AdminService.BlockingInterface server =
                    conn.getAdmin(getLocation().getServerName());
                CompactRegionRequest request =
                    RequestConverter.buildCompactRegionRequest(
                        getLocation().getRegionInfo().getRegionName(), true, null);
                server.compactRegion(null, request);
                numCompactions.incrementAndGet();
                return null;
              }
            };
        caller.callWithRetries(callable, Integer.MAX_VALUE);
      }
    }
    public RegionReplicaSinkWriter(
        RegionReplicaOutputSink sink,
        ClusterConnection connection,
        ExecutorService pool,
        int operationTimeout) {
      this.sink = sink;
      this.connection = connection;
      this.operationTimeout = operationTimeout;
      this.rpcRetryingCallerFactory =
          RpcRetryingCallerFactory.instantiate(connection.getConfiguration());
      this.rpcControllerFactory = RpcControllerFactory.instantiate(connection.getConfiguration());
      this.pool = pool;

      int nonExistentTableCacheExpiryMs =
          connection
              .getConfiguration()
              .getInt(
                  "hbase.region.replica.replication.cache.disabledAndDroppedTables.expiryMs", 5000);
      // A cache for non existing tables that have a default expiry of 5 sec. This means that if the
      // table is created again with the same name, we might miss to replicate for that amount of
      // time. But this cache prevents overloading meta requests for every edit from a deleted file.
      disabledAndDroppedTables =
          CacheBuilder.newBuilder()
              .expireAfterWrite(nonExistentTableCacheExpiryMs, TimeUnit.MILLISECONDS)
              .initialCapacity(10)
              .maximumSize(1000)
              .build();
    }
Exemplo n.º 3
0
  /**
   * Create a new ClientScanner for the specified table Note that the passed {@link Scan}'s start
   * row maybe changed changed.
   *
   * @param conf The {@link Configuration} to use.
   * @param scan {@link Scan} to use in this scanner
   * @param tableName The table that we wish to scan
   * @param connection Connection identifying the cluster
   * @throws IOException
   */
  public ClientScanner(
      final Configuration conf,
      final Scan scan,
      final TableName tableName,
      ClusterConnection connection,
      RpcRetryingCallerFactory rpcFactory,
      RpcControllerFactory controllerFactory,
      ExecutorService pool,
      int primaryOperationTimeout)
      throws IOException {
    if (LOG.isTraceEnabled()) {
      LOG.trace(
          "Scan table=" + tableName + ", startRow=" + Bytes.toStringBinary(scan.getStartRow()));
    }
    this.scan = scan;
    this.tableName = tableName;
    this.lastNext = System.currentTimeMillis();
    this.connection = connection;
    this.pool = pool;
    this.primaryOperationTimeout = primaryOperationTimeout;
    this.retries =
        conf.getInt(
            HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
    if (scan.getMaxResultSize() > 0) {
      this.maxScannerResultSize = scan.getMaxResultSize();
    } else {
      this.maxScannerResultSize =
          conf.getLong(
              HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
              HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
    }
    this.scannerTimeout =
        HBaseConfiguration.getInt(
            conf,
            HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
            HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
            HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);

    // check if application wants to collect scan metrics
    initScanMetrics(scan);

    // Use the caching from the Scan.  If not set, use the default cache setting for this table.
    if (this.scan.getCaching() > 0) {
      this.caching = this.scan.getCaching();
    } else {
      this.caching =
          conf.getInt(
              HConstants.HBASE_CLIENT_SCANNER_CACHING,
              HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
    }

    this.caller = rpcFactory.<Result[]>newCaller();
    this.rpcControllerFactory = controllerFactory;

    this.conf = conf;
    initializeScannerInConstruction();
  }
 @Override
 public V call() throws Exception {
   return factory.<V>newCaller().callWithRetries(callable, timeout);
 }