public IndexPersistenceMgr(
      int pageSize,
      int entriesPerPage,
      ServerConfiguration conf,
      SnapshotMap<Long, Boolean> activeLedgers,
      LedgerDirsManager ledgerDirsManager,
      StatsLogger statsLogger)
      throws IOException {
    this.openFileLimit = conf.getOpenFileLimit();
    this.activeLedgers = activeLedgers;
    this.ledgerDirsManager = ledgerDirsManager;
    this.pageSize = pageSize;
    this.entriesPerPage = entriesPerPage;
    LOG.info("openFileLimit = {}", openFileLimit);
    // Retrieve all of the active ledgers.
    getActiveLedgers();
    ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener());

    // Expose Stats
    evictedLedgersCounter = statsLogger.getCounter(LEDGER_CACHE_NUM_EVICTED_LEDGERS);
    statsLogger.registerGauge(
        NUM_OPEN_LEDGERS,
        new Gauge<Integer>() {
          @Override
          public Integer getDefaultValue() {
            return 0;
          }

          @Override
          public Integer getSample() {
            return getNumOpenLedgers();
          }
        });
  }
Example #2
0
  public IndexInMemPageMgr(
      int pageSize,
      int entriesPerPage,
      ServerConfiguration conf,
      IndexPersistenceMgr indexPersistenceManager,
      StatsLogger statsLogger) {
    this.pageSize = pageSize;
    this.entriesPerPage = entriesPerPage;
    this.indexPersistenceManager = indexPersistenceManager;
    this.pageMapAndList = new InMemPageCollection();

    long maxDirectMemory = DirectMemoryUtils.maxDirectMemory();

    if (conf.getPageLimit() <= 0) {
      // By default, allocate a third of the direct memory to the page cache
      this.pageLimit = (int) ((maxDirectMemory / 3) / this.pageSize);
    } else {
      this.pageLimit = conf.getPageLimit();
    }
    LOG.info(
        "maxDirectMemory = {}, pageSize = {}, pageLimit = {}",
        new Object[] {maxDirectMemory, pageSize, pageLimit});
    // Expose Stats
    statsLogger.registerGauge(
        NUM_INDEX_PAGES,
        new Gauge<Number>() {
          @Override
          public Number getDefaultValue() {
            return 0;
          }

          @Override
          public Number getSample() {
            return getNumUsedPages();
          }
        });
  }
Example #3
0
  public Bookie(ServerConfiguration conf, StatsLogger statsLogger)
      throws IOException, KeeperException, InterruptedException, BookieException {
    super("Bookie-" + conf.getBookiePort());
    this.statsLogger = statsLogger;
    this.bookieRegistrationPath = conf.getZkAvailableBookiesPath() + "/";
    this.bookieReadonlyRegistrationPath = this.bookieRegistrationPath + READONLY;
    this.conf = conf;
    this.journalDirectory = getCurrentDirectory(conf.getJournalDir());
    this.ledgerDirsManager =
        new LedgerDirsManager(
            conf, conf.getLedgerDirs(), statsLogger.scope(BOOKIE_SCOPE).scope(LD_LEDGER_SCOPE));
    File[] idxDirs = conf.getIndexDirs();
    if (null == idxDirs) {
      this.indexDirsManager = this.ledgerDirsManager;
    } else {
      this.indexDirsManager =
          new LedgerDirsManager(
              conf, idxDirs, statsLogger.scope(BOOKIE_SCOPE).scope(LD_INDEX_SCOPE));
    }
    // Expose stats
    this.addEntryStats = statsLogger.getOpStatsLogger(BOOKIE_ADD_ENTRY);
    this.recoveryAddEntryStats = statsLogger.getOpStatsLogger(BOOKIE_RECOVERY_ADD_ENTRY);
    this.readEntryStats = statsLogger.getOpStatsLogger(BOOKIE_READ_ENTRY);
    this.readLastConfirmedStats = statsLogger.getOpStatsLogger(BOOKIE_READ_LAST_CONFIRMED);
    // 1 : up, 0 : readonly, -1 : unregistered
    statsLogger.registerGauge(
        SERVER_STATUS,
        new Gauge<Number>() {
          @Override
          public Number getDefaultValue() {
            return 0;
          }

          @Override
          public Number getSample() {
            return zkRegistered.get() ? (readOnly.get() ? 0 : 1) : -1;
          }
        });
  }
  /**
   * Constructs Safe executor
   *
   * @param numThreads - number of threads
   * @param baseName - base name of executor threads
   * @param threadFactory - for constructing threads
   * @param statsLogger - for reporting executor stats
   * @param traceTaskExecution - should we stat task execution
   * @param warnTimeMicroSec - log long task exec warning after this interval
   */
  @SuppressWarnings("unchecked")
  private OrderedSafeExecutor(
      String baseName,
      int numThreads,
      ThreadFactory threadFactory,
      StatsLogger statsLogger,
      boolean traceTaskExecution,
      long warnTimeMicroSec) {
    Preconditions.checkArgument(numThreads > 0);
    Preconditions.checkArgument(!StringUtils.isBlank(baseName));

    this.warnTimeMicroSec = warnTimeMicroSec;
    name = baseName;
    threads = new ThreadPoolExecutor[numThreads];
    threadIds = new long[numThreads];
    queues = new BlockingQueue[numThreads];
    for (int i = 0; i < numThreads; i++) {
      queues[i] = new LinkedBlockingQueue<Runnable>();
      threads[i] =
          new ThreadPoolExecutor(
              1,
              1,
              0L,
              TimeUnit.MILLISECONDS,
              queues[i],
              new ThreadFactoryBuilder()
                  .setNameFormat(name + "-orderedsafeexecutor-" + i + "-%d")
                  .setThreadFactory(threadFactory)
                  .build());

      // Save thread ids
      final int idx = i;
      try {
        threads[idx]
            .submit(
                new SafeRunnable() {
                  @Override
                  public void safeRun() {
                    threadIds[idx] = Thread.currentThread().getId();
                  }
                })
            .get();
      } catch (InterruptedException e) {
        throw new RuntimeException("Couldn't start thread " + i, e);
      } catch (ExecutionException e) {
        throw new RuntimeException("Couldn't start thread " + i, e);
      }

      // Register gauges
      statsLogger.registerGauge(
          String.format("%s-queue-%d", name, idx),
          new Gauge<Number>() {
            @Override
            public Number getDefaultValue() {
              return 0;
            }

            @Override
            public Number getSample() {
              return queues[idx].size();
            }
          });
      statsLogger.registerGauge(
          String.format("%s-completed-tasks-%d", name, idx),
          new Gauge<Number>() {
            @Override
            public Number getDefaultValue() {
              return 0;
            }

            @Override
            public Number getSample() {
              return threads[idx].getCompletedTaskCount();
            }
          });
      statsLogger.registerGauge(
          String.format("%s-total-tasks-%d", name, idx),
          new Gauge<Number>() {
            @Override
            public Number getDefaultValue() {
              return 0;
            }

            @Override
            public Number getSample() {
              return threads[idx].getTaskCount();
            }
          });
    }

    // Stats
    this.taskExecutionStats = statsLogger.scope(name).getOpStatsLogger("task_execution");
    this.taskPendingStats = statsLogger.scope(name).getOpStatsLogger("task_queued");
    this.traceTaskExecution = traceTaskExecution;
  }