Exemple #1
0
  protected Scanner getScanner(StringBuilder sb)
      throws AccumuloException, AccumuloSecurityException {
    AccumuloConfiguration conf = Monitor.getSystemConfiguration();
    String principal = conf.get(Property.TRACE_USER);
    AuthenticationToken at;
    Map<String, String> loginMap =
        conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
    if (loginMap.isEmpty()) {
      Property p = Property.TRACE_PASSWORD;
      at = new PasswordToken(conf.get(p).getBytes(StandardCharsets.UTF_8));
    } else {
      Properties props = new Properties();
      int prefixLength = Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey().length();
      for (Entry<String, String> entry : loginMap.entrySet()) {
        props.put(entry.getKey().substring(prefixLength), entry.getValue());
      }

      AuthenticationToken token =
          Property.createInstanceFromPropertyName(
              conf, Property.TRACE_TOKEN_TYPE, AuthenticationToken.class, new PasswordToken());
      token.init(props);
      at = token;
    }

    String table = conf.get(Property.TRACE_TABLE);
    try {
      Connector conn = HdfsZooInstance.getInstance().getConnector(principal, at);
      if (!conn.tableOperations().exists(table)) {
        return new NullScanner();
      }
      Scanner scanner =
          conn.createScanner(table, conn.securityOperations().getUserAuthorizations(principal));
      return scanner;
    } catch (AccumuloSecurityException ex) {
      sb.append(
          "<h2>Unable to read trace table: check trace username and password configuration.</h2>\n");
      return null;
    } catch (TableNotFoundException ex) {
      return new NullScanner();
    }
  }
  public TabletServerResourceManager(TabletServer tserver, VolumeManager fs) {
    this.tserver = tserver;
    this.conf = tserver.getServerConfigurationFactory();
    this.fs = fs;
    final AccumuloConfiguration acuConf = conf.getConfiguration();

    long maxMemory = acuConf.getMemoryInBytes(Property.TSERV_MAXMEM);
    boolean usingNativeMap =
        acuConf.getBoolean(Property.TSERV_NATIVEMAP_ENABLED) && NativeMap.isLoaded();

    long blockSize = acuConf.getMemoryInBytes(Property.TSERV_DEFAULT_BLOCKSIZE);
    long dCacheSize = acuConf.getMemoryInBytes(Property.TSERV_DATACACHE_SIZE);
    long iCacheSize = acuConf.getMemoryInBytes(Property.TSERV_INDEXCACHE_SIZE);
    long totalQueueSize = acuConf.getMemoryInBytes(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX);

    _iCache = new LruBlockCache(iCacheSize, blockSize);
    _dCache = new LruBlockCache(dCacheSize, blockSize);

    Runtime runtime = Runtime.getRuntime();
    if (usingNativeMap) {
      // Still check block cache sizes when using native maps.
      if (dCacheSize + iCacheSize + totalQueueSize > runtime.maxMemory()) {
        throw new IllegalArgumentException(
            String.format(
                "Block cache sizes %,d and mutation queue size %,d is too large for this JVM configuration %,d",
                dCacheSize + iCacheSize, totalQueueSize, runtime.maxMemory()));
      }
    } else if (maxMemory + dCacheSize + iCacheSize + totalQueueSize > runtime.maxMemory()) {
      throw new IllegalArgumentException(
          String.format(
              "Maximum tablet server map memory %,d block cache sizes %,d and mutation queue size %,d is too large for this JVM configuration %,d",
              maxMemory, dCacheSize + iCacheSize, totalQueueSize, runtime.maxMemory()));
    }
    runtime.gc();

    // totalMemory - freeMemory = memory in use
    // maxMemory - memory in use = max available memory
    if (!usingNativeMap
        && maxMemory > runtime.maxMemory() - (runtime.totalMemory() - runtime.freeMemory())) {
      log.warn("In-memory map may not fit into local memory space.");
    }

    minorCompactionThreadPool = createEs(Property.TSERV_MINC_MAXCONCURRENT, "minor compactor");

    // make this thread pool have a priority queue... and execute tablets with the most
    // files first!
    majorCompactionThreadPool =
        createEs(
            Property.TSERV_MAJC_MAXCONCURRENT,
            "major compactor",
            new CompactionQueue().asBlockingQueueOfRunnable());
    rootMajorCompactionThreadPool = createEs(0, 1, 300, "md root major compactor");
    defaultMajorCompactionThreadPool = createEs(0, 1, 300, "md major compactor");

    splitThreadPool = createEs(1, "splitter");
    defaultSplitThreadPool = createEs(0, 1, 60, "md splitter");

    defaultMigrationPool = createEs(0, 1, 60, "metadata tablet migration");
    migrationPool = createEs(Property.TSERV_MIGRATE_MAXCONCURRENT, "tablet migration");

    // not sure if concurrent assignments can run safely... even if they could there is probably no
    // benefit at startup because
    // individual tablet servers are already running assignments concurrently... having each
    // individual tablet server run
    // concurrent assignments would put more load on the metadata table at startup
    assignmentPool = createEs(Property.TSERV_ASSIGNMENT_MAXCONCURRENT, "tablet assignment");

    assignMetaDataPool = createEs(0, 1, 60, "metadata tablet assignment");

    activeAssignments = new ConcurrentHashMap<KeyExtent, RunnableStartedAt>();

    readAheadThreadPool = createEs(Property.TSERV_READ_AHEAD_MAXCONCURRENT, "tablet read ahead");
    defaultReadAheadThreadPool =
        createEs(Property.TSERV_METADATA_READ_AHEAD_MAXCONCURRENT, "metadata tablets read ahead");

    int maxOpenFiles = acuConf.getCount(Property.TSERV_SCAN_MAX_OPENFILES);

    fileManager = new FileManager(tserver, fs, maxOpenFiles, _dCache, _iCache);

    memoryManager =
        Property.createInstanceFromPropertyName(
            acuConf, Property.TSERV_MEM_MGMT, MemoryManager.class, new LargestFirstMemoryManager());
    memoryManager.init(tserver.getServerConfigurationFactory());
    memMgmt = new MemoryManagementFramework();
    memMgmt.startThreads();

    SimpleTimer timer = SimpleTimer.getInstance(tserver.getConfiguration());

    // We can use the same map for both metadata and normal assignments since the keyspace (extent)
    // is guaranteed to be unique. Schedule the task once, the task will reschedule itself.
    timer.schedule(new AssignmentWatcher(acuConf, activeAssignments, timer), 5000);
  }