Example #1
0
 private static GCStatus fetchGcStatus() {
   GCStatus result = null;
   HostAndPort address = null;
   try {
     // Read the gc location from its lock
     ZooReaderWriter zk = ZooReaderWriter.getInstance();
     String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
     List<String> locks = zk.getChildren(path, null);
     if (locks != null && locks.size() > 0) {
       Collections.sort(locks);
       address =
           new ServerServices(
                   new String(zk.getData(path + "/" + locks.get(0), null), StandardCharsets.UTF_8))
               .getAddress(Service.GC_CLIENT);
       GCMonitorService.Client client =
           ThriftUtil.getClient(
               new GCMonitorService.Client.Factory(), address, config.getConfiguration());
       try {
         result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
       } finally {
         ThriftUtil.returnClient(client);
       }
     }
   } catch (Exception ex) {
     log.warn("Unable to contact the garbage collector at " + address, ex);
   }
   return result;
 }
Example #2
0
 public static void removeUnusedWALEntries(
     KeyExtent extent, List<LogEntry> logEntries, ZooLock zooLock) {
   if (extent.isRootTablet()) {
     for (LogEntry entry : logEntries) {
       String root = getZookeeperLogLocation();
       while (true) {
         try {
           IZooReaderWriter zoo = ZooReaderWriter.getInstance();
           if (zoo.isLockHeld(zooLock.getLockID()))
             zoo.recursiveDelete(root + "/" + entry.filename, NodeMissingPolicy.SKIP);
           break;
         } catch (Exception e) {
           log.error(e, e);
         }
         UtilWaitThread.sleep(1000);
       }
     }
   } else {
     Mutation m = new Mutation(extent.getMetadataEntry());
     for (LogEntry entry : logEntries) {
       m.putDelete(LogColumnFamily.NAME, new Text(entry.toString()));
     }
     update(SystemCredentials.get(), zooLock, m, extent);
   }
 }
Example #3
0
  public static void removeBulkLoadInProgressFlag(String path) {

    Mutation m = new Mutation(MetadataSchema.BlipSection.getRowPrefix() + path);
    m.putDelete(EMPTY_TEXT, EMPTY_TEXT);

    // new KeyExtent is only added to force update to write to the metadata table, not the root
    // table
    // because bulk loads aren't supported to the metadata table
    update(SystemCredentials.get(), m, new KeyExtent(new Text("anythingNotMetadata"), null, null));
  }
  @Before
  public void setUp() {
    volMgr = createMock(VolumeManager.class);
    instance = createMock(Instance.class);
    SiteConfiguration siteConfig = EasyMock.createMock(SiteConfiguration.class);
    expect(instance.getInstanceID()).andReturn("mock").anyTimes();
    expect(instance.getZooKeepers()).andReturn("localhost").anyTimes();
    expect(instance.getZooKeepersSessionTimeOut()).andReturn(30000).anyTimes();

    opts = new Opts();
    systemConfig = createSystemConfig();
    ServerConfigurationFactory factory = createMock(ServerConfigurationFactory.class);
    expect(factory.getInstance()).andReturn(instance).anyTimes();
    expect(factory.getConfiguration()).andReturn(systemConfig).anyTimes();
    expect(factory.getSiteConfiguration()).andReturn(siteConfig).anyTimes();

    // Just make the SiteConfiguration delegate to our AccumuloConfiguration
    // Presently, we only need get(Property) and iterator().
    EasyMock.expect(siteConfig.get(EasyMock.anyObject(Property.class)))
        .andAnswer(
            new IAnswer<String>() {
              @Override
              public String answer() {
                Object[] args = EasyMock.getCurrentArguments();
                return systemConfig.get((Property) args[0]);
              }
            })
        .anyTimes();
    EasyMock.expect(siteConfig.getBoolean(EasyMock.anyObject(Property.class)))
        .andAnswer(
            new IAnswer<Boolean>() {
              @Override
              public Boolean answer() {
                Object[] args = EasyMock.getCurrentArguments();
                return systemConfig.getBoolean((Property) args[0]);
              }
            })
        .anyTimes();

    EasyMock.expect(siteConfig.iterator())
        .andAnswer(
            new IAnswer<Iterator<Entry<String, String>>>() {
              @Override
              public Iterator<Entry<String, String>> answer() {
                return systemConfig.iterator();
              }
            })
        .anyTimes();

    replay(instance, factory, siteConfig);

    credentials = SystemCredentials.get(instance);
    gc = new SimpleGarbageCollector(opts, volMgr, factory);
  }
  @Override
  protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb)
      throws Exception {
    String tserverAddress = req.getParameter("s");

    // Check to make sure tserver is a known address
    boolean tserverExists = false;
    if (tserverAddress != null && tserverAddress.isEmpty() == false) {
      for (TabletServerStatus ts : Monitor.getMmi().getTServerInfo()) {
        if (tserverAddress.equals(ts.getName())) {
          tserverExists = true;
          break;
        }
      }
    }

    if (tserverAddress == null || tserverAddress.isEmpty() || tserverExists == false) {
      doBadTserverList(req, sb);

      doDeadTserverList(req, sb);

      ArrayList<TabletServerStatus> tservers = new ArrayList<TabletServerStatus>();
      if (Monitor.getMmi() != null) tservers.addAll(Monitor.getMmi().tServerInfo);

      Table tServerList = new Table("tservers", "Tablet&nbsp;Servers");
      tServerList.setSubCaption(
          "Click on the <span style='color: #0000ff;'>server address</span> to view detailed performance statistics for that server.");

      doTserverList(req, sb, tservers, null, tServerList);
      return;
    }

    double totalElapsedForAll = 0;
    double splitStdDev = 0;
    double minorStdDev = 0;
    double minorQueueStdDev = 0;
    double majorStdDev = 0;
    double majorQueueStdDev = 0;
    double currentMinorAvg = 0;
    double currentMajorAvg = 0;
    double currentMinorStdDev = 0;
    double currentMajorStdDev = 0;
    TabletStats total =
        new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);

    InetSocketAddress address = AddressUtil.parseAddress(tserverAddress, -1);
    TabletStats historical =
        new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
    List<TabletStats> tsStats = new ArrayList<TabletStats>();
    try {
      TabletClientService.Client client =
          ThriftUtil.getClient(
              new TabletClientService.Client.Factory(), address, Monitor.getSystemConfiguration());
      try {
        for (String tableId : Monitor.getMmi().tableMap.keySet()) {
          tsStats.addAll(
              client.getTabletStats(
                  Tracer.traceInfo(),
                  SystemCredentials.get().toThrift(Monitor.getInstance()),
                  tableId));
        }
        historical =
            client.getHistoricalStats(
                Tracer.traceInfo(), SystemCredentials.get().toThrift(Monitor.getInstance()));
      } finally {
        ThriftUtil.returnClient(client);
      }
    } catch (Exception e) {
      banner(sb, "error", "No Such Tablet ServerAvailable");
      log.error(e, e);
      return;
    }

    Table perTabletResults = new Table("perTabletResults", "Detailed&nbsp;Current&nbsp;Operations");
    perTabletResults.setSubCaption("Per-tablet&nbsp;Details");
    perTabletResults.addSortableColumn("Table", new TableLinkType(), null);
    perTabletResults.addSortableColumn("Tablet");
    perTabletResults.addSortableColumn("Entries", new NumberType<Long>(), null);
    perTabletResults.addSortableColumn("Ingest", new NumberType<Long>(), null);
    perTabletResults.addSortableColumn("Query", new NumberType<Long>(), null);
    perTabletResults.addSortableColumn("Minor&nbsp;Avg", new SecondType(), null);
    perTabletResults.addSortableColumn("Minor&nbsp;Std&nbsp;Dev", new SecondType(), null);
    perTabletResults.addSortableColumn("Minor&nbsp;Avg&nbsp;e/s", new NumberType<Double>(), null);
    perTabletResults.addSortableColumn("Major&nbsp;Avg", new SecondType(), null);
    perTabletResults.addSortableColumn("Major&nbsp;Std&nbsp;Dev", new SecondType(), null);
    perTabletResults.addSortableColumn("Major&nbsp;Avg&nbsp;e/s", new NumberType<Double>(), null);

    for (TabletStats info : tsStats) {
      if (info.extent == null) {
        historical = info;
        continue;
      }
      total.numEntries += info.numEntries;
      TabletStatsKeeper.update(total.minors, info.minors);
      TabletStatsKeeper.update(total.majors, info.majors);

      KeyExtent extent = new KeyExtent(info.extent);
      String tableId = extent.getTableId().toString();
      MessageDigest digester = MessageDigest.getInstance("MD5");
      if (extent.getEndRow() != null && extent.getEndRow().getLength() > 0) {
        digester.update(extent.getEndRow().getBytes(), 0, extent.getEndRow().getLength());
      }
      String obscuredExtent = new String(Base64.encodeBase64(digester.digest()));
      String displayExtent = String.format("<code>[%s]</code>", obscuredExtent);

      TableRow row = perTabletResults.prepareRow();
      row.add(tableId);
      row.add(displayExtent);
      row.add(info.numEntries);
      row.add(info.ingestRate);
      row.add(info.queryRate);
      row.add(info.minors.num != 0 ? info.minors.elapsed / info.minors.num : null);
      row.add(stddev(info.minors.elapsed, info.minors.num, info.minors.sumDev));
      row.add(info.minors.elapsed != 0 ? info.minors.count / info.minors.elapsed : null);
      row.add(info.majors.num != 0 ? info.majors.elapsed / info.majors.num : null);
      row.add(stddev(info.majors.elapsed, info.majors.num, info.majors.sumDev));
      row.add(info.majors.elapsed != 0 ? info.majors.count / info.majors.elapsed : null);
      perTabletResults.addRow(row);
    }

    // Calculate current averages oldServer adding in historical data
    if (total.minors.num != 0) currentMinorAvg = (long) (total.minors.elapsed / total.minors.num);
    if (total.minors.elapsed != 0 && total.minors.num != 0)
      currentMinorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
    if (total.majors.num != 0) currentMajorAvg = total.majors.elapsed / total.majors.num;
    if (total.majors.elapsed != 0
        && total.majors.num != 0
        && total.majors.elapsed > total.majors.num)
      currentMajorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);

    // After these += operations, these variables are now total for current
    // tablets and historical tablets
    TabletStatsKeeper.update(total.minors, historical.minors);
    TabletStatsKeeper.update(total.majors, historical.majors);
    totalElapsedForAll += total.majors.elapsed + historical.splits.elapsed + total.minors.elapsed;

    minorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
    minorQueueStdDev = stddev(total.minors.queueTime, total.minors.num, total.minors.queueSumDev);
    majorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
    majorQueueStdDev = stddev(total.majors.queueTime, total.majors.num, total.majors.queueSumDev);
    splitStdDev =
        stddev(historical.splits.num, historical.splits.elapsed, historical.splits.sumDev);

    doDetailTable(req, sb, address, tsStats.size(), total, historical);
    doAllTimeTable(
        req,
        sb,
        total,
        historical,
        majorQueueStdDev,
        minorQueueStdDev,
        totalElapsedForAll,
        splitStdDev,
        majorStdDev,
        minorStdDev);
    doCurrentTabletOps(
        req, sb, currentMinorAvg, currentMinorStdDev, currentMajorAvg, currentMajorStdDev);
    perTabletResults.generate(req, sb);
  }
Example #6
0
  public static void fetchData() {
    double totalIngestRate = 0.;
    double totalIngestByteRate = 0.;
    double totalQueryRate = 0.;
    double totalQueryByteRate = 0.;
    double totalScanRate = 0.;
    long totalEntries = 0;
    int totalTabletCount = 0;
    int onlineTabletCount = 0;
    long totalHoldTime = 0;
    long totalLookups = 0;
    boolean retry = true;

    // only recalc every so often
    long currentTime = System.currentTimeMillis();
    if (currentTime - lastRecalc < REFRESH_TIME * 1000) return;

    synchronized (Monitor.class) {
      if (fetching) return;
      fetching = true;
    }

    try {
      while (retry) {
        MasterClientService.Iface client = null;
        try {
          client = MasterClient.getConnection(HdfsZooInstance.getInstance());
          if (client != null) {
            mmi =
                client.getMasterStats(
                    Tracer.traceInfo(),
                    SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()));
            retry = false;
          } else {
            mmi = null;
          }
          Monitor.gcStatus = fetchGcStatus();
        } catch (Exception e) {
          mmi = null;
          log.info("Error fetching stats: " + e);
        } finally {
          if (client != null) {
            MasterClient.close(client);
          }
        }
        if (mmi == null) UtilWaitThread.sleep(1000);
      }
      if (mmi != null) {
        int majorCompactions = 0;
        int minorCompactions = 0;

        lookupRateTracker.startingUpdates();
        indexCacheHitTracker.startingUpdates();
        indexCacheRequestTracker.startingUpdates();
        dataCacheHitTracker.startingUpdates();
        dataCacheRequestTracker.startingUpdates();

        for (TabletServerStatus server : mmi.tServerInfo) {
          TableInfo summary = TableInfoUtil.summarizeTableStats(server);
          totalIngestRate += summary.ingestRate;
          totalIngestByteRate += summary.ingestByteRate;
          totalQueryRate += summary.queryRate;
          totalScanRate += summary.scanRate;
          totalQueryByteRate += summary.queryByteRate;
          totalEntries += summary.recs;
          totalHoldTime += server.holdTime;
          totalLookups += server.lookups;
          majorCompactions += summary.majors.running;
          minorCompactions += summary.minors.running;
          lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
          indexCacheHitTracker.updateTabletServer(
              server.name, server.lastContact, server.indexCacheHits);
          indexCacheRequestTracker.updateTabletServer(
              server.name, server.lastContact, server.indexCacheRequest);
          dataCacheHitTracker.updateTabletServer(
              server.name, server.lastContact, server.dataCacheHits);
          dataCacheRequestTracker.updateTabletServer(
              server.name, server.lastContact, server.dataCacheRequest);
        }

        lookupRateTracker.finishedUpdating();
        indexCacheHitTracker.finishedUpdating();
        indexCacheRequestTracker.finishedUpdating();
        dataCacheHitTracker.finishedUpdating();
        dataCacheRequestTracker.finishedUpdating();

        int totalTables = 0;
        for (TableInfo tInfo : mmi.tableMap.values()) {
          totalTabletCount += tInfo.tablets;
          onlineTabletCount += tInfo.onlineTablets;
          totalTables++;
        }
        Monitor.totalIngestRate = totalIngestRate;
        Monitor.totalTables = totalTables;
        totalIngestByteRate = totalIngestByteRate / 1000000.0;
        Monitor.totalIngestByteRate = totalIngestByteRate;
        Monitor.totalQueryRate = totalQueryRate;
        Monitor.totalScanRate = totalScanRate;
        totalQueryByteRate = totalQueryByteRate / 1000000.0;
        Monitor.totalQueryByteRate = totalQueryByteRate;
        Monitor.totalEntries = totalEntries;
        Monitor.totalTabletCount = totalTabletCount;
        Monitor.onlineTabletCount = onlineTabletCount;
        Monitor.totalHoldTime = totalHoldTime;
        Monitor.totalLookups = totalLookups;

        ingestRateOverTime.add(new Pair<Long, Double>(currentTime, totalIngestRate));
        ingestByteRateOverTime.add(new Pair<Long, Double>(currentTime, totalIngestByteRate));

        double totalLoad = 0.;
        for (TabletServerStatus status : mmi.tServerInfo) {
          if (status != null) totalLoad += status.osLoad;
        }
        loadOverTime.add(new Pair<Long, Double>(currentTime, totalLoad));

        minorCompactionsOverTime.add(new Pair<Long, Integer>(currentTime, minorCompactions));
        majorCompactionsOverTime.add(new Pair<Long, Integer>(currentTime, majorCompactions));

        lookupsOverTime.add(new Pair<Long, Double>(currentTime, lookupRateTracker.calculateRate()));

        queryRateOverTime.add(new Pair<Long, Integer>(currentTime, (int) totalQueryRate));
        queryByteRateOverTime.add(new Pair<Long, Double>(currentTime, totalQueryByteRate));

        scanRateOverTime.add(new Pair<Long, Integer>(currentTime, (int) totalScanRate));

        calcCacheHitRate(
            indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
        calcCacheHitRate(
            dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
      }
      try {
        Monitor.problemSummary = ProblemReports.getInstance().summarize();
        Monitor.problemException = null;
      } catch (Exception e) {
        log.info("Failed to obtain problem reports ", e);
        Monitor.problemSummary = Collections.emptyMap();
        Monitor.problemException = e;
      }

    } finally {
      synchronized (Monitor.class) {
        fetching = false;
        lastRecalc = currentTime;
      }
    }
  }
Example #7
0
 public static void chopped(KeyExtent extent, ZooLock zooLock) {
   Mutation m = new Mutation(extent.getMetadataEntry());
   ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("chopped".getBytes()));
   update(SystemCredentials.get(), zooLock, m, extent);
 }
Example #8
0
  public static void cloneTable(
      Instance instance, String srcTableId, String tableId, VolumeManager volumeManager)
      throws Exception {

    Connector conn =
        instance.getConnector(
            SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
    BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());

    while (true) {

      try {
        initializeClone(srcTableId, tableId, conn, bw);

        // the following loop looks changes in the file that occurred during the copy.. if files
        // were dereferenced then they could have been GCed

        while (true) {
          int rewrites = checkClone(srcTableId, tableId, conn, bw);

          if (rewrites == 0) break;
        }

        bw.flush();
        break;

      } catch (TabletIterator.TabletDeletedException tde) {
        // tablets were merged in the src table
        bw.flush();

        // delete what we have cloned and try again
        deleteTable(tableId, false, SystemCredentials.get(), null);

        log.debug(
            "Tablets merged in table " + srcTableId + " while attempting to clone, trying again");

        UtilWaitThread.sleep(100);
      }
    }

    // delete the clone markers and create directory entries
    Scanner mscanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
    mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
    mscanner.fetchColumnFamily(ClonedColumnFamily.NAME);

    int dirCount = 0;

    for (Entry<Key, Value> entry : mscanner) {
      Key k = entry.getKey();
      Mutation m = new Mutation(k.getRow());
      m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
      String dir =
          volumeManager.choose(ServerConstants.getTablesDirs())
              + "/"
              + tableId
              + new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, "/c-".getBytes()));
      TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes()));
      bw.addMutation(m);
    }

    bw.close();
  }
Example #9
0
 public static void addDeleteEntry(String tableId, String path) throws IOException {
   update(
       SystemCredentials.get(),
       createDeleteMutation(tableId, path),
       new KeyExtent(new Text(tableId), null, null));
 }