Ejemplo n.º 1
0
  @Override
  public CompactionStats call() {
    final String outputFileName = getOutputFile();
    log.debug("Begin minor compaction " + outputFileName + " " + getExtent());

    // output to new MapFile with a temporary name
    int sleepTime = 100;
    double growthFactor = 4;
    int maxSleepTime = 1000 * 60 * 3; // 3 minutes
    boolean reportedProblem = false;

    runningCompactions.add(this);
    try {
      do {
        try {
          CompactionStats ret = super.call();

          // log.debug(String.format("MinC %,d recs in | %,d recs out | %,d recs/sec | %6.3f secs |
          // %,d bytes ",map.size(), entriesCompacted,
          // (int)(map.size()/((t2 - t1)/1000.0)), (t2 - t1)/1000.0, estimatedSizeInBytes()));

          if (reportedProblem) {
            ProblemReports.getInstance(tabletServer)
                .deleteProblemReport(
                    getExtent().getTableId(), ProblemType.FILE_WRITE, outputFileName);
          }

          return ret;
        } catch (IOException e) {
          log.warn("MinC failed ({}) to create {} retrying ...", e.getMessage(), outputFileName);
          ProblemReports.getInstance(tabletServer)
              .report(
                  new ProblemReport(
                      getExtent().getTableId(), ProblemType.FILE_WRITE, outputFileName, e));
          reportedProblem = true;
        } catch (RuntimeException e) {
          // if this is coming from a user iterator, it is possible that the user could change the
          // iterator config and that the
          // minor compaction would succeed
          log.warn("MinC failed ({}) to create {} retrying ...", e.getMessage(), outputFileName, e);
          ProblemReports.getInstance(tabletServer)
              .report(
                  new ProblemReport(
                      getExtent().getTableId(), ProblemType.FILE_WRITE, outputFileName, e));
          reportedProblem = true;
        } catch (CompactionCanceledException e) {
          throw new IllegalStateException(e);
        }

        Random random = new Random();

        int sleep = sleepTime + random.nextInt(sleepTime);
        log.debug("MinC failed sleeping " + sleep + " ms before retrying");
        sleepUninterruptibly(sleep, TimeUnit.MILLISECONDS);
        sleepTime = (int) Math.round(Math.min(maxSleepTime, sleepTime * growthFactor));

        // clean up
        try {
          if (getFileSystem().exists(new Path(outputFileName))) {
            getFileSystem().deleteRecursively(new Path(outputFileName));
          }
        } catch (IOException e) {
          log.warn("Failed to delete failed MinC file {} {}", outputFileName, e.getMessage());
        }

        if (isTableDeleting()) return new CompactionStats(0, 0);

      } while (true);
    } finally {
      thread = null;
      runningCompactions.remove(this);
    }
  }
Ejemplo n.º 2
0
  public static void fetchData() {
    double totalIngestRate = 0.;
    double totalIngestByteRate = 0.;
    double totalQueryRate = 0.;
    double totalQueryByteRate = 0.;
    double totalScanRate = 0.;
    long totalEntries = 0;
    int totalTabletCount = 0;
    int onlineTabletCount = 0;
    long totalHoldTime = 0;
    long totalLookups = 0;
    boolean retry = true;

    // only recalc every so often
    long currentTime = System.currentTimeMillis();
    if (currentTime - lastRecalc < REFRESH_TIME * 1000) return;

    synchronized (Monitor.class) {
      if (fetching) return;
      fetching = true;
    }

    try {
      while (retry) {
        MasterClientService.Iface client = null;
        try {
          client = MasterClient.getConnection(HdfsZooInstance.getInstance());
          if (client != null) {
            mmi =
                client.getMasterStats(
                    Tracer.traceInfo(),
                    SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()));
            retry = false;
          } else {
            mmi = null;
          }
          Monitor.gcStatus = fetchGcStatus();
        } catch (Exception e) {
          mmi = null;
          log.info("Error fetching stats: " + e);
        } finally {
          if (client != null) {
            MasterClient.close(client);
          }
        }
        if (mmi == null) UtilWaitThread.sleep(1000);
      }
      if (mmi != null) {
        int majorCompactions = 0;
        int minorCompactions = 0;

        lookupRateTracker.startingUpdates();
        indexCacheHitTracker.startingUpdates();
        indexCacheRequestTracker.startingUpdates();
        dataCacheHitTracker.startingUpdates();
        dataCacheRequestTracker.startingUpdates();

        for (TabletServerStatus server : mmi.tServerInfo) {
          TableInfo summary = TableInfoUtil.summarizeTableStats(server);
          totalIngestRate += summary.ingestRate;
          totalIngestByteRate += summary.ingestByteRate;
          totalQueryRate += summary.queryRate;
          totalScanRate += summary.scanRate;
          totalQueryByteRate += summary.queryByteRate;
          totalEntries += summary.recs;
          totalHoldTime += server.holdTime;
          totalLookups += server.lookups;
          majorCompactions += summary.majors.running;
          minorCompactions += summary.minors.running;
          lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
          indexCacheHitTracker.updateTabletServer(
              server.name, server.lastContact, server.indexCacheHits);
          indexCacheRequestTracker.updateTabletServer(
              server.name, server.lastContact, server.indexCacheRequest);
          dataCacheHitTracker.updateTabletServer(
              server.name, server.lastContact, server.dataCacheHits);
          dataCacheRequestTracker.updateTabletServer(
              server.name, server.lastContact, server.dataCacheRequest);
        }

        lookupRateTracker.finishedUpdating();
        indexCacheHitTracker.finishedUpdating();
        indexCacheRequestTracker.finishedUpdating();
        dataCacheHitTracker.finishedUpdating();
        dataCacheRequestTracker.finishedUpdating();

        int totalTables = 0;
        for (TableInfo tInfo : mmi.tableMap.values()) {
          totalTabletCount += tInfo.tablets;
          onlineTabletCount += tInfo.onlineTablets;
          totalTables++;
        }
        Monitor.totalIngestRate = totalIngestRate;
        Monitor.totalTables = totalTables;
        totalIngestByteRate = totalIngestByteRate / 1000000.0;
        Monitor.totalIngestByteRate = totalIngestByteRate;
        Monitor.totalQueryRate = totalQueryRate;
        Monitor.totalScanRate = totalScanRate;
        totalQueryByteRate = totalQueryByteRate / 1000000.0;
        Monitor.totalQueryByteRate = totalQueryByteRate;
        Monitor.totalEntries = totalEntries;
        Monitor.totalTabletCount = totalTabletCount;
        Monitor.onlineTabletCount = onlineTabletCount;
        Monitor.totalHoldTime = totalHoldTime;
        Monitor.totalLookups = totalLookups;

        ingestRateOverTime.add(new Pair<Long, Double>(currentTime, totalIngestRate));
        ingestByteRateOverTime.add(new Pair<Long, Double>(currentTime, totalIngestByteRate));

        double totalLoad = 0.;
        for (TabletServerStatus status : mmi.tServerInfo) {
          if (status != null) totalLoad += status.osLoad;
        }
        loadOverTime.add(new Pair<Long, Double>(currentTime, totalLoad));

        minorCompactionsOverTime.add(new Pair<Long, Integer>(currentTime, minorCompactions));
        majorCompactionsOverTime.add(new Pair<Long, Integer>(currentTime, majorCompactions));

        lookupsOverTime.add(new Pair<Long, Double>(currentTime, lookupRateTracker.calculateRate()));

        queryRateOverTime.add(new Pair<Long, Integer>(currentTime, (int) totalQueryRate));
        queryByteRateOverTime.add(new Pair<Long, Double>(currentTime, totalQueryByteRate));

        scanRateOverTime.add(new Pair<Long, Integer>(currentTime, (int) totalScanRate));

        calcCacheHitRate(
            indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
        calcCacheHitRate(
            dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
      }
      try {
        Monitor.problemSummary = ProblemReports.getInstance().summarize();
        Monitor.problemException = null;
      } catch (Exception e) {
        log.info("Failed to obtain problem reports ", e);
        Monitor.problemSummary = Collections.emptyMap();
        Monitor.problemException = e;
      }

    } finally {
      synchronized (Monitor.class) {
        fetching = false;
        lastRecalc = currentTime;
      }
    }
  }