public void run() {

    // don't try to run slop pusher job when rebalancing
    if (metadataStore
        .getServerState()
        .equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
      logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
      return;
    }

    boolean terminatedEarly = false;
    Date startTime = new Date();
    logger.info("Started streaming slop pusher job at " + startTime);

    SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
    ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;

    if (adminClient == null) {
      adminClient =
          new AdminClient(
              cluster,
              new AdminClientConfig()
                  .setMaxThreads(cluster.getNumberOfNodes())
                  .setMaxConnectionsPerNode(1));
    }

    if (voldemortConfig.getSlopZonesDownToTerminate() > 0) {
      // Populating the zone mapping for early termination
      zoneMapping.clear();
      for (Node n : cluster.getNodes()) {
        if (failureDetector.isAvailable(n)) {
          Set<Integer> nodes = zoneMapping.get(n.getZoneId());
          if (nodes == null) {
            nodes = Sets.newHashSet();
            zoneMapping.put(n.getZoneId(), nodes);
          }
          nodes.add(n.getId());
        }
      }

      // Check how many zones are down
      int zonesDown = 0;
      for (Zone zone : cluster.getZones()) {
        if (zoneMapping.get(zone.getId()) == null || zoneMapping.get(zone.getId()).size() == 0)
          zonesDown++;
      }

      // Terminate early
      if (voldemortConfig.getSlopZonesDownToTerminate() <= zoneMapping.size()
          && zonesDown >= voldemortConfig.getSlopZonesDownToTerminate()) {
        logger.info(
            "Completed streaming slop pusher job at "
                + startTime
                + " early because "
                + zonesDown
                + " zones are down");
        stopAdminClient();
        return;
      }
    }

    // Clearing the statistics
    AtomicLong attemptedPushes = new AtomicLong(0);
    for (Node node : cluster.getNodes()) {
      attemptedByNode.put(node.getId(), 0L);
      succeededByNode.put(node.getId(), 0L);
    }

    acquireRepairPermit();
    try {
      StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
      iterator = slopStore.entries();

      while (iterator.hasNext()) {
        Pair<ByteArray, Versioned<Slop>> keyAndVal;
        try {
          keyAndVal = iterator.next();
          Versioned<Slop> versioned = keyAndVal.getSecond();

          // Retrieve the node
          int nodeId = versioned.getValue().getNodeId();
          Node node = cluster.getNodeById(nodeId);

          attemptedPushes.incrementAndGet();
          Long attempted = attemptedByNode.get(nodeId);
          attemptedByNode.put(nodeId, attempted + 1L);
          if (attemptedPushes.get() % 10000 == 0)
            logger.info("Attempted pushing " + attemptedPushes + " slops");

          if (logger.isTraceEnabled())
            logger.trace(
                "Pushing slop for "
                    + versioned.getValue().getNodeId()
                    + " and store  "
                    + versioned.getValue().getStoreName());

          if (failureDetector.isAvailable(node)) {
            SynchronousQueue<Versioned<Slop>> slopQueue = slopQueues.get(nodeId);
            if (slopQueue == null) {
              // No previous slop queue, add one
              slopQueue = new SynchronousQueue<Versioned<Slop>>();
              slopQueues.put(nodeId, slopQueue);
              consumerResults.add(
                  consumerExecutor.submit(new SlopConsumer(nodeId, slopQueue, slopStorageEngine)));
            }
            boolean offered =
                slopQueue.offer(
                    versioned, voldemortConfig.getClientRoutingTimeoutMs(), TimeUnit.MILLISECONDS);
            if (!offered) {
              if (logger.isDebugEnabled())
                logger.debug(
                    "No consumer appeared for slop in "
                        + voldemortConfig.getClientConnectionTimeoutMs()
                        + " ms");
            }
            readThrottler.maybeThrottle(nBytesRead(keyAndVal));
          } else {
            logger.trace(node + " declared down, won't push slop");
          }
        } catch (RejectedExecutionException e) {
          throw new VoldemortException("Ran out of threads in executor", e);
        }
      }

    } catch (InterruptedException e) {
      logger.warn("Interrupted exception", e);
      terminatedEarly = true;
    } catch (Exception e) {
      logger.error(e, e);
      terminatedEarly = true;
    } finally {
      try {
        if (iterator != null) iterator.close();
      } catch (Exception e) {
        logger.warn("Failed to close iterator cleanly as database might be closed", e);
      }

      // Adding the poison pill
      for (SynchronousQueue<Versioned<Slop>> slopQueue : slopQueues.values()) {
        try {
          slopQueue.put(END);
        } catch (InterruptedException e) {
          logger.warn("Error putting poison pill", e);
        }
      }

      for (Future result : consumerResults) {
        try {
          result.get();
        } catch (Exception e) {
          logger.warn("Exception in consumer", e);
        }
      }

      // Only if exception didn't take place do we update the counts
      if (!terminatedEarly) {
        Map<Integer, Long> outstanding =
            Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
        for (int nodeId : succeededByNode.keySet()) {
          logger.info(
              "Slops to node "
                  + nodeId
                  + " - Succeeded - "
                  + succeededByNode.get(nodeId)
                  + " - Attempted - "
                  + attemptedByNode.get(nodeId));
          outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
        }
        slopStorageEngine.resetStats(outstanding);
        logger.info("Completed streaming slop pusher job which started at " + startTime);
      } else {
        for (int nodeId : succeededByNode.keySet()) {
          logger.info(
              "Slops to node "
                  + nodeId
                  + " - Succeeded - "
                  + succeededByNode.get(nodeId)
                  + " - Attempted - "
                  + attemptedByNode.get(nodeId));
        }
        logger.info("Completed early streaming slop pusher job which started at " + startTime);
      }

      // Shut down admin client as not to waste connections
      consumerResults.clear();
      slopQueues.clear();
      stopAdminClient();
      this.repairPermits.release();
    }
  }
Esempio n. 2
0
  /**
   * Create a PipelineRoutedStore
   *
   * @param innerStores The mapping of node to client
   * @param nonblockingStores
   * @param slopStores The stores for hints
   * @param nonblockingSlopStores
   * @param cluster Cluster definition
   * @param storeDef Store definition
   */
  public PipelineRoutedStore(
      Map<Integer, Store<ByteArray, byte[], byte[]>> innerStores,
      Map<Integer, NonblockingStore> nonblockingStores,
      Map<Integer, Store<ByteArray, Slop, byte[]>> slopStores,
      Map<Integer, NonblockingStore> nonblockingSlopStores,
      Cluster cluster,
      StoreDefinition storeDef,
      FailureDetector failureDetector,
      boolean repairReads,
      TimeoutConfig timeoutConfig,
      int clientZoneId,
      boolean isJmxEnabled,
      String identifierString,
      ZoneAffinity zoneAffinity) {
    super(
        storeDef.getName(),
        innerStores,
        cluster,
        storeDef,
        repairReads,
        timeoutConfig,
        failureDetector,
        SystemTime.INSTANCE);
    if (zoneAffinity != null
        && storeDef.getZoneCountReads() != null
        && storeDef.getZoneCountReads() > 0) {
      if (zoneAffinity.isGetOpZoneAffinityEnabled()) {
        throw new IllegalArgumentException(
            "storeDef.getZoneCountReads() is non-zero while zoneAffinityGet is enabled");
      }
      if (zoneAffinity.isGetAllOpZoneAffinityEnabled()) {
        throw new IllegalArgumentException(
            "storeDef.getZoneCountReads() is non-zero while zoneAffinityGetAll is enabled");
      }
    }
    this.nonblockingSlopStores = nonblockingSlopStores;
    if (clientZoneId == Zone.UNSET_ZONE_ID) {
      Collection<Zone> availableZones = cluster.getZones();
      this.clientZone = availableZones.iterator().next();
      if (availableZones.size() > 1) {
        String format =
            "Client Zone is not specified. Default to Zone %d. The servers could be in a remote zone";
        logger.warn(String.format(format, this.clientZone.getId()));
      } else {
        if (logger.isDebugEnabled())
          logger.debug(
              String.format(
                  "Client Zone is not specified. Default to Zone %d", this.clientZone.getId()));
      }
    } else {
      this.clientZone = cluster.getZoneById(clientZoneId);
    }
    this.nonblockingStores = new ConcurrentHashMap<Integer, NonblockingStore>(nonblockingStores);
    this.slopStores = slopStores;
    if (storeDef.getRoutingStrategyType().compareTo(RoutingStrategyType.ZONE_STRATEGY) == 0) {
      zoneRoutingEnabled = true;
    } else {
      zoneRoutingEnabled = false;
    }
    if (storeDef.hasHintedHandoffStrategyType()) {
      HintedHandoffStrategyFactory factory =
          new HintedHandoffStrategyFactory(zoneRoutingEnabled, clientZone.getId());
      this.handoffStrategy = factory.updateHintedHandoffStrategy(storeDef, cluster);
    } else {
      this.handoffStrategy = null;
    }

    this.jmxEnabled = isJmxEnabled;
    this.identifierString = identifierString;
    if (this.jmxEnabled) {
      stats = new PipelineRoutedStats();
      JmxUtils.registerMbean(
          stats,
          JmxUtils.createObjectName(
              JmxUtils.getPackageName(stats.getClass()), getName() + identifierString));
    }
    if (zoneAffinity != null) {
      this.zoneAffinity = zoneAffinity;
    } else {
      this.zoneAffinity = new ZoneAffinity();
    }
  }