Exemplo n.º 1
0
  @Override
  protected void startInner() {
    registerInternalEngine(metadata, false, "metadata");

    /* Initialize storage configurations */
    for (String configClassName : voldemortConfig.getStorageConfigurations())
      initStorageConfig(configClassName);

    /* Initialize view storage configuration */
    storageConfigs.put(
        ViewStorageConfiguration.TYPE_NAME,
        new ViewStorageConfiguration(voldemortConfig, metadata.getStoreDefList(), storeRepository));

    /* Initialize system stores */
    initSystemStores();

    /* Register slop store */
    if (voldemortConfig.isSlopEnabled()) {

      logger.info("Initializing the slop store using " + voldemortConfig.getSlopStoreType());
      StorageConfiguration config = storageConfigs.get(voldemortConfig.getSlopStoreType());
      if (config == null)
        throw new ConfigurationException(
            "Attempt to open store "
                + SlopStorageEngine.SLOP_STORE_NAME
                + " but "
                + voldemortConfig.getSlopStoreType()
                + " storage engine has not been enabled.");

      // make a dummy store definition object
      StoreDefinition slopStoreDefinition =
          new StoreDefinition(
              SlopStorageEngine.SLOP_STORE_NAME,
              null,
              null,
              null,
              null,
              null,
              null,
              RoutingStrategyType.CONSISTENT_STRATEGY,
              0,
              null,
              0,
              null,
              0,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              0);
      SlopStorageEngine slopEngine =
          new SlopStorageEngine(
              config.getStore(
                  slopStoreDefinition,
                  new RoutingStrategyFactory()
                      .updateRoutingStrategy(slopStoreDefinition, metadata.getCluster())),
              metadata.getCluster());
      registerInternalEngine(slopEngine, false, "slop");
      storeRepository.setSlopStore(slopEngine);

      if (voldemortConfig.isSlopPusherJobEnabled()) {
        // Now initialize the pusher job after some time
        GregorianCalendar cal = new GregorianCalendar();
        cal.add(Calendar.SECOND, (int) (voldemortConfig.getSlopFrequencyMs() / Time.MS_PER_SECOND));
        Date nextRun = cal.getTime();
        logger.info(
            "Initializing slop pusher job type "
                + voldemortConfig.getPusherType()
                + " at "
                + nextRun);

        scheduler.schedule(
            "slop",
            (voldemortConfig.getPusherType().compareTo(BlockingSlopPusherJob.TYPE_NAME) == 0)
                ? new BlockingSlopPusherJob(
                    storeRepository, metadata, failureDetector, voldemortConfig, scanPermitWrapper)
                : new StreamingSlopPusherJob(
                    storeRepository, metadata, failureDetector, voldemortConfig, scanPermitWrapper),
            nextRun,
            voldemortConfig.getSlopFrequencyMs());
      }

      // Create a SlopPurgeJob object and register it
      if (voldemortConfig.isSlopPurgeJobEnabled()) {
        logger.info("Initializing Slop Purge job");
        SlopPurgeJob job =
            new SlopPurgeJob(
                storeRepository,
                metadata,
                scanPermitWrapper,
                voldemortConfig.getSlopPurgeJobMaxKeysScannedPerSec());
        JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
        storeRepository.registerSlopPurgeJob(job);
      }
    }

    // Create a repair job object and register it with Store repository
    if (voldemortConfig.isRepairEnabled()) {
      logger.info("Initializing repair job.");
      RepairJob job =
          new RepairJob(
              storeRepository,
              metadata,
              scanPermitWrapper,
              voldemortConfig.getRepairJobMaxKeysScannedPerSec());
      JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
      storeRepository.registerRepairJob(job);
    }

    // Create a prune job object and register it
    if (voldemortConfig.isPruneJobEnabled()) {
      logger.info("Intializing prune job");
      VersionedPutPruneJob job =
          new VersionedPutPruneJob(
              storeRepository,
              metadata,
              scanPermitWrapper,
              voldemortConfig.getPruneJobMaxKeysScannedPerSec());
      JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
      storeRepository.registerPruneJob(job);
    }

    List<StoreDefinition> storeDefs =
        new ArrayList<StoreDefinition>(this.metadata.getStoreDefList());
    logger.info("Initializing stores:");

    logger.info("Validating schemas:");
    String AVRO_GENERIC_VERSIONED_TYPE_NAME = "avro-generic-versioned";

    for (StoreDefinition storeDef : storeDefs) {
      SerializerDefinition keySerDef = storeDef.getKeySerializer();
      SerializerDefinition valueSerDef = storeDef.getValueSerializer();

      if (keySerDef.getName().equals(AVRO_GENERIC_VERSIONED_TYPE_NAME)) {

        SchemaEvolutionValidator.checkSchemaCompatibility(keySerDef);
      }

      if (valueSerDef.getName().equals(AVRO_GENERIC_VERSIONED_TYPE_NAME)) {

        SchemaEvolutionValidator.checkSchemaCompatibility(valueSerDef);
      }
    }
    // first initialize non-view stores
    for (StoreDefinition def : storeDefs) if (!def.isView()) openStore(def);

    // now that we have all our stores, we can initialize views pointing at
    // those stores
    for (StoreDefinition def : storeDefs) {
      if (def.isView()) openStore(def);
    }

    initializeMetadataVersions(storeDefs);

    // enable aggregate jmx statistics
    if (voldemortConfig.isStatTrackingEnabled())
      if (this.voldemortConfig.isEnableJmxClusterName())
        JmxUtils.registerMbean(
            new StoreStatsJmx(this.storeStats),
            JmxUtils.createObjectName(
                metadata.getCluster().getName() + ".voldemort.store.stats.aggregate",
                "aggregate-perf"));
      else
        JmxUtils.registerMbean(
            new StoreStatsJmx(this.storeStats),
            JmxUtils.createObjectName("voldemort.store.stats.aggregate", "aggregate-perf"));

    logger.info("All stores initialized.");
  }
Exemplo n.º 2
0
  @Override
  protected void startInner() {
    registerInternalEngine(metadata, false, "metadata");

    /* Initialize storage configurations */
    for (String configClassName : voldemortConfig.getStorageConfigurations())
      initStorageConfig(configClassName);

    /* Initialize view storage configuration */
    storageConfigs.put(
        ViewStorageConfiguration.TYPE_NAME,
        new ViewStorageConfiguration(voldemortConfig, metadata.getStoreDefList(), storeRepository));

    /* Initialize system stores */
    initSystemStores();

    /* Register slop store */
    if (voldemortConfig.isSlopEnabled()) {

      logger.info("Initializing the slop store using " + voldemortConfig.getSlopStoreType());
      StorageConfiguration config = storageConfigs.get(voldemortConfig.getSlopStoreType());
      if (config == null)
        throw new ConfigurationException(
            "Attempt to open store "
                + SlopStorageEngine.SLOP_STORE_NAME
                + " but "
                + voldemortConfig.getSlopStoreType()
                + " storage engine has not been enabled.");

      // make a dummy store definition object
      StoreDefinition slopStoreDefinition =
          new StoreDefinition(
              SlopStorageEngine.SLOP_STORE_NAME,
              null,
              null,
              null,
              null,
              null,
              null,
              RoutingStrategyType.CONSISTENT_STRATEGY,
              0,
              null,
              0,
              null,
              0,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              null,
              0);
      SlopStorageEngine slopEngine =
          new SlopStorageEngine(
              config.getStore(
                  slopStoreDefinition,
                  new RoutingStrategyFactory()
                      .updateRoutingStrategy(slopStoreDefinition, metadata.getCluster())),
              metadata.getCluster());
      registerInternalEngine(slopEngine, false, "slop");
      storeRepository.setSlopStore(slopEngine);

      if (voldemortConfig.isSlopPusherJobEnabled()) {
        // Now initialize the pusher job after some time
        GregorianCalendar cal = new GregorianCalendar();
        cal.add(Calendar.SECOND, (int) (voldemortConfig.getSlopFrequencyMs() / Time.MS_PER_SECOND));
        Date nextRun = cal.getTime();
        logger.info(
            "Initializing slop pusher job type "
                + voldemortConfig.getPusherType()
                + " at "
                + nextRun);

        scheduler.schedule(
            "slop",
            (voldemortConfig.getPusherType().compareTo(BlockingSlopPusherJob.TYPE_NAME) == 0)
                ? new BlockingSlopPusherJob(
                    storeRepository, metadata, failureDetector, voldemortConfig, scanPermitWrapper)
                : new StreamingSlopPusherJob(
                    storeRepository,
                    metadata,
                    slopStreamingFailureDetector,
                    voldemortConfig,
                    scanPermitWrapper),
            nextRun,
            voldemortConfig.getSlopFrequencyMs());
      }

      // Create a SlopPurgeJob object and register it
      if (voldemortConfig.isSlopPurgeJobEnabled()) {
        logger.info("Initializing Slop Purge job");
        SlopPurgeJob job =
            new SlopPurgeJob(
                storeRepository,
                metadata,
                scanPermitWrapper,
                voldemortConfig.getSlopPurgeJobMaxKeysScannedPerSec());
        JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
        storeRepository.registerSlopPurgeJob(job);
      }
    }

    // Create a repair job object and register it with Store repository
    if (voldemortConfig.isRepairEnabled()) {
      logger.info("Initializing repair job.");
      RepairJob job =
          new RepairJob(
              storeRepository,
              metadata,
              scanPermitWrapper,
              voldemortConfig.getRepairJobMaxKeysScannedPerSec());
      JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
      storeRepository.registerRepairJob(job);
    }

    // Create a prune job object and register it
    if (voldemortConfig.isPruneJobEnabled()) {
      logger.info("Intializing prune job");
      VersionedPutPruneJob job =
          new VersionedPutPruneJob(
              storeRepository,
              metadata,
              scanPermitWrapper,
              voldemortConfig.getPruneJobMaxKeysScannedPerSec());
      JmxUtils.registerMbean(job, JmxUtils.createObjectName(job.getClass()));
      storeRepository.registerPruneJob(job);
    }

    List<StoreDefinition> storeDefs =
        new ArrayList<StoreDefinition>(this.metadata.getStoreDefList());
    logger.info("Initializing stores:");

    logger.info("Validating schemas:");
    StoreDefinitionUtils.validateSchemasAsNeeded(storeDefs);
    // first initialize non-view stores
    for (StoreDefinition def : storeDefs) if (!def.isView()) openStore(def);

    // now that we have all our stores, we can initialize views pointing at
    // those stores
    for (StoreDefinition def : storeDefs) {
      if (def.isView()) openStore(def);
    }

    initializeMetadataVersions(storeDefs);

    // enable aggregate jmx statistics
    if (voldemortConfig.isStatTrackingEnabled())
      if (this.voldemortConfig.isEnableJmxClusterName())
        JmxUtils.registerMbean(
            new StoreStatsJmx(this.storeStats),
            JmxUtils.createObjectName(
                metadata.getCluster().getName() + ".voldemort.store.stats.aggregate",
                "aggregate-perf"));
      else
        JmxUtils.registerMbean(
            new StoreStatsJmx(this.storeStats),
            JmxUtils.createObjectName("voldemort.store.stats.aggregate", "aggregate-perf"));

    List<StorageEngine> listOfDisabledStores = Lists.newArrayList();
    for (StorageEngine storageEngine : storeRepository.getAllStorageEngines()) {
      try {
        StoreVersionManager storeVersionManager =
            (StoreVersionManager)
                storageEngine.getCapability(StoreCapabilityType.DISABLE_STORE_VERSION);
        if (storeVersionManager.hasAnyDisabledVersion()) {
          listOfDisabledStores.add(storageEngine);
          logger.warn("The following store is marked as disabled: " + storageEngine.getName());
          // Must put server in offline mode.
        }
      } catch (NoSuchCapabilityException e) {
        // Not a read-only store: no-op
      }
    }
    if (listOfDisabledStores.isEmpty()) {
      logger.info("All stores initialized.");
    } else {
      throw new DisabledStoreException(
          "All stores initialized, but the server needs to go "
              + "in offline mode because some store(s) are disabled.");
    }
  }