Esempio n. 1
0
  private RowLogProcessorSettings createProcessorSettings(Conf conf) {
    RowLogProcessorSettings settings = new RowLogProcessorSettings();

    settings.setMsgTimestampMargin(
        conf.getChild("messageTimestampMargin")
            .getValueAsInteger(RowLogProcessor.DEFAULT_MSG_TIMESTAMP_MARGIN));

    settings.setScanThreadCount(
        conf.getChild("scanThreadCount").getValueAsInteger(settings.getScanThreadCount()));

    settings.setScanBatchSize(
        conf.getChild("scanBatchSize").getValueAsInteger(settings.getScanBatchSize()));

    settings.setMessagesWorkQueueSize(
        conf.getChild("messagesWorkQueueSize")
            .getValueAsInteger(settings.getMessagesWorkQueueSize()));

    return settings;
  }
Esempio n. 2
0
  private RowLogConfig createRowLogConfig(Conf initialConf) {
    boolean respectOrder = initialConf.getChild("respectOrder").getValueAsBoolean();
    boolean enableNotify = initialConf.getChild("enableNotify").getValueAsBoolean();
    long notifyDelay = initialConf.getChild("notifyDelay").getValueAsLong();
    long minimalProcessDelay = initialConf.getChild("minimalProcessDelay").getValueAsLong();
    long wakeupTimeout = initialConf.getChild("wakeupTimeout").getValueAsLong();
    long orphanedMessageDelay = initialConf.getChild("orphanedMessageDelay").getValueAsLong();
    int deleteBufferSize = initialConf.getChild("deleteBufferSize").getValueAsInteger();

    return new RowLogConfig(
        respectOrder,
        enableNotify,
        notifyDelay,
        minimalProcessDelay,
        wakeupTimeout,
        orphanedMessageDelay,
        deleteBufferSize);
  }
Esempio n. 3
0
  @PostConstruct
  public void start()
      throws InterruptedException, KeeperException, IOException, LeaderElectionSetupException,
          RowLogException {
    if (!confMgr.rowLogExists("wal")) {
      confMgr.addRowLog("wal", createRowLogConfig(rowLogConf.getChild("walConfig")));
    }

    if (!confMgr.rowLogExists("mq")) {
      confMgr.addRowLog("mq", createRowLogConfig(rowLogConf.getChild("mqConfig")));
    } else {
      // Before Lily 1.0.4, the respectOrder parameter for the MQ was true. Change it if necessary.
      for (Map.Entry<String, RowLogConfig> entry : confMgr.getRowLogs().entrySet()) {
        if (entry.getKey().equals("mq") && entry.getValue().isRespectOrder()) {
          log.warn("Changing MQ respect order to false.");
          RowLogConfig config = entry.getValue();
          config.setRespectOrder(false);
          confMgr.updateRowLog("mq", config);
        }
      }
    }

    // Link index updater disabled by default since this task can now be done my the derefMap
    boolean linkIdxEnabled =
        rowLogConf.getChild("linkIndexUpdater").getAttributeAsBoolean("enabled", false);
    if (linkIdxEnabled) {
      if (!confMgr.subscriptionExists("wal", "LinkIndexUpdater")) {
        // If the subscription already exists, this method will silently return
        confMgr.addSubscription("wal", "LinkIndexUpdater", RowLogSubscription.Type.VM, 10);
      }
    } else {
      log.info("LinkIndexUpdater is disabled.");
      if (confMgr.subscriptionExists("wal", "LinkIndexUpdater")) {
        confMgr.removeSubscription("wal", "LinkIndexUpdater");
      }
    }

    boolean mqFeederEnabled =
        rowLogConf.getChild("mqFeeder").getAttributeAsBoolean("enabled", true);
    if (mqFeederEnabled) {
      if (!confMgr.subscriptionExists("wal", "MQFeeder")) {
        confMgr.addSubscription("wal", "MQFeeder", RowLogSubscription.Type.VM, 20);
      }
    } else {
      log.info("MQFeeder is disabled.");
      if (confMgr.subscriptionExists("wal", "MQFeeder")) {
        confMgr.removeSubscription("wal", "MQFeeder");
      }
    }

    int shardCount = rowLogConf.getChild("shardCount").getValueAsInteger();

    messageQueue =
        new RowLogImpl(
            "mq",
            LilyHBaseSchema.getRecordTable(hbaseTableFactory),
            RecordCf.ROWLOG.bytes,
            RecordColumn.MQ_PREFIX,
            confMgr,
            null,
            new RowLogHashShardRouter());
    RowLogShardSetup.setupShards(shardCount, messageQueue, hbaseTableFactory);

    writeAheadLog =
        new WalRowLog(
            "wal",
            LilyHBaseSchema.getRecordTable(hbaseTableFactory),
            RecordCf.ROWLOG.bytes,
            RecordColumn.WAL_PREFIX,
            confMgr,
            rowLocker,
            new RowLogHashShardRouter());
    RowLogShardSetup.setupShards(shardCount, writeAheadLog, hbaseTableFactory);

    RowLogMessageListenerMapping.INSTANCE.put(
        WalListener.ID, new WalListener(writeAheadLog, rowLocker));
    // Instead of using the default MQFeeder, a custom one is used to do selective feeding of
    // indexer
    // related subscriptions, see IndexAwareMQFeeder.
    // RowLogMessageListenerMapping.INSTANCE.put("MQFeeder", new MessageQueueFeeder(messageQueue));

    // Start the message queue processor
    Conf mqProcessorConf = rowLogConf.getChild("mqProcessor");
    boolean mqProcEnabled = mqProcessorConf.getAttributeAsBoolean("enabled", true);
    if (mqProcEnabled) {
      List<String> mqProcessorNodes = Collections.EMPTY_LIST;
      Conf nodesConf = mqProcessorConf.getChild("nodes");
      if (nodesConf != null) {
        String nodes = nodesConf.getValue("");
        if (!nodes.isEmpty()) {
          mqProcessorNodes = Arrays.asList(nodes.split(","));
        }
      }
      RowLogProcessorSettings settings = createProcessorSettings(mqProcessorConf);
      RowLogProcessor processor =
          new RowLogProcessorImpl(messageQueue, confMgr, hbaseConf, settings);
      messageQueueProcessorLeader = new RowLogProcessorElection(zk, processor, lilyInfo);
      if (mqProcessorNodes.isEmpty() || mqProcessorNodes.contains(hostName)) {
        messageQueueProcessorLeader.start();
      }
    } else {
      log.info("Not participating in MQ processor election.");
    }

    if (linkIdxEnabled) {
      confMgr.addListener("wal", "LinkIndexUpdater", "LinkIndexUpdaterListener");
    }

    if (mqFeederEnabled) {
      confMgr.addListener("wal", "MQFeeder", "MQFeederListener");
    }

    // Start the wal processor
    Conf walProcessorConf = rowLogConf.getChild("walProcessor");
    boolean walProcEnabled = walProcessorConf.getAttributeAsBoolean("enabled", true);
    if (walProcEnabled) {
      List<String> walProcessorNodes = Collections.EMPTY_LIST;
      Conf nodesConf = walProcessorConf.getChild("nodes");
      if (nodesConf != null) {
        String nodes = nodesConf.getValue("");
        if (!nodes.isEmpty()) {
          walProcessorNodes = Arrays.asList(nodes.split(","));
        }
      }
      RowLogProcessorSettings settings = createProcessorSettings(walProcessorConf);
      RowLogProcessor processor = new WalProcessor(writeAheadLog, confMgr, hbaseConf, settings);
      writeAheadLogProcessorLeader = new RowLogProcessorElection(zk, processor, lilyInfo);
      // The WAL processor should only be started once the LinkIndexUpdater listener is available
      walProcessorStartupThread = new Thread(new DelayedWALProcessorStartup());
      if (walProcessorNodes.isEmpty() || walProcessorNodes.contains(hostName)) {
        walProcessorStartupThread.start();
      }
    } else {
      log.info("Not participating in WAL processor election.");
    }
  }