@BeforeMethod(groups = {"dev"})
  public static void setup() throws UnknownHostException {
    mongo = new Mongo("localhost", 27017);

    Map<String, String> ctxMap = new HashMap<String, String>();
    ctxMap.put(MongoSink.HOST, "localhost");
    ctxMap.put(MongoSink.PORT, "27017");
    ctxMap.put(MongoSink.DB_NAME, "test_events");
    ctxMap.put(MongoSink.COLLECTION, "test_log");
    ctxMap.put(MongoSink.BATCH_SIZE, "100");

    ctx.putAll(ctxMap);

    Context channelCtx = new Context();
    channelCtx.put("capacity", "1000000");
    channelCtx.put("transactionCapacity", "1000000");
    channel = new MemoryChannel();
    Configurables.configure(channel, channelCtx);
  }
Example #2
0
  public void configure(Context context) throws ConfigurationException {
    super.configure(context);
    try {
      String channelList = context.getString(BasicConfigurationConstants.CONFIG_CHANNELS);
      if (channelList != null) {
        this.channels = new HashSet<String>(Arrays.asList(channelList.split("\\s+")));
      }
      if (channels.isEmpty()) {
        errors.add(
            new FlumeConfigurationError(
                componentName,
                ComponentType.CHANNEL.getComponentType(),
                FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
                ErrorOrWarning.ERROR));
        throw new ConfigurationException("No channels set for " + this.getComponentName());
      }
      Map<String, String> selectorParams =
          context.getSubProperties(
              BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);
      String selType;
      if (selectorParams != null && !selectorParams.isEmpty()) {
        selType = selectorParams.get(BasicConfigurationConstants.CONFIG_TYPE);
        System.out.println("Loading selector: " + selType);
      } else {
        selType = ChannelSelectorConfigurationType.REPLICATING.toString();
      }

      if (selType == null || selType.isEmpty()) {
        selType = ChannelSelectorConfigurationType.REPLICATING.toString();
      }
      ChannelSelectorType selectorType = this.getKnownChannelSelector(selType);
      Context selectorContext = new Context();
      selectorContext.putAll(selectorParams);
      String config = null;
      if (selectorType == null) {
        config = selectorContext.getString(BasicConfigurationConstants.CONFIG_CONFIG);
        if (config == null || config.isEmpty()) {
          config = "OTHER";
        }
      } else {
        config = selectorType.toString().toUpperCase();
      }

      this.selectorConf =
          (ChannelSelectorConfiguration)
              ComponentConfigurationFactory.create(
                  ComponentType.CHANNELSELECTOR.getComponentType(),
                  config,
                  ComponentType.CHANNELSELECTOR);
      selectorConf.setChannels(channels);
      selectorConf.configure(selectorContext);
    } catch (Exception e) {
      errors.add(
          new FlumeConfigurationError(
              componentName,
              ComponentType.CHANNELSELECTOR.getComponentType(),
              FlumeConfigurationErrorType.CONFIG_ERROR,
              ErrorOrWarning.ERROR));
      throw new ConfigurationException("Failed to configure component!", e);
    }
  }
  @Override
  public void configure(Context context) {
    if (!isLocal) {
      if (StringUtils.isNotBlank(context.getString(HOSTNAMES))) {
        serverAddresses = StringUtils.deleteWhitespace(context.getString(HOSTNAMES)).split(",");
      }
      Preconditions.checkState(
          serverAddresses != null && serverAddresses.length > 0, "Missing Param:" + HOSTNAMES);
    }

    if (StringUtils.isNotBlank(context.getString(INDEX_NAME))) {
      this.indexName = context.getString(INDEX_NAME);
    }

    if (StringUtils.isNotBlank(context.getString(INDEX_TYPE))) {
      this.indexType = context.getString(INDEX_TYPE);
    }

    if (StringUtils.isNotBlank(context.getString(CLUSTER_NAME))) {
      this.clusterName = context.getString(CLUSTER_NAME);
    }

    if (StringUtils.isNotBlank(context.getString(BATCH_SIZE))) {
      this.batchSize = Integer.parseInt(context.getString(BATCH_SIZE));
    }

    if (StringUtils.isNotBlank(context.getString(TTL))) {
      this.ttlMs = parseTTL(context.getString(TTL));
      Preconditions.checkState(ttlMs > 0, TTL + " must be greater than 0 or not set.");
    }

    if (StringUtils.isNotBlank(context.getString(CLIENT_TYPE))) {
      clientType = context.getString(CLIENT_TYPE);
    }

    elasticSearchClientContext = new Context();
    elasticSearchClientContext.putAll(context.getSubProperties(CLIENT_PREFIX));

    String serializerClazz = DEFAULT_SERIALIZER_CLASS;
    if (StringUtils.isNotBlank(context.getString(SERIALIZER))) {
      serializerClazz = context.getString(SERIALIZER);
    }

    Context serializerContext = new Context();
    serializerContext.putAll(context.getSubProperties(SERIALIZER_PREFIX));

    try {
      @SuppressWarnings("unchecked")
      Class<? extends Configurable> clazz =
          (Class<? extends Configurable>) Class.forName(serializerClazz);
      Configurable serializer = clazz.newInstance();

      if (serializer instanceof ElasticSearchIndexRequestBuilderFactory) {
        indexRequestFactory = (ElasticSearchIndexRequestBuilderFactory) serializer;
        indexRequestFactory.configure(serializerContext);
      } else if (serializer instanceof ElasticSearchEventSerializer) {
        eventSerializer = (ElasticSearchEventSerializer) serializer;
        eventSerializer.configure(serializerContext);
      } else {
        throw new IllegalArgumentException(
            serializerClazz + " is not an ElasticSearchEventSerializer");
      }
    } catch (Exception e) {
      logger.error("Could not instantiate event serializer.", e);
      Throwables.propagate(e);
    }

    if (sinkCounter == null) {
      sinkCounter = new SinkCounter(getName());
    }

    String indexNameBuilderClass = DEFAULT_INDEX_NAME_BUILDER_CLASS;
    if (StringUtils.isNotBlank(context.getString(INDEX_NAME_BUILDER))) {
      indexNameBuilderClass = context.getString(INDEX_NAME_BUILDER);
    }

    Context indexnameBuilderContext = new Context();
    indexnameBuilderContext.putAll(context.getSubProperties(INDEX_NAME_BUILDER_PREFIX));

    try {
      @SuppressWarnings("unchecked")
      Class<? extends IndexNameBuilder> clazz =
          (Class<? extends IndexNameBuilder>) Class.forName(indexNameBuilderClass);
      indexNameBuilder = clazz.newInstance();
      indexnameBuilderContext.put(INDEX_NAME, indexName);
      indexNameBuilder.configure(indexnameBuilderContext);
    } catch (Exception e) {
      logger.error("Could not instantiate index name builder.", e);
      Throwables.propagate(e);
    }

    if (sinkCounter == null) {
      sinkCounter = new SinkCounter(getName());
    }

    Preconditions.checkState(StringUtils.isNotBlank(indexName), "Missing Param:" + INDEX_NAME);
    Preconditions.checkState(StringUtils.isNotBlank(indexType), "Missing Param:" + INDEX_TYPE);
    Preconditions.checkState(StringUtils.isNotBlank(clusterName), "Missing Param:" + CLUSTER_NAME);
    Preconditions.checkState(batchSize >= 1, BATCH_SIZE + " must be greater than 0");
  }