Exemplo n.º 1
0
  @Override
  public void configure(Context context) {
    if (morphlineContext == null) {
      FaultTolerance faultTolerance =
          new FaultTolerance(
              context.getBoolean(FaultTolerance.IS_PRODUCTION_MODE, false),
              context.getBoolean(FaultTolerance.IS_IGNORING_RECOVERABLE_EXCEPTIONS, false),
              context.getString(FaultTolerance.RECOVERABLE_EXCEPTION_CLASSES));

      morphlineContext =
          new MorphlineContext.Builder()
              .setExceptionHandler(faultTolerance)
              .setMetricRegistry(new MetricRegistry())
              .build();
    }

    String morphlineFile = context.getString(MORPHLINE_FILE_PARAM);
    String morphlineId = context.getString(MORPHLINE_ID_PARAM);
    if (morphlineFile == null || morphlineFile.trim().length() == 0) {
      throw new MorphlineCompilationException("Missing parameter: " + MORPHLINE_FILE_PARAM, null);
    }
    Config override =
        ConfigFactory.parseMap(context.getSubProperties(MORPHLINE_VARIABLE_PARAM + "."));
    morphline =
        new Compiler()
            .compile(new File(morphlineFile), morphlineId, morphlineContext, finalChild, override);
    morphlineFileAndId = morphlineFile + "@" + morphlineId;
  }
  /**
   * Constructor to initialize hibernate configuration parameters
   *
   * @param sqlSourceHelper Contains the configuration parameters from flume config file
   */
  public HibernateHelper(SQLSourceHelper sqlSourceHelper) {

    this.sqlSourceHelper = sqlSourceHelper;
    Context context = sqlSourceHelper.getContext();

    Map<String, String> hibernateProperties = context.getSubProperties("hibernate.");
    Iterator<Map.Entry<String, String>> it = hibernateProperties.entrySet().iterator();

    config = new Configuration();
    Map.Entry<String, String> e;

    while (it.hasNext()) {
      e = it.next();
      config.setProperty("hibernate." + e.getKey(), e.getValue());
    }
  }
Exemplo n.º 3
0
  @Override
  public synchronized void configure(Context context) {
    spoolDirectory = context.getString(SPOOL_DIRECTORY);
    Preconditions.checkState(
        spoolDirectory != null, "Configuration must specify a spooling directory");

    completedSuffix = context.getString(SPOOLED_FILE_SUFFIX, DEFAULT_SPOOLED_FILE_SUFFIX);
    deletePolicy = context.getString(DELETE_POLICY, DEFAULT_DELETE_POLICY);
    fileHeader = context.getBoolean(FILENAME_HEADER, DEFAULT_FILE_HEADER);
    fileHeaderKey = context.getString(FILENAME_HEADER_KEY, DEFAULT_FILENAME_HEADER_KEY);
    basenameHeader = context.getBoolean(BASENAME_HEADER, DEFAULT_BASENAME_HEADER);
    basenameHeaderKey = context.getString(BASENAME_HEADER_KEY, DEFAULT_BASENAME_HEADER_KEY);
    batchSize = context.getInteger(BATCH_SIZE, DEFAULT_BATCH_SIZE);
    inputCharset = context.getString(INPUT_CHARSET, DEFAULT_INPUT_CHARSET);
    decodeErrorPolicy =
        DecodeErrorPolicy.valueOf(
            context
                .getString(DECODE_ERROR_POLICY, DEFAULT_DECODE_ERROR_POLICY)
                .toUpperCase(Locale.ENGLISH));

    ignorePattern = context.getString(IGNORE_PAT, DEFAULT_IGNORE_PAT);
    trackerDirPath = context.getString(TRACKER_DIR, DEFAULT_TRACKER_DIR);

    deserializerType = context.getString(DESERIALIZER, "ZipDeserializer");
    deserializerContext = new Context(context.getSubProperties(DESERIALIZER + "."));

    consumeOrder =
        ConsumeOrder.valueOf(
            context
                .getString(CONSUME_ORDER, DEFAULT_CONSUME_ORDER.toString())
                .toUpperCase(Locale.ENGLISH));

    // "Hack" to support backwards compatibility with previous generation of
    // spooling directory source, which did not support deserializers
    Integer bufferMaxLineLength = context.getInteger(BUFFER_MAX_LINE_LENGTH);
    if (bufferMaxLineLength != null
        && deserializerType != null
        && deserializerType.equalsIgnoreCase(DEFAULT_DESERIALIZER)) {
      deserializerContext.put(LineDeserializer.MAXLINE_KEY, bufferMaxLineLength.toString());
    }

    maxBackoff = context.getInteger(MAX_BACKOFF, DEFAULT_MAX_BACKOFF);
    if (sourceCounter == null) {
      sourceCounter = new SourceCounter(getName());
    }
  }
Exemplo n.º 4
0
  public void configure(Context context) throws ConfigurationException {
    super.configure(context);
    try {
      String channelList = context.getString(BasicConfigurationConstants.CONFIG_CHANNELS);
      if (channelList != null) {
        this.channels = new HashSet<String>(Arrays.asList(channelList.split("\\s+")));
      }
      if (channels.isEmpty()) {
        errors.add(
            new FlumeConfigurationError(
                componentName,
                ComponentType.CHANNEL.getComponentType(),
                FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
                ErrorOrWarning.ERROR));
        throw new ConfigurationException("No channels set for " + this.getComponentName());
      }
      Map<String, String> selectorParams =
          context.getSubProperties(
              BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);
      String selType;
      if (selectorParams != null && !selectorParams.isEmpty()) {
        selType = selectorParams.get(BasicConfigurationConstants.CONFIG_TYPE);
        System.out.println("Loading selector: " + selType);
      } else {
        selType = ChannelSelectorConfigurationType.REPLICATING.toString();
      }

      if (selType == null || selType.isEmpty()) {
        selType = ChannelSelectorConfigurationType.REPLICATING.toString();
      }
      ChannelSelectorType selectorType = this.getKnownChannelSelector(selType);
      Context selectorContext = new Context();
      selectorContext.putAll(selectorParams);
      String config = null;
      if (selectorType == null) {
        config = selectorContext.getString(BasicConfigurationConstants.CONFIG_CONFIG);
        if (config == null || config.isEmpty()) {
          config = "OTHER";
        }
      } else {
        config = selectorType.toString().toUpperCase();
      }

      this.selectorConf =
          (ChannelSelectorConfiguration)
              ComponentConfigurationFactory.create(
                  ComponentType.CHANNELSELECTOR.getComponentType(),
                  config,
                  ComponentType.CHANNELSELECTOR);
      selectorConf.setChannels(channels);
      selectorConf.configure(selectorContext);
    } catch (Exception e) {
      errors.add(
          new FlumeConfigurationError(
              componentName,
              ComponentType.CHANNELSELECTOR.getComponentType(),
              FlumeConfigurationErrorType.CONFIG_ERROR,
              ErrorOrWarning.ERROR));
      throw new ConfigurationException("Failed to configure component!", e);
    }
  }
  @Override
  public void configure(Context context) {
    if (!isLocal) {
      if (StringUtils.isNotBlank(context.getString(HOSTNAMES))) {
        serverAddresses = StringUtils.deleteWhitespace(context.getString(HOSTNAMES)).split(",");
      }
      Preconditions.checkState(
          serverAddresses != null && serverAddresses.length > 0, "Missing Param:" + HOSTNAMES);
    }

    if (StringUtils.isNotBlank(context.getString(INDEX_NAME))) {
      this.indexName = context.getString(INDEX_NAME);
    }

    if (StringUtils.isNotBlank(context.getString(INDEX_TYPE))) {
      this.indexType = context.getString(INDEX_TYPE);
    }

    if (StringUtils.isNotBlank(context.getString(CLUSTER_NAME))) {
      this.clusterName = context.getString(CLUSTER_NAME);
    }

    if (StringUtils.isNotBlank(context.getString(BATCH_SIZE))) {
      this.batchSize = Integer.parseInt(context.getString(BATCH_SIZE));
    }

    if (StringUtils.isNotBlank(context.getString(TTL))) {
      this.ttlMs = parseTTL(context.getString(TTL));
      Preconditions.checkState(ttlMs > 0, TTL + " must be greater than 0 or not set.");
    }

    if (StringUtils.isNotBlank(context.getString(CLIENT_TYPE))) {
      clientType = context.getString(CLIENT_TYPE);
    }

    elasticSearchClientContext = new Context();
    elasticSearchClientContext.putAll(context.getSubProperties(CLIENT_PREFIX));

    String serializerClazz = DEFAULT_SERIALIZER_CLASS;
    if (StringUtils.isNotBlank(context.getString(SERIALIZER))) {
      serializerClazz = context.getString(SERIALIZER);
    }

    Context serializerContext = new Context();
    serializerContext.putAll(context.getSubProperties(SERIALIZER_PREFIX));

    try {
      @SuppressWarnings("unchecked")
      Class<? extends Configurable> clazz =
          (Class<? extends Configurable>) Class.forName(serializerClazz);
      Configurable serializer = clazz.newInstance();

      if (serializer instanceof ElasticSearchIndexRequestBuilderFactory) {
        indexRequestFactory = (ElasticSearchIndexRequestBuilderFactory) serializer;
        indexRequestFactory.configure(serializerContext);
      } else if (serializer instanceof ElasticSearchEventSerializer) {
        eventSerializer = (ElasticSearchEventSerializer) serializer;
        eventSerializer.configure(serializerContext);
      } else {
        throw new IllegalArgumentException(
            serializerClazz + " is not an ElasticSearchEventSerializer");
      }
    } catch (Exception e) {
      logger.error("Could not instantiate event serializer.", e);
      Throwables.propagate(e);
    }

    if (sinkCounter == null) {
      sinkCounter = new SinkCounter(getName());
    }

    String indexNameBuilderClass = DEFAULT_INDEX_NAME_BUILDER_CLASS;
    if (StringUtils.isNotBlank(context.getString(INDEX_NAME_BUILDER))) {
      indexNameBuilderClass = context.getString(INDEX_NAME_BUILDER);
    }

    Context indexnameBuilderContext = new Context();
    indexnameBuilderContext.putAll(context.getSubProperties(INDEX_NAME_BUILDER_PREFIX));

    try {
      @SuppressWarnings("unchecked")
      Class<? extends IndexNameBuilder> clazz =
          (Class<? extends IndexNameBuilder>) Class.forName(indexNameBuilderClass);
      indexNameBuilder = clazz.newInstance();
      indexnameBuilderContext.put(INDEX_NAME, indexName);
      indexNameBuilder.configure(indexnameBuilderContext);
    } catch (Exception e) {
      logger.error("Could not instantiate index name builder.", e);
      Throwables.propagate(e);
    }

    if (sinkCounter == null) {
      sinkCounter = new SinkCounter(getName());
    }

    Preconditions.checkState(StringUtils.isNotBlank(indexName), "Missing Param:" + INDEX_NAME);
    Preconditions.checkState(StringUtils.isNotBlank(indexType), "Missing Param:" + INDEX_TYPE);
    Preconditions.checkState(StringUtils.isNotBlank(clusterName), "Missing Param:" + CLUSTER_NAME);
    Preconditions.checkState(batchSize >= 1, BATCH_SIZE + " must be greater than 0");
  }