コード例 #1
0
  private static void upgradeV6ToV7(List<Config> configs) {
    // rename advancedConfig to proxyConfig
    List<Config> configsToRemove = new ArrayList<>();
    List<Config> configsToAdd = new ArrayList<>();

    for (Config config : configs) {
      switch (config.getName()) {
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.useProxy":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyHost":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyPort":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyUser":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyPassword":
          configsToRemove.add(config);
          configsToAdd.add(
              new Config(
                  config.getName().replace("advancedConfig", "proxyConfig"), config.getValue()));
          break;
        default:
          // no upgrade needed
          break;
      }
    }

    configs.removeAll(configsToRemove);
    configs.addAll(configsToAdd);

    // add transfer manager configs
    configs.add(new Config(S3TargetConfigBean.S3_TM_CONFIG_PREFIX + "threadPoolSize", 10));
    configs.add(
        new Config(S3TargetConfigBean.S3_TM_CONFIG_PREFIX + "minimumUploadPartSize", 5242880));
    configs.add(
        new Config(S3TargetConfigBean.S3_TM_CONFIG_PREFIX + "multipartUploadThreshold", 268435456));
  }
コード例 #2
0
  private static void upgradeV1ToV2(List<Config> configs) {

    List<Config> configsToRemove = new ArrayList<>();
    List<Config> configsToAdd = new ArrayList<>();

    for (Config config : configs) {
      switch (config.getName()) {
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "charset":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "csvFileFormat":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "csvHeader":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "csvReplaceNewLines":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "jsonMode":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "textFieldPath":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "textEmptyLineIfNull":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "avroSchema":
        case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "binaryFieldPath":
          configsToRemove.add(config);
          configsToAdd.add(
              new Config(
                  S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX
                      + "dataGeneratorFormatConfig."
                      + config.getName().substring(19),
                  config.getValue()));
          break;
        default:
          // no upgrade needed
          break;
      }
    }

    configs.removeAll(configsToRemove);
    configs.addAll(configsToAdd);

    configs.add(
        new Config(
            S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX
                + "dataGeneratorFormatConfig.csvCustomDelimiter",
            '|'));
    configs.add(
        new Config(
            S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX
                + "dataGeneratorFormatConfig.csvCustomEscape",
            '\\'));
    configs.add(
        new Config(
            S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX
                + "dataGeneratorFormatConfig.csvCustomQuote",
            '\"'));
  }
コード例 #3
0
  private void upgradeToConsumerConfigBeanV1(List<Config> configs) {
    for (Config config : configs) {
      // Migrate existing configs that were moved into the Kinesis Consumer config bean
      switch (config.getName()) {
        case "applicationName":
          // fall through
        case "maxBatchSize":
          // fall through
        case "idleTimeBetweenReads":
          // fall through
        case "maxWaitTime":
          // fall through
        case "previewWaitTime":
          moveConfigToBean(config, KINESIS_CONFIG_BEAN);
          break;
        default:
          // no-op
      }
    }
    commitMove(configs);

    configs.add(
        new Config(
            KINESIS_CONFIG_BEAN + ".initialPositionInStream", InitialPositionInStream.LATEST));
  }
コード例 #4
0
  private void upgradeV1ToV2(List<Config> configs) {
    List<Config> configsToRemove = new ArrayList<Config>();
    List<Config> configsToAdd = new ArrayList<Config>();
    for (Config config : configs) {
      switch (config.getName()) {
        case FIELD_HASHER_CONFIGS:
          configsToRemove.add(config);

          // Handle InPlace Field Hasher Config
          // Move fieldHasherConfigs to the new config hasherConfig.inPlaceFieldHasherConfigs
          List<LinkedHashMap<String, Object>> fieldHasherConfigs =
              (List<LinkedHashMap<String, Object>>) config.getValue();
          for (LinkedHashMap<String, Object> fieldHasherConfig : fieldHasherConfigs) {
            List<String> fieldsToHash = (List<String>) fieldHasherConfig.get(FIELDS_TO_HASH);
            fieldHasherConfig.remove(FIELDS_TO_HASH);
            fieldHasherConfig.put(SOURCE_FIELDS_TO_HASH, fieldsToHash);
          }
          configsToAdd.add(
              new Config(
                  JOINER.join(HASHER_CONFIG, INPLACE_FIELD_HASHER_CONFIGS), fieldHasherConfigs));

          // Add new Record Hasher Config
          configsToAdd.add(
              new Config(
                  JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, HASH_ENTIRE_RECORD), false));
          configsToAdd.add(
              new Config(
                  JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, INCLUDE_RECORD_HEADER), false));
          configsToAdd.add(
              new Config(
                  JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, HASH_TYPE), HashType.MD5));
          configsToAdd.add(
              new Config(JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, TARGET_FIELD), ""));
          configsToAdd.add(
              new Config(JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, HEADER_ATTRIBUTE), ""));

          // Add new Target Field Hasher Config
          configsToAdd.add(
              new Config(
                  JOINER.join(HASHER_CONFIG, TARGET_FIELD_HASHER_CONFIGS),
                  new ArrayList<LinkedHashMap<String, Object>>()));
          break;
        default: // NO OP for others
      }
    }
    configs.removeAll(configsToRemove);
    configs.addAll(configsToAdd);
  }
コード例 #5
0
  private static void upgradeV3ToV4(List<Config> configs) {
    List<Config> configsToRemove = new ArrayList<>();
    List<Config> configsToAdd = new ArrayList<>();

    for (Config config : configs) {
      switch (config.getName()) {
        case S3TargetConfigBean.S3_CONFIG_PREFIX + "folder":
          configsToAdd.add(
              new Config(S3TargetConfigBean.S3_CONFIG_PREFIX + "commonPrefix", config.getValue()));
          configsToRemove.add(config);
          break;
        default:
          // no op
      }
    }

    configs.addAll(configsToAdd);
    configs.removeAll(configsToRemove);

    configs.add(
        new Config(S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "partitionTemplate", ""));
  }
コード例 #6
0
  protected void upgradeToCommonConfigBeanV1(List<Config> configs, String beanName) {
    for (Config config : configs) {
      // Migrate existing configs that were moved into the common Kinesis config bean
      switch (config.getName()) {
        case "region":
          // fall through
        case "streamName":
          // fall through:
        case "dataFormat":
          // fall through:
        case "awsAccessKeyId":
          // fall through
        case "awsSecretAccessKey":
          moveConfigToBean(config, beanName);
          break;
        default:
          // no-op
      }
    }

    commitMove(configs);
  }
コード例 #7
0
 protected void moveConfigToBean(Config config, String beanName) {
   configsToRemove.add(config);
   configsToAdd.add(new Config(beanName + "." + config.getName(), config.getValue()));
 }