private static void upgradeV6ToV7(List<Config> configs) { // rename advancedConfig to proxyConfig List<Config> configsToRemove = new ArrayList<>(); List<Config> configsToAdd = new ArrayList<>(); for (Config config : configs) { switch (config.getName()) { case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.useProxy": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyHost": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyPort": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyUser": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyPassword": configsToRemove.add(config); configsToAdd.add( new Config( config.getName().replace("advancedConfig", "proxyConfig"), config.getValue())); break; default: // no upgrade needed break; } } configs.removeAll(configsToRemove); configs.addAll(configsToAdd); // add transfer manager configs configs.add(new Config(S3TargetConfigBean.S3_TM_CONFIG_PREFIX + "threadPoolSize", 10)); configs.add( new Config(S3TargetConfigBean.S3_TM_CONFIG_PREFIX + "minimumUploadPartSize", 5242880)); configs.add( new Config(S3TargetConfigBean.S3_TM_CONFIG_PREFIX + "multipartUploadThreshold", 268435456)); }
private void upgradeToConsumerConfigBeanV1(List<Config> configs) { for (Config config : configs) { // Migrate existing configs that were moved into the Kinesis Consumer config bean switch (config.getName()) { case "applicationName": // fall through case "maxBatchSize": // fall through case "idleTimeBetweenReads": // fall through case "maxWaitTime": // fall through case "previewWaitTime": moveConfigToBean(config, KINESIS_CONFIG_BEAN); break; default: // no-op } } commitMove(configs); configs.add( new Config( KINESIS_CONFIG_BEAN + ".initialPositionInStream", InitialPositionInStream.LATEST)); }
private static void upgradeV1ToV2(List<Config> configs) { List<Config> configsToRemove = new ArrayList<>(); List<Config> configsToAdd = new ArrayList<>(); for (Config config : configs) { switch (config.getName()) { case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "charset": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "csvFileFormat": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "csvHeader": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "csvReplaceNewLines": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "jsonMode": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "textFieldPath": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "textEmptyLineIfNull": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "avroSchema": case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "binaryFieldPath": configsToRemove.add(config); configsToAdd.add( new Config( S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "dataGeneratorFormatConfig." + config.getName().substring(19), config.getValue())); break; default: // no upgrade needed break; } } configs.removeAll(configsToRemove); configs.addAll(configsToAdd); configs.add( new Config( S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "dataGeneratorFormatConfig.csvCustomDelimiter", '|')); configs.add( new Config( S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "dataGeneratorFormatConfig.csvCustomEscape", '\\')); configs.add( new Config( S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "dataGeneratorFormatConfig.csvCustomQuote", '\"')); }
private void upgradeV1ToV2(List<Config> configs) { List<Config> configsToRemove = new ArrayList<Config>(); List<Config> configsToAdd = new ArrayList<Config>(); for (Config config : configs) { switch (config.getName()) { case FIELD_HASHER_CONFIGS: configsToRemove.add(config); // Handle InPlace Field Hasher Config // Move fieldHasherConfigs to the new config hasherConfig.inPlaceFieldHasherConfigs List<LinkedHashMap<String, Object>> fieldHasherConfigs = (List<LinkedHashMap<String, Object>>) config.getValue(); for (LinkedHashMap<String, Object> fieldHasherConfig : fieldHasherConfigs) { List<String> fieldsToHash = (List<String>) fieldHasherConfig.get(FIELDS_TO_HASH); fieldHasherConfig.remove(FIELDS_TO_HASH); fieldHasherConfig.put(SOURCE_FIELDS_TO_HASH, fieldsToHash); } configsToAdd.add( new Config( JOINER.join(HASHER_CONFIG, INPLACE_FIELD_HASHER_CONFIGS), fieldHasherConfigs)); // Add new Record Hasher Config configsToAdd.add( new Config( JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, HASH_ENTIRE_RECORD), false)); configsToAdd.add( new Config( JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, INCLUDE_RECORD_HEADER), false)); configsToAdd.add( new Config( JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, HASH_TYPE), HashType.MD5)); configsToAdd.add( new Config(JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, TARGET_FIELD), "")); configsToAdd.add( new Config(JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, HEADER_ATTRIBUTE), "")); // Add new Target Field Hasher Config configsToAdd.add( new Config( JOINER.join(HASHER_CONFIG, TARGET_FIELD_HASHER_CONFIGS), new ArrayList<LinkedHashMap<String, Object>>())); break; default: // NO OP for others } } configs.removeAll(configsToRemove); configs.addAll(configsToAdd); }
private static void upgradeV3ToV4(List<Config> configs) { List<Config> configsToRemove = new ArrayList<>(); List<Config> configsToAdd = new ArrayList<>(); for (Config config : configs) { switch (config.getName()) { case S3TargetConfigBean.S3_CONFIG_PREFIX + "folder": configsToAdd.add( new Config(S3TargetConfigBean.S3_CONFIG_PREFIX + "commonPrefix", config.getValue())); configsToRemove.add(config); break; default: // no op } } configs.addAll(configsToAdd); configs.removeAll(configsToRemove); configs.add( new Config(S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "partitionTemplate", "")); }
protected void upgradeToCommonConfigBeanV1(List<Config> configs, String beanName) { for (Config config : configs) { // Migrate existing configs that were moved into the common Kinesis config bean switch (config.getName()) { case "region": // fall through case "streamName": // fall through: case "dataFormat": // fall through: case "awsAccessKeyId": // fall through case "awsSecretAccessKey": moveConfigToBean(config, beanName); break; default: // no-op } } commitMove(configs); }
protected void moveConfigToBean(Config config, String beanName) { configsToRemove.add(config); configsToAdd.add(new Config(beanName + "." + config.getName(), config.getValue())); }