private static void upgradeV6ToV7(List<Config> configs) {
// rename advancedConfig to proxyConfig
List<Config> configsToRemove = new ArrayList<>();
List<Config> configsToAdd = new ArrayList<>();
for (Config config : configs) {
switch (config.getName()) {
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.useProxy":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyHost":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyPort":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyUser":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "advancedConfig.proxyPassword":
configsToRemove.add(config);
configsToAdd.add(
new Config(
config.getName().replace("advancedConfig", "proxyConfig"), config.getValue()));
break;
default:
// no upgrade needed
break;
}
}
configs.removeAll(configsToRemove);
configs.addAll(configsToAdd);
// add transfer manager configs
configs.add(new Config(S3TargetConfigBean.S3_TM_CONFIG_PREFIX + "threadPoolSize", 10));
configs.add(
new Config(S3TargetConfigBean.S3_TM_CONFIG_PREFIX + "minimumUploadPartSize", 5242880));
configs.add(
new Config(S3TargetConfigBean.S3_TM_CONFIG_PREFIX + "multipartUploadThreshold", 268435456));
}
private void upgradeToConsumerConfigBeanV1(List<Config> configs) {
for (Config config : configs) {
// Migrate existing configs that were moved into the Kinesis Consumer config bean
switch (config.getName()) {
case "applicationName":
// fall through
case "maxBatchSize":
// fall through
case "idleTimeBetweenReads":
// fall through
case "maxWaitTime":
// fall through
case "previewWaitTime":
moveConfigToBean(config, KINESIS_CONFIG_BEAN);
break;
default:
// no-op
}
}
commitMove(configs);
configs.add(
new Config(
KINESIS_CONFIG_BEAN + ".initialPositionInStream", InitialPositionInStream.LATEST));
}
private static void upgradeV1ToV2(List<Config> configs) {
List<Config> configsToRemove = new ArrayList<>();
List<Config> configsToAdd = new ArrayList<>();
for (Config config : configs) {
switch (config.getName()) {
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "charset":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "csvFileFormat":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "csvHeader":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "csvReplaceNewLines":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "jsonMode":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "textFieldPath":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "textEmptyLineIfNull":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "avroSchema":
case S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "binaryFieldPath":
configsToRemove.add(config);
configsToAdd.add(
new Config(
S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX
+ "dataGeneratorFormatConfig."
+ config.getName().substring(19),
config.getValue()));
break;
default:
// no upgrade needed
break;
}
}
configs.removeAll(configsToRemove);
configs.addAll(configsToAdd);
configs.add(
new Config(
S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX
+ "dataGeneratorFormatConfig.csvCustomDelimiter",
'|'));
configs.add(
new Config(
S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX
+ "dataGeneratorFormatConfig.csvCustomEscape",
'\\'));
configs.add(
new Config(
S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX
+ "dataGeneratorFormatConfig.csvCustomQuote",
'\"'));
}
private void upgradeV1ToV2(List<Config> configs) {
List<Config> configsToRemove = new ArrayList<Config>();
List<Config> configsToAdd = new ArrayList<Config>();
for (Config config : configs) {
switch (config.getName()) {
case FIELD_HASHER_CONFIGS:
configsToRemove.add(config);
// Handle InPlace Field Hasher Config
// Move fieldHasherConfigs to the new config hasherConfig.inPlaceFieldHasherConfigs
List<LinkedHashMap<String, Object>> fieldHasherConfigs =
(List<LinkedHashMap<String, Object>>) config.getValue();
for (LinkedHashMap<String, Object> fieldHasherConfig : fieldHasherConfigs) {
List<String> fieldsToHash = (List<String>) fieldHasherConfig.get(FIELDS_TO_HASH);
fieldHasherConfig.remove(FIELDS_TO_HASH);
fieldHasherConfig.put(SOURCE_FIELDS_TO_HASH, fieldsToHash);
}
configsToAdd.add(
new Config(
JOINER.join(HASHER_CONFIG, INPLACE_FIELD_HASHER_CONFIGS), fieldHasherConfigs));
// Add new Record Hasher Config
configsToAdd.add(
new Config(
JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, HASH_ENTIRE_RECORD), false));
configsToAdd.add(
new Config(
JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, INCLUDE_RECORD_HEADER), false));
configsToAdd.add(
new Config(
JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, HASH_TYPE), HashType.MD5));
configsToAdd.add(
new Config(JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, TARGET_FIELD), ""));
configsToAdd.add(
new Config(JOINER.join(HASHER_CONFIG, RECORD_HASHER_CONFIG, HEADER_ATTRIBUTE), ""));
// Add new Target Field Hasher Config
configsToAdd.add(
new Config(
JOINER.join(HASHER_CONFIG, TARGET_FIELD_HASHER_CONFIGS),
new ArrayList<LinkedHashMap<String, Object>>()));
break;
default: // NO OP for others
}
}
configs.removeAll(configsToRemove);
configs.addAll(configsToAdd);
}
private static void upgradeV3ToV4(List<Config> configs) {
List<Config> configsToRemove = new ArrayList<>();
List<Config> configsToAdd = new ArrayList<>();
for (Config config : configs) {
switch (config.getName()) {
case S3TargetConfigBean.S3_CONFIG_PREFIX + "folder":
configsToAdd.add(
new Config(S3TargetConfigBean.S3_CONFIG_PREFIX + "commonPrefix", config.getValue()));
configsToRemove.add(config);
break;
default:
// no op
}
}
configs.addAll(configsToAdd);
configs.removeAll(configsToRemove);
configs.add(
new Config(S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + "partitionTemplate", ""));
}
protected void upgradeToCommonConfigBeanV1(List<Config> configs, String beanName) {
for (Config config : configs) {
// Migrate existing configs that were moved into the common Kinesis config bean
switch (config.getName()) {
case "region":
// fall through
case "streamName":
// fall through:
case "dataFormat":
// fall through:
case "awsAccessKeyId":
// fall through
case "awsSecretAccessKey":
moveConfigToBean(config, beanName);
break;
default:
// no-op
}
}
commitMove(configs);
}
protected void moveConfigToBean(Config config, String beanName) {
configsToRemove.add(config);
configsToAdd.add(new Config(beanName + "." + config.getName(), config.getValue()));
}