/**
* Sets the value for the appropriate key in the {@link Properties}.
*
* @param key the key to set
* @param value the value for the key
*/
public static void set(String key, String value) {
Preconditions.checkArgument(
key != null && value != null,
String.format("the key value pair (%s, %s) cannot have null", key, value));
PROPERTIES.put(key, value);
checkUserFileBufferBytes();
}
/**
* Merges the current configuration properties with alternate properties. A property from the new
* configuration wins if it also appears in the current configuration.
*
* @param properties The source {@link Properties} to be merged
*/
public static void merge(Map<?, ?> properties) {
if (properties != null) {
// merge the system properties
PROPERTIES.putAll(properties);
}
checkUserFileBufferBytes();
}
/**
* Constructor with a flag to indicate whether system properties should be included. When the flag
* is set to false, it is used for {@link Configuration} test class.
*
* @param sitePropertiesFile site-wide properties
* @param includeSystemProperties whether to include the system properties
*/
private static void init(String sitePropertiesFile, boolean includeSystemProperties) {
// Load default
Properties defaultProps = ConfigurationUtils.loadPropertiesFromResource(DEFAULT_PROPERTIES);
if (defaultProps == null) {
throw new RuntimeException(
ExceptionMessage.DEFAULT_PROPERTIES_FILE_DOES_NOT_EXIST.getMessage());
}
// Override runtime default
defaultProps.setProperty(Constants.MASTER_HOSTNAME, NetworkAddressUtils.getLocalHostName(250));
defaultProps.setProperty(
Constants.WORKER_NETWORK_NETTY_CHANNEL, String.valueOf(ChannelType.defaultType()));
defaultProps.setProperty(
Constants.USER_NETWORK_NETTY_CHANNEL, String.valueOf(ChannelType.defaultType()));
String confPaths;
// If site conf is overwritten in system properties, overwrite the default setting
if (System.getProperty(Constants.SITE_CONF_DIR) != null) {
confPaths = System.getProperty(Constants.SITE_CONF_DIR);
} else {
confPaths = defaultProps.getProperty(Constants.SITE_CONF_DIR);
}
String[] confPathList = confPaths.split(",");
// Load site specific properties file
Properties siteProps =
ConfigurationUtils.searchPropertiesFile(sitePropertiesFile, confPathList);
// Load system properties
Properties systemProps = new Properties();
if (includeSystemProperties) {
systemProps.putAll(System.getProperties());
}
// Now lets combine, order matters here
PROPERTIES.putAll(defaultProps);
if (siteProps != null) {
PROPERTIES.putAll(siteProps);
}
PROPERTIES.putAll(systemProps);
String masterHostname = PROPERTIES.getProperty(Constants.MASTER_HOSTNAME);
String masterPort = PROPERTIES.getProperty(Constants.MASTER_RPC_PORT);
boolean useZk = Boolean.parseBoolean(PROPERTIES.getProperty(Constants.ZOOKEEPER_ENABLED));
String masterAddress =
(useZk ? Constants.HEADER_FT : Constants.HEADER) + masterHostname + ":" + masterPort;
PROPERTIES.setProperty(Constants.MASTER_ADDRESS, masterAddress);
checkUserFileBufferBytes();
// Make sure the user hasn't set worker ports when there may be multiple workers per host
int maxWorkersPerHost = getInt(Constants.INTEGRATION_YARN_WORKERS_PER_HOST_MAX);
if (maxWorkersPerHost > 1) {
String message =
"%s cannot be specified when allowing multiple workers per host with "
+ Constants.INTEGRATION_YARN_WORKERS_PER_HOST_MAX
+ "="
+ maxWorkersPerHost;
Preconditions.checkState(
System.getProperty(Constants.WORKER_DATA_PORT) == null,
String.format(message, Constants.WORKER_DATA_PORT));
Preconditions.checkState(
System.getProperty(Constants.WORKER_RPC_PORT) == null,
String.format(message, Constants.WORKER_RPC_PORT));
Preconditions.checkState(
System.getProperty(Constants.WORKER_WEB_PORT) == null,
String.format(message, Constants.WORKER_WEB_PORT));
PROPERTIES.setProperty(Constants.WORKER_DATA_PORT, "0");
PROPERTIES.setProperty(Constants.WORKER_RPC_PORT, "0");
PROPERTIES.setProperty(Constants.WORKER_WEB_PORT, "0");
}
}