/**
* Sets the connector information needed to communicate with Accumulo in this job.
*
* <p><b>WARNING:</b> Some tokens, when serialized, divulge sensitive information in the
* configuration as a means to pass the token to MapReduce tasks. This information is BASE64
* encoded to provide a charset safe conversion to a string, but this conversion is not intended
* to be secure. {@link PasswordToken} is one example that is insecure in this way; however {@link
* DelegationToken}s, acquired using {@link
* SecurityOperations#getDelegationToken(DelegationTokenConfig)}, is not subject to this concern.
*
* @param job the Hadoop job instance to be configured
* @param principal a valid Accumulo user name (user must have Table.CREATE permission if {@link
* #setCreateTables(Job, boolean)} is set to true)
* @param token the user's password
* @since 1.5.0
*/
public static void setConnectorInfo(Job job, String principal, AuthenticationToken token)
throws AccumuloSecurityException {
if (token instanceof KerberosToken) {
log.info("Received KerberosToken, attempting to fetch DelegationToken");
try {
Instance instance = getInstance(job);
Connector conn = instance.getConnector(principal, token);
token = conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
} catch (Exception e) {
log.warn(
"Failed to automatically obtain DelegationToken, Mappers/Reducers will likely fail to communicate with Accumulo",
e);
}
}
// DelegationTokens can be passed securely from user to task without serializing insecurely in
// the configuration
if (token instanceof DelegationTokenImpl) {
DelegationTokenImpl delegationToken = (DelegationTokenImpl) token;
// Convert it into a Hadoop Token
AuthenticationTokenIdentifier identifier = delegationToken.getIdentifier();
Token<AuthenticationTokenIdentifier> hadoopToken =
new Token<>(
identifier.getBytes(),
delegationToken.getPassword(),
identifier.getKind(),
delegationToken.getServiceName());
// Add the Hadoop Token to the Job so it gets serialized and passed along.
job.getCredentials().addToken(hadoopToken.getService(), hadoopToken);
}
OutputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, token);
}
/**
* Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric
* and underscores.
*
* <p>By default, this feature is <b>disabled</b>.
*
* @param job the Hadoop job instance to be configured
* @param enableFeature the feature is enabled if true, disabled otherwise
* @since 1.5.0
*/
public static void setCreateTables(Job job, boolean enableFeature) {
OutputConfigurator.setCreateTables(CLASS, job.getConfiguration(), enableFeature);
}
/**
* Gets the {@link BatchWriterConfig} settings.
*
* @param context the Hadoop context for the configured job
* @return the configuration object
* @since 1.5.0
* @see #setBatchWriterOptions(Job, BatchWriterConfig)
*/
protected static BatchWriterConfig getBatchWriterOptions(JobContext context) {
return OutputConfigurator.getBatchWriterOptions(CLASS, context.getConfiguration());
}
/**
* Sets the configuration for for the job's {@link BatchWriter} instances. If not set, a new
* {@link BatchWriterConfig}, with sensible built-in defaults is used. Setting the configuration
* multiple times overwrites any previous configuration.
*
* @param job the Hadoop job instance to be configured
* @param bwConfig the configuration for the {@link BatchWriter}
* @since 1.5.0
*/
public static void setBatchWriterOptions(Job job, BatchWriterConfig bwConfig) {
OutputConfigurator.setBatchWriterOptions(CLASS, job.getConfiguration(), bwConfig);
}
/**
* Gets the default table name from the configuration.
*
* @param context the Hadoop context for the configured job
* @return the default table name
* @since 1.5.0
* @see #setDefaultTableName(Job, String)
*/
protected static String getDefaultTableName(JobContext context) {
return OutputConfigurator.getDefaultTableName(CLASS, context.getConfiguration());
}
/**
* Sets the default table name to use if one emits a null in place of a table name for a given
* mutation. Table names can only be alpha-numeric and underscores.
*
* @param job the Hadoop job instance to be configured
* @param tableName the table to use when the tablename is null in the write call
* @since 1.5.0
*/
public static void setDefaultTableName(Job job, String tableName) {
OutputConfigurator.setDefaultTableName(CLASS, job.getConfiguration(), tableName);
}
/**
* Sets the connector information needed to communicate with Accumulo in this job.
*
* <p>Stores the password in a file in HDFS and pulls that into the Distributed Cache in an
* attempt to be more secure than storing it in the Configuration.
*
* @param job the Hadoop job instance to be configured
* @param principal a valid Accumulo user name (user must have Table.CREATE permission if {@link
* #setCreateTables(Job, boolean)} is set to true)
* @param tokenFile the path to the token file
* @since 1.6.0
*/
public static void setConnectorInfo(Job job, String principal, String tokenFile)
throws AccumuloSecurityException {
OutputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, tokenFile);
}
/**
* Determines whether this feature is enabled.
*
* @param context the Hadoop context for the configured job
* @return true if the feature is enabled, false otherwise
* @since 1.5.0
* @see #setSimulationMode(Job, boolean)
*/
protected static Boolean getSimulationMode(JobContext context) {
return OutputConfigurator.getSimulationMode(CLASS, context.getConfiguration());
}
/**
* Initializes an Accumulo {@link Instance} based on the configuration.
*
* @param context the Hadoop context for the configured job
* @return an Accumulo instance
* @since 1.5.0
* @see #setZooKeeperInstance(Job, ClientConfiguration)
*/
protected static Instance getInstance(JobContext context) {
return OutputConfigurator.getInstance(CLASS, context.getConfiguration());
}
/**
* Configures a {@link org.apache.accumulo.core.client.mock.MockInstance} for this job.
*
* @param job the Hadoop job instance to be configured
* @param instanceName the Accumulo instance name
* @since 1.5.0
*/
@Deprecated
public static void setMockInstance(Job job, String instanceName) {
OutputConfigurator.setMockInstance(CLASS, job.getConfiguration(), instanceName);
}
/**
* Configures a {@link ZooKeeperInstance} for this job.
*
* @param job the Hadoop job instance to be configured
* @param clientConfig client configuration for specifying connection timeouts, SSL connection
* options, etc.
* @since 1.6.0
*/
public static void setZooKeeperInstance(Job job, ClientConfiguration clientConfig) {
OutputConfigurator.setZooKeeperInstance(CLASS, job.getConfiguration(), clientConfig);
}
/**
* Gets the authenticated token from either the specified token file or directly from the
* configuration, whichever was used when the job was configured.
*
* @param context the Hadoop context for the configured job
* @return the principal's authentication token
* @since 1.6.0
* @see #setConnectorInfo(Job, String, AuthenticationToken)
* @see #setConnectorInfo(Job, String, String)
*/
protected static AuthenticationToken getAuthenticationToken(JobContext context) {
AuthenticationToken token =
OutputConfigurator.getAuthenticationToken(CLASS, context.getConfiguration());
return ConfiguratorBase.unwrapAuthenticationToken(context, token);
}
/**
* Gets the user name from the configuration.
*
* @param context the Hadoop context for the configured job
* @return the user name
* @since 1.5.0
* @see #setConnectorInfo(Job, String, AuthenticationToken)
*/
protected static String getPrincipal(JobContext context) {
return OutputConfigurator.getPrincipal(CLASS, context.getConfiguration());
}
/**
* Determines if the connector has been configured.
*
* @param context the Hadoop context for the configured job
* @return true if the connector has been configured, false otherwise
* @since 1.5.0
* @see #setConnectorInfo(Job, String, AuthenticationToken)
*/
protected static Boolean isConnectorInfoSet(JobContext context) {
return OutputConfigurator.isConnectorInfoSet(CLASS, context.getConfiguration());
}
/**
* Determines whether tables are permitted to be created as needed.
*
* @param context the Hadoop context for the configured job
* @return true if the feature is disabled, false otherwise
* @since 1.5.0
* @see #setCreateTables(Job, boolean)
*/
protected static Boolean canCreateTables(JobContext context) {
return OutputConfigurator.canCreateTables(CLASS, context.getConfiguration());
}
/**
* Sets the directive to use simulation mode for this job. In simulation mode, no output is
* produced. This is useful for testing.
*
* <p>By default, this feature is <b>disabled</b>.
*
* @param job the Hadoop job instance to be configured
* @param enableFeature the feature is enabled if true, disabled otherwise
* @since 1.5.0
*/
public static void setSimulationMode(Job job, boolean enableFeature) {
OutputConfigurator.setSimulationMode(CLASS, job.getConfiguration(), enableFeature);
}
/**
* Sets the log level for this job.
*
* @param job the Hadoop job instance to be configured
* @param level the logging level
* @since 1.5.0
*/
public static void setLogLevel(Job job, Level level) {
OutputConfigurator.setLogLevel(CLASS, job.getConfiguration(), level);
}
/**
* Set all the initial parameters needed in this class for connectivity out to Accumulo.
*
* @param context
*/
private void initialize(JobContext context) { // Configuration conf){
Configuration conf = context.getConfiguration();
try {
// output zoom level
log.info("Working from zoom level = " + zoomLevel);
if (zoomLevel == -1) {
zoomLevel = Integer.parseInt(conf.get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_ZOOMLEVEL));
}
table = conf.get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_OUTPUT_TABLE);
username = conf.get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_USER);
instanceName = conf.get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_INSTANCE);
zooKeepers = conf.get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_ZOOKEEPERS);
String pl = conf.get(MrGeoConstants.MRGEO_PROTECTION_LEVEL);
if (pl != null) {
colViz = new ColumnVisibility(pl);
} else if (colViz == null) {
vizStr = conf.get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_VIZ);
if (vizStr == null) {
colViz = new ColumnVisibility();
} else {
colViz = new ColumnVisibility(vizStr);
}
}
password = conf.get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD);
String isEnc = conf.get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PWENCODED64, "false");
if (isEnc.equalsIgnoreCase("true")) {
password = Base64Utils.decodeToString(password);
}
if (_innerFormat != null) {
return;
}
_innerFormat = AccumuloOutputFormat.class.newInstance();
AuthenticationToken token = new PasswordToken(password.getBytes());
// log.info("Setting output with: u = " + username);
// log.info("Setting output with: p = " + password);
// log.info("Setting output with: i = " + instanceName);
// log.info("Setting output with: z = " + zooKeepers);
boolean connSet = ConfiguratorBase.isConnectorInfoSet(AccumuloOutputFormat.class, conf);
if (!connSet) {
// job not always available - do it how Accumulo does it
OutputConfigurator.setConnectorInfo(AccumuloOutputFormat.class, conf, username, token);
ClientConfiguration cc = ClientConfiguration.loadDefault().withInstance(instanceName);
cc.setProperty(ClientProperty.INSTANCE_ZK_HOST, zooKeepers);
OutputConfigurator.setZooKeeperInstance(AccumuloOutputFormat.class, conf, cc);
OutputConfigurator.setDefaultTableName(AccumuloOutputFormat.class, conf, table);
OutputConfigurator.setCreateTables(AccumuloOutputFormat.class, conf, true);
outputInfoSet = true;
}
} catch (InstantiationException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalAccessException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (AccumuloSecurityException ase) {
ase.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
} // end initialize
/**
* Gets the log level from this configuration.
*
* @param context the Hadoop context for the configured job
* @return the log level
* @since 1.5.0
* @see #setLogLevel(Job, Level)
*/
protected static Level getLogLevel(JobContext context) {
return OutputConfigurator.getLogLevel(CLASS, context.getConfiguration());
}