@Parameters
public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {
LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
for (int i = 1; i <= NUM_PROGRAMS; i++) {
Configuration config = new Configuration();
config.setInteger("ProgramId", i);
tConfigs.add(config);
}
return toParameterList(tConfigs);
}
/**
* Configures this JDBCOutputFormat.
*
* @param parameters Configuration containing all parameters.
*/
@Override
public void configure(Configuration parameters) {
this.driverName = parameters.getString(DRIVER_KEY, null);
this.username = parameters.getString(USERNAME_KEY, null);
this.password = parameters.getString(PASSWORD_KEY, null);
this.dbURL = parameters.getString(URL_KEY, null);
this.query = parameters.getString(QUERY_KEY, null);
this.fieldCount = parameters.getInteger(FIELD_COUNT_KEY, 0);
this.batchInterval = parameters.getInteger(BATCH_INTERVAL, DEFAULT_BATCH_INTERVERAL);
@SuppressWarnings("unchecked")
Class<Value>[] classes = new Class[this.fieldCount];
this.fieldClasses = classes;
for (int i = 0; i < this.fieldCount; i++) {
@SuppressWarnings("unchecked")
Class<? extends Value> clazz =
(Class<? extends Value>) parameters.getClass(FIELD_TYPE_KEY + i, null);
if (clazz == null) {
throw new IllegalArgumentException(
"Invalid configuration for JDBCOutputFormat: " + "No type class for parameter " + i);
}
this.fieldClasses[i] = clazz;
}
}
/**
* Registers an newly incoming runtime task with the task manager.
*
* @param id the ID of the task to register
* @param jobConfiguration the job configuration that has been attached to the original job graph
* @param environment the environment of the task to be registered
* @return the task to be started or <code>null</code> if a task with the same ID was already
* running
*/
private Task createAndRegisterTask(
final ExecutionVertexID id,
final Configuration jobConfiguration,
final RuntimeEnvironment environment)
throws InsufficientResourcesException, IOException {
if (id == null) {
throw new IllegalArgumentException("Argument id is null");
}
if (environment == null) {
throw new IllegalArgumentException("Argument environment is null");
}
// Task creation and registration must be atomic
Task task;
synchronized (this) {
final Task runningTask = this.runningTasks.get(id);
boolean registerTask = true;
if (runningTask == null) {
task = new Task(id, environment, this);
} else {
if (runningTask instanceof Task) {
// Task is already running
return null;
} else {
// There is already a replay task running, we will simply restart it
task = runningTask;
registerTask = false;
}
}
if (registerTask) {
// Register the task with the byte buffered channel manager
this.channelManager.register(task);
boolean enableProfiling = false;
if (this.profiler != null
&& jobConfiguration.getBoolean(ProfilingUtils.PROFILE_JOB_KEY, true)) {
enableProfiling = true;
}
// Register environment, input, and output gates for profiling
if (enableProfiling) {
task.registerProfiler(this.profiler, jobConfiguration);
}
this.runningTasks.put(id, task);
}
}
return task;
}
@Override
public void configure(Configuration parameters) {
super.configure(parameters);
// read own parameters
this.blockSize = parameters.getLong(BLOCK_SIZE_PARAMETER_KEY, NATIVE_BLOCK_SIZE);
if (this.blockSize < 1 && this.blockSize != NATIVE_BLOCK_SIZE)
throw new IllegalArgumentException("The block size parameter must be set and larger than 0.");
if (this.blockSize > Integer.MAX_VALUE)
throw new UnsupportedOperationException(
"Currently only block size up to Integer.MAX_VALUE are supported");
}
@Override
public void configure(Configuration parameters) {
super.configure(parameters);
// get the charset for the decoding
String charsetName = parameters.getString(CHARSET_NAME, DEFAULT_CHARSET_NAME);
if (charsetName == null || !Charset.isSupported(charsetName)) {
throw new RuntimeException("Unsupported charset: " + charsetName);
}
if (charsetName.equals("ISO-8859-1") || charsetName.equalsIgnoreCase("ASCII")) {
this.ascii = true;
} else {
this.decoder = Charset.forName(charsetName).newDecoder();
this.byteWrapper = ByteBuffer.allocate(1);
}
// get the field position to write in the record
this.pos = parameters.getInteger(FIELD_POS, 0);
if (this.pos < 0) {
throw new RuntimeException(
"Illegal configuration value for the target position: " + this.pos);
}
}
/**
* Entry point for the program.
*
* @param args arguments from the command line
* @throws IOException
*/
@SuppressWarnings("static-access")
public static void main(String[] args) throws IOException {
Option configDirOpt =
OptionBuilder.withArgName("config directory")
.hasArg()
.withDescription("Specify configuration directory.")
.create("configDir");
// tempDir option is used by the YARN client.
Option tempDir =
OptionBuilder.withArgName("temporary directory (overwrites configured option)")
.hasArg()
.withDescription("Specify temporary directory.")
.create(ARG_CONF_DIR);
configDirOpt.setRequired(true);
tempDir.setRequired(false);
Options options = new Options();
options.addOption(configDirOpt);
options.addOption(tempDir);
CommandLineParser parser = new GnuParser();
CommandLine line = null;
try {
line = parser.parse(options, args);
} catch (ParseException e) {
System.err.println("CLI Parsing failed. Reason: " + e.getMessage());
System.exit(FAILURE_RETURN_CODE);
}
String configDir = line.getOptionValue(configDirOpt.getOpt(), null);
String tempDirVal = line.getOptionValue(tempDir.getOpt(), null);
// First, try to load global configuration
GlobalConfiguration.loadConfiguration(configDir);
if (tempDirVal != null // the YARN TM runner has set a value for the temp dir
// the configuration does not contain a temp direcory
&& GlobalConfiguration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, null) == null) {
Configuration c = GlobalConfiguration.getConfiguration();
c.setString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, tempDirVal);
LOG.info("Setting temporary directory to " + tempDirVal);
GlobalConfiguration.includeConfiguration(c);
}
System.err.println("Configuration " + GlobalConfiguration.getConfiguration());
LOG.info("Current user " + UserGroupInformation.getCurrentUser().getShortUserName());
{
// log the available JVM memory
long maxMemoryMiBytes = Runtime.getRuntime().maxMemory() >>> 20;
LOG.info(
"Starting TaskManager in a JVM with " + maxMemoryMiBytes + " MiBytes maximum heap size.");
}
// Create a new task manager object
try {
new TaskManager(ExecutionMode.CLUSTER);
} catch (Exception e) {
LOG.fatal("Taskmanager startup failed: " + e.getMessage(), e);
System.exit(FAILURE_RETURN_CODE);
}
// park the main thread to keep the JVM alive (all other threads may be daemon threads)
Object mon = new Object();
synchronized (mon) {
try {
mon.wait();
} catch (InterruptedException ex) {
}
}
}
public boolean usesConvergenceCriterion() {
return config.getString(ITERATION_CONVERGENCE_CRITERION, null) != null;
}