/**
* Writes driver configuration to disk.
*
* @param jobFolder The folder in which the job is staged.
* @param jobId id of the job to be submitted
* @param clientRemoteId The client remote id
* @return the configuration
* @throws IOException
*/
public Configuration writeConfiguration(
final File jobFolder, final String jobId, final String clientRemoteId) throws IOException {
final File driverFolder = new File(jobFolder, PreparedDriverFolderLauncher.DRIVER_FOLDER_NAME);
final Configuration driverConfiguration1 =
driverConfigurationProvider.getDriverConfiguration(
jobFolder.toURI(),
clientRemoteId,
jobId,
Constants.DRIVER_CONFIGURATION_WITH_HTTP_AND_NAMESERVER);
final ConfigurationBuilder configurationBuilder =
Tang.Factory.getTang().newConfigurationBuilder();
for (final ConfigurationProvider configurationProvider : this.configurationProviders) {
configurationBuilder.addConfiguration(configurationProvider.getConfiguration());
}
final Configuration providedConfigurations = configurationBuilder.build();
final Configuration driverConfiguration =
Configurations.merge(
driverConfiguration1,
Tang.Factory.getTang()
.newConfigurationBuilder()
.bindNamedParameter(JobSubmissionDirectory.class, driverFolder.toString())
.build(),
providedConfigurations);
final File driverConfigurationFile =
new File(driverFolder, fileNames.getDriverConfigurationPath());
configurationSerializer.toFile(driverConfiguration, driverConfigurationFile);
return driverConfiguration;
}
private static Configuration cloneCommandLineConfiguration(final Configuration commandLineConf)
throws InjectionException, BindException {
final Injector injector = Tang.Factory.getTang().newInjector(commandLineConf);
final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder();
cb.bindNamedParameter(
NumCycles.class, String.valueOf(injector.getNamedInstance(NumCycles.class)));
cb.bindNamedParameter(Delay.class, String.valueOf(injector.getNamedInstance(Delay.class)));
cb.bindNamedParameter(
SuspendClientControl.Port.class,
String.valueOf(injector.getNamedInstance(SuspendClientControl.Port.class)));
return cb.build();
}
public static void main(String[] argv) throws IOException, BindException, InjectionException {
@SuppressWarnings("unchecked")
JavaConfigurationBuilder cb =
Tang.Factory.getTang()
.newConfigurationBuilder(
(Class<? extends ExternalConstructor<?>>[]) new Class[] {FileParser.class});
CommandLine cl = new CommandLine(cb);
cl.processCommandLine(
argv, Target.class, ClassHierarchyIn.class, ConfigurationIn.class, ConfigurationOut.class);
ValidateConfiguration bip =
Tang.Factory.getTang().newInjector(cb.build()).getInstance(ValidateConfiguration.class);
bip.validatePlan();
}
public static void main(final String[] args) throws InjectionException, IOException {
final File localAppSubmissionParametersFile = new File(args[0]);
final File jobSubmissionParametersFile = new File(args[1]);
if (!(jobSubmissionParametersFile.exists() && jobSubmissionParametersFile.canRead())) {
throw new IOException(
"Unable to open and read " + jobSubmissionParametersFile.getAbsolutePath());
}
if (!(localAppSubmissionParametersFile.exists()
&& localAppSubmissionParametersFile.canRead())) {
throw new IOException(
"Unable to open and read " + localAppSubmissionParametersFile.getAbsolutePath());
}
final LocalSubmissionFromCS localSubmission =
LocalSubmissionFromCS.fromSubmissionParameterFiles(
jobSubmissionParametersFile, localAppSubmissionParametersFile);
LOG.log(Level.FINE, "Local driver config generation received from C#: {0}", localSubmission);
final Configuration localRuntimeConfiguration = localSubmission.getRuntimeConfiguration();
final LocalRuntimeDriverConfigurationGenerator localConfigurationGenerator =
Tang.Factory.getTang()
.newInjector(localRuntimeConfiguration)
.getInstance(LocalRuntimeDriverConfigurationGenerator.class);
localConfigurationGenerator.writeConfiguration(
localSubmission.getJobFolder(), localSubmission.getJobId(), ClientRemoteIdentifier.NONE);
}
/**
* Parse command line arguments and create TANG configuration ready to be submitted to REEF.
*
* @param args Command line arguments, as passed into main().
* @return (immutable) TANG Configuration object.
* @throws BindException if configuration commandLineInjector fails.
* @throws InjectionException if configuration commandLineInjector fails.
* @throws IOException error reading the configuration.
*/
private static Configuration getClientConfiguration(final String[] args)
throws BindException, InjectionException, IOException {
final Configuration commandLineConf = parseCommandLine(args);
final Configuration clientConfiguration =
ClientConfiguration.CONF
.set(ClientConfiguration.ON_JOB_RUNNING, SuspendClient.RunningJobHandler.class)
.set(ClientConfiguration.ON_JOB_FAILED, SuspendClient.FailedJobHandler.class)
.set(ClientConfiguration.ON_JOB_COMPLETED, SuspendClient.CompletedJobHandler.class)
.set(ClientConfiguration.ON_RUNTIME_ERROR, SuspendClient.RuntimeErrorHandler.class)
.build();
final Injector commandLineInjector = Tang.Factory.getTang().newInjector(commandLineConf);
final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
final Configuration runtimeConfiguration;
if (isLocal) {
LOG.log(Level.INFO, "Running on the local runtime");
runtimeConfiguration =
LocalRuntimeConfiguration.CONF
.set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
.build();
} else {
LOG.log(Level.INFO, "Running on YARN");
runtimeConfiguration = YarnClientConfiguration.CONF.build();
}
return Configurations.merge(
runtimeConfiguration, clientConfiguration, cloneCommandLineConfiguration(commandLineConf));
}
public void validatePlan() throws IOException, BindException, InjectionException {
final Tang t = Tang.Factory.getTang();
// TODO Use the AvroClassHierarchySerializer
final ClassHierarchyProto.Node root;
try (final InputStream chin = new FileInputStream(this.ch)) {
root = ClassHierarchyProto.Node.parseFrom(chin);
}
final ClassHierarchy classHierarchy = new ProtocolBufferClassHierarchy(root);
final ConfigurationBuilder cb = t.newConfigurationBuilder(classHierarchy);
if (!inConfig.canRead()) {
throw new IOException("Cannot read input config file: " + inConfig);
}
ConfigurationFile.addConfiguration(cb, inConfig);
if (target != null) {
final Injector i = t.newInjector(cb.build());
final InjectionPlan<?> ip = i.getInjectionPlan(target);
if (!ip.isInjectable()) {
throw new InjectionException(target + " is not injectable: " + ip.toCantInjectString());
}
}
ConfigurationFile.writeConfigurationFile(cb.build(), outConfig);
// Injector i = t.newInjector(cb.build());
// InjectionPlan<?> ip = i.getInjectionPlan(target);
// try (final OutputStream ipout = new FileOutputStream(injectionPlan)) {
// new ProtocolBufferInjectionPlan().serialize(ip).writeTo(ipout);
// }
}
/**
* @param args command line arguments, as passed to main()
* @return Configuration object.
*/
private static Configuration parseCommandLine(final String[] args)
throws IOException, BindException {
final JavaConfigurationBuilder confBuilder = Tang.Factory.getTang().newConfigurationBuilder();
final CommandLine cl = new CommandLine(confBuilder);
cl.registerShortNameOfClass(Local.class);
cl.registerShortNameOfClass(NumCycles.class);
cl.registerShortNameOfClass(Delay.class);
cl.registerShortNameOfClass(SuspendClientControl.Port.class);
cl.processCommandLine(args);
return confBuilder.build();
}
/**
* Main method that runs the example.
*
* @param args command line parameters.
*/
public static void main(final String[] args) {
try {
final Configuration config = getClientConfiguration(args);
LOG.log(Level.INFO, "Configuration:\n--\n{0}--", Configurations.toString(config, true));
final Injector injector = Tang.Factory.getTang().newInjector(config);
final SuspendClient client = injector.getInstance(SuspendClient.class);
client.submit();
client.waitForCompletion();
LOG.info("Done!");
} catch (final BindException | IOException | InjectionException ex) {
LOG.log(Level.SEVERE, "Cannot launch: configuration error", ex);
} catch (final Exception ex) {
LOG.log(Level.SEVERE, "Cleanup error", ex);
}
}
