public void validatePlan() throws IOException, BindException, InjectionException {
final Tang t = Tang.Factory.getTang();
// TODO Use the AvroClassHierarchySerializer
final ClassHierarchyProto.Node root;
try (final InputStream chin = new FileInputStream(this.ch)) {
root = ClassHierarchyProto.Node.parseFrom(chin);
}
final ClassHierarchy classHierarchy = new ProtocolBufferClassHierarchy(root);
final ConfigurationBuilder cb = t.newConfigurationBuilder(classHierarchy);
if (!inConfig.canRead()) {
throw new IOException("Cannot read input config file: " + inConfig);
}
ConfigurationFile.addConfiguration(cb, inConfig);
if (target != null) {
final Injector i = t.newInjector(cb.build());
final InjectionPlan<?> ip = i.getInjectionPlan(target);
if (!ip.isInjectable()) {
throw new InjectionException(target + " is not injectable: " + ip.toCantInjectString());
}
}
ConfigurationFile.writeConfigurationFile(cb.build(), outConfig);
// Injector i = t.newInjector(cb.build());
// InjectionPlan<?> ip = i.getInjectionPlan(target);
// try (final OutputStream ipout = new FileOutputStream(injectionPlan)) {
// new ProtocolBufferInjectionPlan().serialize(ip).writeTo(ipout);
// }
}
/**
* Parse command line arguments and create TANG configuration ready to be submitted to REEF.
*
* @param args Command line arguments, as passed into main().
* @return (immutable) TANG Configuration object.
* @throws BindException if configuration commandLineInjector fails.
* @throws InjectionException if configuration commandLineInjector fails.
* @throws IOException error reading the configuration.
*/
private static Configuration getClientConfiguration(final String[] args)
throws BindException, InjectionException, IOException {
final Configuration commandLineConf = parseCommandLine(args);
final Configuration clientConfiguration =
ClientConfiguration.CONF
.set(ClientConfiguration.ON_JOB_RUNNING, SuspendClient.RunningJobHandler.class)
.set(ClientConfiguration.ON_JOB_FAILED, SuspendClient.FailedJobHandler.class)
.set(ClientConfiguration.ON_JOB_COMPLETED, SuspendClient.CompletedJobHandler.class)
.set(ClientConfiguration.ON_RUNTIME_ERROR, SuspendClient.RuntimeErrorHandler.class)
.build();
final Injector commandLineInjector = Tang.Factory.getTang().newInjector(commandLineConf);
final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
final Configuration runtimeConfiguration;
if (isLocal) {
LOG.log(Level.INFO, "Running on the local runtime");
runtimeConfiguration =
LocalRuntimeConfiguration.CONF
.set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
.build();
} else {
LOG.log(Level.INFO, "Running on YARN");
runtimeConfiguration = YarnClientConfiguration.CONF.build();
}
return Configurations.merge(
runtimeConfiguration, clientConfiguration, cloneCommandLineConfiguration(commandLineConf));
}
/**
* NameServer and NameLookupClient test.
*
* @throws Exception
*/
@Test
public void testNamingLookup() throws Exception {
final String localAddress = localAddressProvider.getLocalAddress();
LOG.log(Level.FINEST, this.name.getMethodName());
// names
final Map<Identifier, InetSocketAddress> idToAddrMap =
new HashMap<Identifier, InetSocketAddress>();
idToAddrMap.put(
this.factory.getNewInstance("task1"), new InetSocketAddress(localAddress, 7001));
idToAddrMap.put(
this.factory.getNewInstance("task2"), new InetSocketAddress(localAddress, 7002));
// run a server
final Injector injector = Tang.Factory.getTang().newInjector();
injector.bindVolatileParameter(
NameServerParameters.NameServerIdentifierFactory.class, this.factory);
injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);
try (final NameServer server = injector.getInstance(NameServer.class)) {
this.port = server.getPort();
for (final Identifier id : idToAddrMap.keySet()) {
server.register(id, idToAddrMap.get(id));
}
// run a client
try (final NameLookupClient client =
new NameLookupClient(
localAddress,
this.port,
10000,
this.factory,
RETRY_COUNT,
RETRY_TIMEOUT,
new NameCache(this.TTL),
this.localAddressProvider)) {
final Identifier id1 = this.factory.getNewInstance("task1");
final Identifier id2 = this.factory.getNewInstance("task2");
final Map<Identifier, InetSocketAddress> respMap =
new HashMap<Identifier, InetSocketAddress>();
InetSocketAddress addr1 = client.lookup(id1);
respMap.put(id1, addr1);
InetSocketAddress addr2 = client.lookup(id2);
respMap.put(id2, addr2);
for (final Identifier id : respMap.keySet()) {
LOG.log(Level.FINEST, "Mapping: {0} -> {1}", new Object[] {id, respMap.get(id)});
}
Assert.assertTrue(isEqual(idToAddrMap, respMap));
}
}
}
static {
try {
final Injector injector = Tang.Factory.getTang().newInjector();
RETRY_COUNT = injector.getNamedInstance(NameResolverRetryCount.class);
RETRY_TIMEOUT = injector.getNamedInstance(NameResolverRetryTimeout.class);
} catch (final InjectionException ex) {
final String msg = "Exception while trying to find default values for retryCount & Timeout";
LOG.log(Level.SEVERE, msg, ex);
throw new RuntimeException(msg, ex);
}
}
private static Configuration cloneCommandLineConfiguration(final Configuration commandLineConf)
throws InjectionException, BindException {
final Injector injector = Tang.Factory.getTang().newInjector(commandLineConf);
final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder();
cb.bindNamedParameter(
NumCycles.class, String.valueOf(injector.getNamedInstance(NumCycles.class)));
cb.bindNamedParameter(Delay.class, String.valueOf(injector.getNamedInstance(Delay.class)));
cb.bindNamedParameter(
SuspendClientControl.Port.class,
String.valueOf(injector.getNamedInstance(SuspendClientControl.Port.class)));
return cb.build();
}
/**
* Main method that runs the example.
*
* @param args command line parameters.
*/
public static void main(final String[] args) {
try {
final Configuration config = getClientConfiguration(args);
LOG.log(Level.INFO, "Configuration:\n--\n{0}--", Configurations.toString(config, true));
final Injector injector = Tang.Factory.getTang().newInjector(config);
final SuspendClient client = injector.getInstance(SuspendClient.class);
client.submit();
client.waitForCompletion();
LOG.info("Done!");
} catch (final BindException | IOException | InjectionException ex) {
LOG.log(Level.SEVERE, "Cannot launch: configuration error", ex);
} catch (final Exception ex) {
LOG.log(Level.SEVERE, "Cleanup error", ex);
}
}
/**
* NameServer and NameClient test.
*
* @throws Exception
*/
@Test
public void testNameClient() throws Exception {
LOG.log(Level.FINEST, this.name.getMethodName());
final String localAddress = localAddressProvider.getLocalAddress();
final Injector injector = Tang.Factory.getTang().newInjector();
injector.bindVolatileParameter(
NameServerParameters.NameServerIdentifierFactory.class, this.factory);
injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);
try (final NameServer server = injector.getInstance(NameServer.class)) {
this.port = server.getPort();
final Map<Identifier, InetSocketAddress> idToAddrMap =
new HashMap<Identifier, InetSocketAddress>();
idToAddrMap.put(
this.factory.getNewInstance("task1"), new InetSocketAddress(localAddress, 7001));
idToAddrMap.put(
this.factory.getNewInstance("task2"), new InetSocketAddress(localAddress, 7002));
// registration
// invoke registration from the client side
Configuration nameResolverConf =
NameResolverConfiguration.CONF
.set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddress)
.set(NameResolverConfiguration.NAME_SERVICE_PORT, this.port)
.set(NameResolverConfiguration.CACHE_TIMEOUT, this.TTL)
.set(NameResolverConfiguration.RETRY_TIMEOUT, RETRY_TIMEOUT)
.set(NameResolverConfiguration.RETRY_COUNT, RETRY_COUNT)
.build();
try (final NameResolver client =
Tang.Factory.getTang().newInjector(nameResolverConf).getInstance(NameClient.class)) {
for (final Identifier id : idToAddrMap.keySet()) {
client.register(id, idToAddrMap.get(id));
}
// wait
final Set<Identifier> ids = idToAddrMap.keySet();
busyWait(server, ids.size(), ids);
// lookup
final Identifier id1 = this.factory.getNewInstance("task1");
final Identifier id2 = this.factory.getNewInstance("task2");
final Map<Identifier, InetSocketAddress> respMap =
new HashMap<Identifier, InetSocketAddress>();
InetSocketAddress addr1 = client.lookup(id1);
respMap.put(id1, addr1);
InetSocketAddress addr2 = client.lookup(id2);
respMap.put(id2, addr2);
for (final Identifier id : respMap.keySet()) {
LOG.log(Level.FINEST, "Mapping: {0} -> {1}", new Object[] {id, respMap.get(id)});
}
Assert.assertTrue(isEqual(idToAddrMap, respMap));
// un-registration
for (final Identifier id : idToAddrMap.keySet()) {
client.unregister(id);
}
// wait
busyWait(server, 0, ids);
final Map<Identifier, InetSocketAddress> serverMap =
new HashMap<Identifier, InetSocketAddress>();
addr1 = server.lookup(id1);
if (addr1 != null) {
serverMap.put(id1, addr1);
}
addr2 = server.lookup(id1);
if (addr2 != null) {
serverMap.put(id2, addr2);
}
Assert.assertEquals(0, serverMap.size());
}
}
}
/**
* NameServer and NameRegistryClient test.
*
* @throws Exception
*/
@Test
public void testNamingRegistry() throws Exception {
LOG.log(Level.FINEST, this.name.getMethodName());
final Injector injector = Tang.Factory.getTang().newInjector();
injector.bindVolatileParameter(
NameServerParameters.NameServerIdentifierFactory.class, this.factory);
injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);
try (final NameServer server = injector.getInstance(NameServer.class)) {
this.port = server.getPort();
final String localAddress = localAddressProvider.getLocalAddress();
// names to start with
final Map<Identifier, InetSocketAddress> idToAddrMap =
new HashMap<Identifier, InetSocketAddress>();
idToAddrMap.put(
this.factory.getNewInstance("task1"), new InetSocketAddress(localAddress, 7001));
idToAddrMap.put(
this.factory.getNewInstance("task2"), new InetSocketAddress(localAddress, 7002));
// registration
// invoke registration from the client side
try (final NameRegistryClient client =
new NameRegistryClient(
localAddress, this.port, this.factory, this.localAddressProvider)) {
for (final Identifier id : idToAddrMap.keySet()) {
client.register(id, idToAddrMap.get(id));
}
// wait
final Set<Identifier> ids = idToAddrMap.keySet();
busyWait(server, ids.size(), ids);
// check the server side
Map<Identifier, InetSocketAddress> serverMap = new HashMap<Identifier, InetSocketAddress>();
Iterable<NameAssignment> nas = server.lookup(ids);
for (final NameAssignment na : nas) {
LOG.log(
Level.FINEST,
"Mapping: {0} -> {1}",
new Object[] {na.getIdentifier(), na.getAddress()});
serverMap.put(na.getIdentifier(), na.getAddress());
}
Assert.assertTrue(isEqual(idToAddrMap, serverMap));
// un-registration
for (final Identifier id : idToAddrMap.keySet()) {
client.unregister(id);
}
// wait
busyWait(server, 0, ids);
serverMap = new HashMap<Identifier, InetSocketAddress>();
nas = server.lookup(ids);
for (final NameAssignment na : nas) {
serverMap.put(na.getIdentifier(), na.getAddress());
}
Assert.assertEquals(0, serverMap.size());
}
}
}
/**
* Test concurrent lookups (threads share a client).
*
* @throws Exception
*/
@Test
public void testConcurrentNamingLookup() throws Exception {
LOG.log(Level.FINEST, this.name.getMethodName());
final String localAddress = localAddressProvider.getLocalAddress();
// test it 3 times to make failure likely
for (int i = 0; i < 3; i++) {
LOG.log(Level.FINEST, "test {0}", i);
// names
final Map<Identifier, InetSocketAddress> idToAddrMap =
new HashMap<Identifier, InetSocketAddress>();
idToAddrMap.put(
this.factory.getNewInstance("task1"), new InetSocketAddress(localAddress, 7001));
idToAddrMap.put(
this.factory.getNewInstance("task2"), new InetSocketAddress(localAddress, 7002));
idToAddrMap.put(
this.factory.getNewInstance("task3"), new InetSocketAddress(localAddress, 7003));
// run a server
final Injector injector = Tang.Factory.getTang().newInjector();
injector.bindVolatileParameter(
NameServerParameters.NameServerIdentifierFactory.class, this.factory);
injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);
try (final NameServer server = injector.getInstance(NameServer.class)) {
this.port = server.getPort();
for (final Identifier id : idToAddrMap.keySet()) {
server.register(id, idToAddrMap.get(id));
}
// run a client
try (final NameLookupClient client =
new NameLookupClient(
localAddress,
this.port,
10000,
this.factory,
RETRY_COUNT,
RETRY_TIMEOUT,
new NameCache(this.TTL),
this.localAddressProvider)) {
final Identifier id1 = this.factory.getNewInstance("task1");
final Identifier id2 = this.factory.getNewInstance("task2");
final Identifier id3 = this.factory.getNewInstance("task3");
final ExecutorService e = Executors.newCachedThreadPool();
final ConcurrentMap<Identifier, InetSocketAddress> respMap =
new ConcurrentHashMap<Identifier, InetSocketAddress>();
final Future<?> f1 =
e.submit(
new Runnable() {
@Override
public void run() {
InetSocketAddress addr = null;
try {
addr = client.lookup(id1);
} catch (final Exception e) {
LOG.log(Level.SEVERE, "Lookup failed", e);
Assert.fail(e.toString());
}
respMap.put(id1, addr);
}
});
final Future<?> f2 =
e.submit(
new Runnable() {
@Override
public void run() {
InetSocketAddress addr = null;
try {
addr = client.lookup(id2);
} catch (final Exception e) {
LOG.log(Level.SEVERE, "Lookup failed", e);
Assert.fail(e.toString());
}
respMap.put(id2, addr);
}
});
final Future<?> f3 =
e.submit(
new Runnable() {
@Override
public void run() {
InetSocketAddress addr = null;
try {
addr = client.lookup(id3);
} catch (final Exception e) {
LOG.log(Level.SEVERE, "Lookup failed", e);
Assert.fail(e.toString());
}
respMap.put(id3, addr);
}
});
f1.get();
f2.get();
f3.get();
for (final Identifier id : respMap.keySet()) {
LOG.log(Level.FINEST, "Mapping: {0} -> {1}", new Object[] {id, respMap.get(id)});
}
Assert.assertTrue(isEqual(idToAddrMap, respMap));
}
}
}
}