public void validatePlan() throws IOException, BindException, InjectionException {

    final Tang t = Tang.Factory.getTang();

    // TODO Use the AvroClassHierarchySerializer
    final ClassHierarchyProto.Node root;
    try (final InputStream chin = new FileInputStream(this.ch)) {
      root = ClassHierarchyProto.Node.parseFrom(chin);
    }

    final ClassHierarchy classHierarchy = new ProtocolBufferClassHierarchy(root);
    final ConfigurationBuilder cb = t.newConfigurationBuilder(classHierarchy);

    if (!inConfig.canRead()) {
      throw new IOException("Cannot read input config file: " + inConfig);
    }

    ConfigurationFile.addConfiguration(cb, inConfig);

    if (target != null) {
      final Injector i = t.newInjector(cb.build());
      final InjectionPlan<?> ip = i.getInjectionPlan(target);
      if (!ip.isInjectable()) {
        throw new InjectionException(target + " is not injectable: " + ip.toCantInjectString());
      }
    }

    ConfigurationFile.writeConfigurationFile(cb.build(), outConfig);

    //    Injector i = t.newInjector(cb.build());
    //    InjectionPlan<?> ip = i.getInjectionPlan(target);
    //    try (final OutputStream ipout = new FileOutputStream(injectionPlan)) {
    //      new ProtocolBufferInjectionPlan().serialize(ip).writeTo(ipout);
    //    }
  }
Пример #2
0
  /**
   * Parse command line arguments and create TANG configuration ready to be submitted to REEF.
   *
   * @param args Command line arguments, as passed into main().
   * @return (immutable) TANG Configuration object.
   * @throws BindException if configuration commandLineInjector fails.
   * @throws InjectionException if configuration commandLineInjector fails.
   * @throws IOException error reading the configuration.
   */
  private static Configuration getClientConfiguration(final String[] args)
      throws BindException, InjectionException, IOException {
    final Configuration commandLineConf = parseCommandLine(args);

    final Configuration clientConfiguration =
        ClientConfiguration.CONF
            .set(ClientConfiguration.ON_JOB_RUNNING, SuspendClient.RunningJobHandler.class)
            .set(ClientConfiguration.ON_JOB_FAILED, SuspendClient.FailedJobHandler.class)
            .set(ClientConfiguration.ON_JOB_COMPLETED, SuspendClient.CompletedJobHandler.class)
            .set(ClientConfiguration.ON_RUNTIME_ERROR, SuspendClient.RuntimeErrorHandler.class)
            .build();

    final Injector commandLineInjector = Tang.Factory.getTang().newInjector(commandLineConf);
    final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
    final Configuration runtimeConfiguration;
    if (isLocal) {
      LOG.log(Level.INFO, "Running on the local runtime");
      runtimeConfiguration =
          LocalRuntimeConfiguration.CONF
              .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
              .build();
    } else {
      LOG.log(Level.INFO, "Running on YARN");
      runtimeConfiguration = YarnClientConfiguration.CONF.build();
    }

    return Configurations.merge(
        runtimeConfiguration, clientConfiguration, cloneCommandLineConfiguration(commandLineConf));
  }
Пример #3
0
  /**
   * NameServer and NameLookupClient test.
   *
   * @throws Exception
   */
  @Test
  public void testNamingLookup() throws Exception {

    final String localAddress = localAddressProvider.getLocalAddress();
    LOG.log(Level.FINEST, this.name.getMethodName());

    // names
    final Map<Identifier, InetSocketAddress> idToAddrMap =
        new HashMap<Identifier, InetSocketAddress>();
    idToAddrMap.put(
        this.factory.getNewInstance("task1"), new InetSocketAddress(localAddress, 7001));
    idToAddrMap.put(
        this.factory.getNewInstance("task2"), new InetSocketAddress(localAddress, 7002));

    // run a server
    final Injector injector = Tang.Factory.getTang().newInjector();
    injector.bindVolatileParameter(
        NameServerParameters.NameServerIdentifierFactory.class, this.factory);
    injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);
    try (final NameServer server = injector.getInstance(NameServer.class)) {
      this.port = server.getPort();
      for (final Identifier id : idToAddrMap.keySet()) {
        server.register(id, idToAddrMap.get(id));
      }

      // run a client
      try (final NameLookupClient client =
          new NameLookupClient(
              localAddress,
              this.port,
              10000,
              this.factory,
              RETRY_COUNT,
              RETRY_TIMEOUT,
              new NameCache(this.TTL),
              this.localAddressProvider)) {

        final Identifier id1 = this.factory.getNewInstance("task1");
        final Identifier id2 = this.factory.getNewInstance("task2");

        final Map<Identifier, InetSocketAddress> respMap =
            new HashMap<Identifier, InetSocketAddress>();
        InetSocketAddress addr1 = client.lookup(id1);
        respMap.put(id1, addr1);
        InetSocketAddress addr2 = client.lookup(id2);
        respMap.put(id2, addr2);

        for (final Identifier id : respMap.keySet()) {
          LOG.log(Level.FINEST, "Mapping: {0} -> {1}", new Object[] {id, respMap.get(id)});
        }

        Assert.assertTrue(isEqual(idToAddrMap, respMap));
      }
    }
  }
Пример #4
0
 static {
   try {
     final Injector injector = Tang.Factory.getTang().newInjector();
     RETRY_COUNT = injector.getNamedInstance(NameResolverRetryCount.class);
     RETRY_TIMEOUT = injector.getNamedInstance(NameResolverRetryTimeout.class);
   } catch (final InjectionException ex) {
     final String msg = "Exception while trying to find default values for retryCount & Timeout";
     LOG.log(Level.SEVERE, msg, ex);
     throw new RuntimeException(msg, ex);
   }
 }
Пример #5
0
 private static Configuration cloneCommandLineConfiguration(final Configuration commandLineConf)
     throws InjectionException, BindException {
   final Injector injector = Tang.Factory.getTang().newInjector(commandLineConf);
   final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder();
   cb.bindNamedParameter(
       NumCycles.class, String.valueOf(injector.getNamedInstance(NumCycles.class)));
   cb.bindNamedParameter(Delay.class, String.valueOf(injector.getNamedInstance(Delay.class)));
   cb.bindNamedParameter(
       SuspendClientControl.Port.class,
       String.valueOf(injector.getNamedInstance(SuspendClientControl.Port.class)));
   return cb.build();
 }
Пример #6
0
  /**
   * Main method that runs the example.
   *
   * @param args command line parameters.
   */
  public static void main(final String[] args) {
    try {
      final Configuration config = getClientConfiguration(args);

      LOG.log(Level.INFO, "Configuration:\n--\n{0}--", Configurations.toString(config, true));

      final Injector injector = Tang.Factory.getTang().newInjector(config);
      final SuspendClient client = injector.getInstance(SuspendClient.class);

      client.submit();
      client.waitForCompletion();
      LOG.info("Done!");

    } catch (final BindException | IOException | InjectionException ex) {
      LOG.log(Level.SEVERE, "Cannot launch: configuration error", ex);
    } catch (final Exception ex) {
      LOG.log(Level.SEVERE, "Cleanup error", ex);
    }
  }
Пример #7
0
  /**
   * NameServer and NameClient test.
   *
   * @throws Exception
   */
  @Test
  public void testNameClient() throws Exception {

    LOG.log(Level.FINEST, this.name.getMethodName());

    final String localAddress = localAddressProvider.getLocalAddress();
    final Injector injector = Tang.Factory.getTang().newInjector();
    injector.bindVolatileParameter(
        NameServerParameters.NameServerIdentifierFactory.class, this.factory);
    injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);
    try (final NameServer server = injector.getInstance(NameServer.class)) {
      this.port = server.getPort();

      final Map<Identifier, InetSocketAddress> idToAddrMap =
          new HashMap<Identifier, InetSocketAddress>();
      idToAddrMap.put(
          this.factory.getNewInstance("task1"), new InetSocketAddress(localAddress, 7001));
      idToAddrMap.put(
          this.factory.getNewInstance("task2"), new InetSocketAddress(localAddress, 7002));

      // registration
      // invoke registration from the client side
      Configuration nameResolverConf =
          NameResolverConfiguration.CONF
              .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddress)
              .set(NameResolverConfiguration.NAME_SERVICE_PORT, this.port)
              .set(NameResolverConfiguration.CACHE_TIMEOUT, this.TTL)
              .set(NameResolverConfiguration.RETRY_TIMEOUT, RETRY_TIMEOUT)
              .set(NameResolverConfiguration.RETRY_COUNT, RETRY_COUNT)
              .build();

      try (final NameResolver client =
          Tang.Factory.getTang().newInjector(nameResolverConf).getInstance(NameClient.class)) {
        for (final Identifier id : idToAddrMap.keySet()) {
          client.register(id, idToAddrMap.get(id));
        }

        // wait
        final Set<Identifier> ids = idToAddrMap.keySet();
        busyWait(server, ids.size(), ids);

        // lookup
        final Identifier id1 = this.factory.getNewInstance("task1");
        final Identifier id2 = this.factory.getNewInstance("task2");

        final Map<Identifier, InetSocketAddress> respMap =
            new HashMap<Identifier, InetSocketAddress>();
        InetSocketAddress addr1 = client.lookup(id1);
        respMap.put(id1, addr1);
        InetSocketAddress addr2 = client.lookup(id2);
        respMap.put(id2, addr2);

        for (final Identifier id : respMap.keySet()) {
          LOG.log(Level.FINEST, "Mapping: {0} -> {1}", new Object[] {id, respMap.get(id)});
        }

        Assert.assertTrue(isEqual(idToAddrMap, respMap));

        // un-registration
        for (final Identifier id : idToAddrMap.keySet()) {
          client.unregister(id);
        }

        // wait
        busyWait(server, 0, ids);

        final Map<Identifier, InetSocketAddress> serverMap =
            new HashMap<Identifier, InetSocketAddress>();
        addr1 = server.lookup(id1);
        if (addr1 != null) {
          serverMap.put(id1, addr1);
        }
        addr2 = server.lookup(id1);
        if (addr2 != null) {
          serverMap.put(id2, addr2);
        }

        Assert.assertEquals(0, serverMap.size());
      }
    }
  }
Пример #8
0
  /**
   * NameServer and NameRegistryClient test.
   *
   * @throws Exception
   */
  @Test
  public void testNamingRegistry() throws Exception {

    LOG.log(Level.FINEST, this.name.getMethodName());

    final Injector injector = Tang.Factory.getTang().newInjector();
    injector.bindVolatileParameter(
        NameServerParameters.NameServerIdentifierFactory.class, this.factory);
    injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);
    try (final NameServer server = injector.getInstance(NameServer.class)) {
      this.port = server.getPort();
      final String localAddress = localAddressProvider.getLocalAddress();

      // names to start with
      final Map<Identifier, InetSocketAddress> idToAddrMap =
          new HashMap<Identifier, InetSocketAddress>();
      idToAddrMap.put(
          this.factory.getNewInstance("task1"), new InetSocketAddress(localAddress, 7001));
      idToAddrMap.put(
          this.factory.getNewInstance("task2"), new InetSocketAddress(localAddress, 7002));

      // registration
      // invoke registration from the client side
      try (final NameRegistryClient client =
          new NameRegistryClient(
              localAddress, this.port, this.factory, this.localAddressProvider)) {
        for (final Identifier id : idToAddrMap.keySet()) {
          client.register(id, idToAddrMap.get(id));
        }

        // wait
        final Set<Identifier> ids = idToAddrMap.keySet();
        busyWait(server, ids.size(), ids);

        // check the server side
        Map<Identifier, InetSocketAddress> serverMap = new HashMap<Identifier, InetSocketAddress>();
        Iterable<NameAssignment> nas = server.lookup(ids);

        for (final NameAssignment na : nas) {
          LOG.log(
              Level.FINEST,
              "Mapping: {0} -> {1}",
              new Object[] {na.getIdentifier(), na.getAddress()});
          serverMap.put(na.getIdentifier(), na.getAddress());
        }

        Assert.assertTrue(isEqual(idToAddrMap, serverMap));

        // un-registration
        for (final Identifier id : idToAddrMap.keySet()) {
          client.unregister(id);
        }

        // wait
        busyWait(server, 0, ids);

        serverMap = new HashMap<Identifier, InetSocketAddress>();
        nas = server.lookup(ids);
        for (final NameAssignment na : nas) {
          serverMap.put(na.getIdentifier(), na.getAddress());
        }

        Assert.assertEquals(0, serverMap.size());
      }
    }
  }
Пример #9
0
  /**
   * Test concurrent lookups (threads share a client).
   *
   * @throws Exception
   */
  @Test
  public void testConcurrentNamingLookup() throws Exception {

    LOG.log(Level.FINEST, this.name.getMethodName());

    final String localAddress = localAddressProvider.getLocalAddress();
    // test it 3 times to make failure likely
    for (int i = 0; i < 3; i++) {

      LOG.log(Level.FINEST, "test {0}", i);

      // names
      final Map<Identifier, InetSocketAddress> idToAddrMap =
          new HashMap<Identifier, InetSocketAddress>();
      idToAddrMap.put(
          this.factory.getNewInstance("task1"), new InetSocketAddress(localAddress, 7001));
      idToAddrMap.put(
          this.factory.getNewInstance("task2"), new InetSocketAddress(localAddress, 7002));
      idToAddrMap.put(
          this.factory.getNewInstance("task3"), new InetSocketAddress(localAddress, 7003));

      // run a server
      final Injector injector = Tang.Factory.getTang().newInjector();
      injector.bindVolatileParameter(
          NameServerParameters.NameServerIdentifierFactory.class, this.factory);
      injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);
      try (final NameServer server = injector.getInstance(NameServer.class)) {
        this.port = server.getPort();
        for (final Identifier id : idToAddrMap.keySet()) {
          server.register(id, idToAddrMap.get(id));
        }

        // run a client
        try (final NameLookupClient client =
            new NameLookupClient(
                localAddress,
                this.port,
                10000,
                this.factory,
                RETRY_COUNT,
                RETRY_TIMEOUT,
                new NameCache(this.TTL),
                this.localAddressProvider)) {

          final Identifier id1 = this.factory.getNewInstance("task1");
          final Identifier id2 = this.factory.getNewInstance("task2");
          final Identifier id3 = this.factory.getNewInstance("task3");

          final ExecutorService e = Executors.newCachedThreadPool();

          final ConcurrentMap<Identifier, InetSocketAddress> respMap =
              new ConcurrentHashMap<Identifier, InetSocketAddress>();

          final Future<?> f1 =
              e.submit(
                  new Runnable() {
                    @Override
                    public void run() {
                      InetSocketAddress addr = null;
                      try {
                        addr = client.lookup(id1);
                      } catch (final Exception e) {
                        LOG.log(Level.SEVERE, "Lookup failed", e);
                        Assert.fail(e.toString());
                      }
                      respMap.put(id1, addr);
                    }
                  });
          final Future<?> f2 =
              e.submit(
                  new Runnable() {
                    @Override
                    public void run() {
                      InetSocketAddress addr = null;
                      try {
                        addr = client.lookup(id2);
                      } catch (final Exception e) {
                        LOG.log(Level.SEVERE, "Lookup failed", e);
                        Assert.fail(e.toString());
                      }
                      respMap.put(id2, addr);
                    }
                  });
          final Future<?> f3 =
              e.submit(
                  new Runnable() {
                    @Override
                    public void run() {
                      InetSocketAddress addr = null;
                      try {
                        addr = client.lookup(id3);
                      } catch (final Exception e) {
                        LOG.log(Level.SEVERE, "Lookup failed", e);
                        Assert.fail(e.toString());
                      }
                      respMap.put(id3, addr);
                    }
                  });

          f1.get();
          f2.get();
          f3.get();

          for (final Identifier id : respMap.keySet()) {
            LOG.log(Level.FINEST, "Mapping: {0} -> {1}", new Object[] {id, respMap.get(id)});
          }

          Assert.assertTrue(isEqual(idToAddrMap, respMap));
        }
      }
    }
  }