コード例 #1
0
ファイル: StramClient.java プロジェクト: andyperlitch/Apex
  public static LinkedHashSet<String> findJars(LogicalPlan dag, Class<?>[] defaultClasses) {
    List<Class<?>> jarClasses = new ArrayList<Class<?>>();

    for (String className : dag.getClassNames()) {
      try {
        Class<?> clazz = Thread.currentThread().getContextClassLoader().loadClass(className);
        jarClasses.add(clazz);
      } catch (ClassNotFoundException e) {
        throw new IllegalArgumentException("Failed to load class " + className, e);
      }
    }

    for (Class<?> clazz : Lists.newArrayList(jarClasses)) {
      // process class and super classes (super does not require deploy annotation)
      for (Class<?> c = clazz; c != Object.class && c != null; c = c.getSuperclass()) {
        // LOG.debug("checking " + c);
        jarClasses.add(c);
        jarClasses.addAll(Arrays.asList(c.getInterfaces()));
      }
    }

    jarClasses.addAll(Arrays.asList(defaultClasses));

    if (dag.isDebug()) {
      LOG.debug("Deploy dependencies: {}", jarClasses);
    }

    LinkedHashSet<String> localJarFiles = new LinkedHashSet<String>(); // avoid duplicates
    HashMap<String, String> sourceToJar = new HashMap<String, String>();

    for (Class<?> jarClass : jarClasses) {
      if (jarClass.getProtectionDomain().getCodeSource() == null) {
        // system class
        continue;
      }
      String sourceLocation =
          jarClass.getProtectionDomain().getCodeSource().getLocation().toString();
      String jar = sourceToJar.get(sourceLocation);
      if (jar == null) {
        // don't create jar file from folders multiple times
        jar = JarFinder.getJar(jarClass);
        sourceToJar.put(sourceLocation, jar);
        LOG.debug("added sourceLocation {} as {}", sourceLocation, jar);
      }
      if (jar == null) {
        throw new AssertionError("Cannot resolve jar file for " + jarClass);
      }
      localJarFiles.add(jar);
    }

    String libJarsPath = dag.getValue(LogicalPlan.LIBRARY_JARS);
    if (!StringUtils.isEmpty(libJarsPath)) {
      String[] libJars = StringUtils.splitByWholeSeparator(libJarsPath, LIB_JARS_SEP);
      localJarFiles.addAll(Arrays.asList(libJars));
    }

    LOG.info("Local jar file dependencies: " + localJarFiles);

    return localJarFiles;
  }
コード例 #2
0
  public void runMainLoop() throws Exception {

    rmClient.registerApplicationMaster("", 0, "");

    initRegistry();

    startMasterNode();

    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(ClusterConf.get().getWorkerHeapSize());
    capability.setVirtualCores(ClusterConf.get().getNumWorkerThreads());

    List<ContainerRequest> containerReq = new ArrayList<ContainerRequest>();
    for (int i = 0; i < ClusterConf.get().getNumWorkers(); ++i) {
      ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
      rmClient.addContainerRequest(containerAsk);
      containerReq.add(containerAsk);
    }

    while (alive) {
      Thread.sleep(1000);
    }
    finish();

    /*
    for (ContainerRequest req : containerReq) {
        rmClient.removeContainerRequest(req);
    }
    int containersToAdd = 2;
    numContainersToWaitFor = containersToAdd;

    System.out.println("[Am] finished all containers. Asking for more containers, total=" + numContainersToWaitFor);
    for (int i = 0; i < containersToAdd; ++i) {
        ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
        System.out.println("[AM] Making res-req " + i);
        rmClient.addContainerRequest(containerAsk);
    }

    System.out.println("[AM] waiting for containers to finish once more!!!");
    while (!doneWithContainers()) {
        Thread.sleep(100);
    }
    */

  }