public void configure(JobConf conf) {
    /*
     * It reads all the configurations and distributed cache from outside.
     */

    // Read number of nodes in input layer and output layer from configuration
    inputNumdims = conf.get("numdims");
    inputNumhid = conf.get("numhid");

    // Read the weights from distributed cache
    Path[] pathwaysFiles = new Path[0];
    try {
      pathwaysFiles = DistributedCache.getLocalCacheFiles(conf);
      for (Path path : pathwaysFiles) {
        /*
         * this loop reads all the distributed cache files
         * In fact, the driver program ensures that there is only one distributed cache file
         */
        BufferedReader fis = new BufferedReader(new FileReader(path.toString()));
        weightline = fis.readLine();
      }
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
  @Override
  protected void reduce(Text key, Iterable<Text> vals, Context ctx) {
    String name = null;
    String year = null;

    for (Text xx : vals) {
      String[] parts = xx.toString().split("=");

      if (parts[0].equals("name")) {
        name = parts[1];
      }

      if (parts[0].equals("year")) {
        year = parts[1];
      }
    }

    try {
      if (name != null && year != null) {
        ctx.write(new Text(year), new Text(name));
      }
    } catch (Exception ee) {
      ee.printStackTrace(System.err);
      throw new Error("I give up");
    }
  }
Exemplo n.º 3
0
  @Override
  public void executeTest() {

    int iRange1 =
        (int) (Math.pow(1, iDecimalPlaces)) * (int) (this.fMaximumValue - this.fMinimumValue) + 1;

    String sLocalOutput = null;
    if (isHadoop) {
      Job conf = null;
      try {
        conf = this.test();
      } catch (Exception e) {
        e.printStackTrace();
      }

      try {
        sLocalOutput = UtilityHadoop.getFileFromHDFS(sOutput + File.separator + sFileName, conf);
      } catch (IOException e) {
        e.printStackTrace();
      }
    } else {
      try {
        this.testLinear();
      } catch (IOException e) {
        e.printStackTrace();
      }
      sLocalOutput = sOutput + File.separator + this.sFileName;
    }
    int numOfCombinations = UtilityMath.getCombinationAmount(iRange1, 1, true, true).intValue();
    double[] dArrExpected = new double[numOfCombinations];
    for (int i = 0; i < dArrExpected.length; i++) {
      dArrExpected[i] = 1.0;
    }
    Evaluator evaluator = null;
    if (sLocalOutput != null) {
      evaluator =
          new Evaluator(
              sEvaluation, sLocalOutput, dArrExpected, dSignificance, iRange1, iDecimalPlaces);
    }

    boolean isPass = false;
    if (evaluator != null) {
      try {
        isPass = evaluator.evaluate();
      } catch (MathException e) {
        e.printStackTrace();
      } catch (ParameterNotValidException e) {
        e.printStackTrace();
      } catch (ArrayLengthNotEqualException e) {
        e.printStackTrace();
      }
    }

    System.out.println("Passed: " + isPass);
  }
Exemplo n.º 4
0
  private static String lock(String lock) {
    String realPath = "";
    String parent = "/lock";
    String lockName = parent + "/" + lock;

    logger.debug("Getting lock " + lockName);

    try {
      if (zkInstance.exists(parent, false) == null)
        zkInstance.create(parent, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.fromFlag(0));
    } catch (Exception E) {
      logger.error("Error creating lock node: " + E.toString());
      return null;
    }

    List<String> children = new LinkedList<String>();
    try {
      // List <ACL> ACLList = zkInstance.getACL(lockName, zkInstance.exists(lock, false));

      realPath =
          zkInstance.create(
              lockName, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
      // children = zkInstance.getChildren(realPath, false);
      checkLock:
      while (true) {
        children = zkInstance.getChildren(parent, false);
        for (String curChild : children) {
          String child = parent + "/" + curChild;
          // System.out.println(child + " " + realPath + " " +
          // Integer.toString(child.compareTo(realPath)));
          if (child.compareTo(realPath) < 0
              && child.length() == realPath.length()
              && curChild.startsWith(lock)) {
            // System.out.println(child + " cmp to " + realPath);
            Thread.sleep(300);
            continue checkLock;
          }
        }
        logger.info("Got lock " + lockName);
        return realPath;
      }
    } catch (Exception E) {
      logger.error("Exception while trying to get lock " + lockName + " :" + E.toString());
      E.printStackTrace();
      return null;
    }
  }