public void copyFilesToLocal(int i) throws InterruptedException {
   Process p;
   BufferedReader brb;
   String s;
   try {
     p =
         Runtime.getRuntime()
             .exec(
                 "hadoop fs -copyToLocal /user/cloudera/Index_"
                     + i
                     + "/indexFile"
                     + i
                     + " /home/cloudera/Desktop/datasets/IndexFiles");
     brb = new BufferedReader(new InputStreamReader(p.getInputStream()));
     while ((s = brb.readLine()) != null) {
       System.out.println(s);
       p.waitFor();
       // System.out.println("exit: "+p.exitValue());
       // p.destroy();
     }
     p.destroy();
   } catch (IOException e) {
     // TODO Auto-generated catch block
     e.printStackTrace();
   }
   System.out.println(i + " File Copied to local indexFile folder from HDFS..");
 }
  public void mergeFiles(int noOfIndices) throws IOException, InterruptedException {
    System.out.println("Index file merge program...");
    Process p;
    String s;
    BufferedReader br;
    String dir = "/home/cloudera/Desktop/datasets/IndexFiles";

    System.out.println("Deleting all the exisitng file from local indexFile directory..");
    deleteLocalIndexFiles(dir);

    for (int i = 1; i <= noOfIndices; i++) {
      p =
          Runtime.getRuntime()
              .exec(
                  "hadoop fs -mv /user/cloudera/Index_"
                      + i
                      + "/part* /user/cloudera/Index_"
                      + i
                      + "/indexFile"
                      + i);
      br = new BufferedReader(new InputStreamReader(p.getInputStream()));
      while ((s = br.readLine()) != null) {
        System.out.println(s);
        p.waitFor();
      }
      p.destroy();
      copyFilesToLocal(i);
    }
    mergeFilesOnLocal(dir, noOfIndices);
    MainIndexTableInput mti = new MainIndexTableInput();
    mti.createMainIndexFile("/home/cloudera/Desktop/datasets/IndexFiles/MergedOne");
  }
Example #3
0
  //
  // The main work loop
  //
  public void run() {

    //
    // Poll the Namenode (once every 5 minutes) to find the size of the
    // pending edit log.
    //
    long period = 5 * 60; // 5 minutes
    long lastCheckpointTime = 0;
    if (checkpointPeriod < period) {
      period = checkpointPeriod;
    }

    while (shouldRun) {
      try {
        Thread.sleep(1000 * period);
      } catch (InterruptedException ie) {
        // do nothing
      }
      if (!shouldRun) {
        break;
      }
      try {
        long now = System.currentTimeMillis();

        long size = namenode.getEditLogSize();
        if (size >= checkpointSize || now >= lastCheckpointTime + 1000 * checkpointPeriod) {
          doCheckpoint();
          lastCheckpointTime = now;
        }
      } catch (IOException e) {
        LOG.error("Exception in doCheckpoint: ");
        LOG.error(StringUtils.stringifyException(e));
        e.printStackTrace();
        checkpointImage.imageDigest = null;
      } catch (Throwable e) {
        LOG.error("Throwable Exception in doCheckpoint: ");
        LOG.error(StringUtils.stringifyException(e));
        e.printStackTrace();
        Runtime.getRuntime().exit(-1);
      }
    }
  }