public void copyFilesToLocal(int i) throws InterruptedException {
Process p;
BufferedReader brb;
String s;
try {
p =
Runtime.getRuntime()
.exec(
"hadoop fs -copyToLocal /user/cloudera/Index_"
+ i
+ "/indexFile"
+ i
+ " /home/cloudera/Desktop/datasets/IndexFiles");
brb = new BufferedReader(new InputStreamReader(p.getInputStream()));
while ((s = brb.readLine()) != null) {
System.out.println(s);
p.waitFor();
// System.out.println("exit: "+p.exitValue());
// p.destroy();
}
p.destroy();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println(i + " File Copied to local indexFile folder from HDFS..");
}
public void mergeFiles(int noOfIndices) throws IOException, InterruptedException {
System.out.println("Index file merge program...");
Process p;
String s;
BufferedReader br;
String dir = "/home/cloudera/Desktop/datasets/IndexFiles";
System.out.println("Deleting all the exisitng file from local indexFile directory..");
deleteLocalIndexFiles(dir);
for (int i = 1; i <= noOfIndices; i++) {
p =
Runtime.getRuntime()
.exec(
"hadoop fs -mv /user/cloudera/Index_"
+ i
+ "/part* /user/cloudera/Index_"
+ i
+ "/indexFile"
+ i);
br = new BufferedReader(new InputStreamReader(p.getInputStream()));
while ((s = br.readLine()) != null) {
System.out.println(s);
p.waitFor();
}
p.destroy();
copyFilesToLocal(i);
}
mergeFilesOnLocal(dir, noOfIndices);
MainIndexTableInput mti = new MainIndexTableInput();
mti.createMainIndexFile("/home/cloudera/Desktop/datasets/IndexFiles/MergedOne");
}
//
// The main work loop
//
public void run() {
//
// Poll the Namenode (once every 5 minutes) to find the size of the
// pending edit log.
//
long period = 5 * 60; // 5 minutes
long lastCheckpointTime = 0;
if (checkpointPeriod < period) {
period = checkpointPeriod;
}
while (shouldRun) {
try {
Thread.sleep(1000 * period);
} catch (InterruptedException ie) {
// do nothing
}
if (!shouldRun) {
break;
}
try {
long now = System.currentTimeMillis();
long size = namenode.getEditLogSize();
if (size >= checkpointSize || now >= lastCheckpointTime + 1000 * checkpointPeriod) {
doCheckpoint();
lastCheckpointTime = now;
}
} catch (IOException e) {
LOG.error("Exception in doCheckpoint: ");
LOG.error(StringUtils.stringifyException(e));
e.printStackTrace();
checkpointImage.imageDigest = null;
} catch (Throwable e) {
LOG.error("Throwable Exception in doCheckpoint: ");
LOG.error(StringUtils.stringifyException(e));
e.printStackTrace();
Runtime.getRuntime().exit(-1);
}
}
}