public void configure(JobConf conf) {
numberOfCenters = Integer.valueOf(conf.get("numberOfCenters"));
centersDirectory = conf.get("centersReadDirectory");
try {
Configuration c = new Configuration();
FileSystem fs = FileSystem.get(c);
for (int index = 0; index < numberOfCenters; ++index) {
SequenceFile.Reader reader =
new SequenceFile.Reader(fs, new Path(centersDirectory + "/centers/" + index), c);
LongWritable key = new LongWritable();
Point value = new Point();
reader.next(key, value);
Point center = (Point) value;
centers.add(center);
reader.close();
}
} catch (IOException e) {
// do nothing
// I hope this doesn't happen
System.out.println("well, damn.");
e.printStackTrace();
}
}
/**
* Get the cause of an I/O exception if caused by a possible disk error
*
* @param ioe an I/O exception
* @return cause if the I/O exception is caused by a possible disk error; null otherwise.
*/
static IOException getCauseIfDiskError(IOException ioe) {
if (ioe.getMessage() != null && ioe.getMessage().startsWith(DISK_ERROR)) {
return (IOException) ioe.getCause();
} else {
return null;
}
}
public BackupHdfs() {
try {
m_wrMkdirs = new PrintWriter(new BufferedWriter(new FileWriter("hdfs-mkdirs.sh")));
m_wrMkdirs.println("hadoop fs -mkdir /mapred");
m_wrMkdirs.println("hadoop fs -mkdir /mapred/system");
m_wrChmods = new PrintWriter(new BufferedWriter(new FileWriter("hdfs-chmods.sh")));
m_wrChmods.println("hadoop fs -chmod 775 /");
m_wrChmods.println("hadoop fs -chown hdfs:hadoop /");
m_wrChmods.println("hadoop fs -chown mapred:hadoop /mapred/system");
m_wrChmods.println("hadoop fs -chmod 700 /mapred/system");
} catch (IOException e) {
System.err.println("ERROR: failed to open files for writing: " + e.toString());
}
}
public BufferedReader loadDataFromFile(String filepath, long offset) {
try {
Path pt = new Path(filepath);
FileSystem fs = FileSystem.get(fsConf);
InputStreamReader isr;
if (fs.isDirectory(pt)) { // multiple parts
isr = new InputStreamReader(OpenMultiplePartsWithOffset(fs, pt, offset));
} else { // single file
FSDataInputStream fileHandler = fs.open(pt);
if (offset > 0) fileHandler.seek(offset);
isr = new InputStreamReader(fileHandler);
}
BufferedReader reader = new BufferedReader(isr);
if (offset > 0) reader.readLine(); // skip first line in case of seek
return reader;
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
assert false : "Should not reach here!";
return null;
}
/** * Compare the checksums of the hdfs file as well as the local copied file. * * @author [email protected] * @date Fri Jan 27 06:06:00 2012 */ boolean compareChecksums(FileSystem fs, Path p, String sFsPath) { try { // get hdfs file info FileStatus stat = fs.getFileStatus(p); // get HDFS checksum FileChecksum ck = fs.getFileChecksum(p); String sCk, sCkShort; if (ck == null) { sCk = sCkShort = "<null>"; } else { sCk = ck.toString(); sCkShort = sCk.replaceAll("^.*:", ""); } // System.out.println(p.toUri().getPath() + " len=" + stat.getLen() // + " " + stat.getOwner() + "/" + stat.getGroup() // + " checksum=" + sCk); // find the local file File fLocal = new File(sFsPath); if (!fLocal.exists()) { System.out.println("CHECKSUM-ERROR: file does not exist: " + sFsPath); return false; } if (!fLocal.isFile()) { System.out.println("CHECKSUM-ERROR: path is not a file: " + sFsPath); return false; } if (stat.getLen() != fLocal.length()) { System.out.println( "CHECKSUM-ERROR: length mismatch: " + sFsPath + " hdfslen=" + stat.getLen() + " fslen=" + fLocal.length()); return false; } // get local fs checksum FileChecksum ckLocal = getLocalFileChecksum(sFsPath); if (ckLocal == null) { System.out.println("ERROR Failed to get checksum for local file " + sFsPath); return false; } // compare checksums as a string, after stripping the // algorithm name from the beginning String sCkLocal = ckLocal.toString(); String sCkLocalShort = sCkLocal.replaceAll("^.*:", ""); if (false == sCkShort.equals(sCkLocalShort)) { System.out.println( "CHECKSUM-ERROR: checksum mismatch: " + sFsPath + "\nhdfs = " + sCk + "\nlocal= " + sCkLocal); return false; } return true; } catch (IOException e) { System.out.println("CHECKSUM-ERROR: " + sFsPath + " exception " + e.toString()); } return false; }