public BackupHdfs() {
try {
m_wrMkdirs = new PrintWriter(new BufferedWriter(new FileWriter("hdfs-mkdirs.sh")));
m_wrMkdirs.println("hadoop fs -mkdir /mapred");
m_wrMkdirs.println("hadoop fs -mkdir /mapred/system");
m_wrChmods = new PrintWriter(new BufferedWriter(new FileWriter("hdfs-chmods.sh")));
m_wrChmods.println("hadoop fs -chmod 775 /");
m_wrChmods.println("hadoop fs -chown hdfs:hadoop /");
m_wrChmods.println("hadoop fs -chown mapred:hadoop /mapred/system");
m_wrChmods.println("hadoop fs -chmod 700 /mapred/system");
} catch (IOException e) {
System.err.println("ERROR: failed to open files for writing: " + e.toString());
}
}
private void executeParsedFile(NameListMessage nameListMessage, ArrayList cmds) {
BufferedWriter buffer = null;
int connectionId = nameListMessage.connectionId;
boolean commaSeparated = nameListMessage.commaSeparated;
// get java.sql.Connection from connectionId
Connection connection = ConnectionService.getConnection(connectionId);
if (connection == null) {
nameListMessage.errorMessage = kConnectionClosed;
} else {
try {
Thread th = null;
buffer = new BufferedWriter(writer);
// if reader exists, create reading thread
if (reader != null) {
final ObjectNameReader objectReader = new ObjectNameReader(nameListMessage.nameList);
// This thread will read stream until meets EOL (ie. when
// writer is closed)
th =
new Thread() {
public void run() {
objectReader.readInputFile(reader);
}
};
} // end if
// if no reader (offline reverse), never start the reading
// thread
// (because output is written down directly to the disk without
// processing in counterpart).
if (reader != null) th.start();
// execute each SQL statement
// we skip objects that don't need to be reversed
Iterator iter = cmds.iterator();
int index = -1;
while (iter.hasNext()) {
String cmd = (String) iter.next();
index++;
// check if this statement id is marked as <to skip>
if (nameListMessage.ignoredStatementIds != null
&& nameListMessage.ignoredStatementIds.contains(new Integer(index))) {
if (commaSeparated) {
buffer.flush();
buffer.newLine();
}
continue;
}
if (writer != null) {
if (cmd != null) {
try {
execute(connection, cmd, buffer, commaSeparated);
} catch (SQLException e) {
if (Debug.isDebug()) e.printStackTrace();
nameListMessage.errorMessage =
nameListMessage.errorMessage == null
? e.toString()
: nameListMessage.errorMessage.concat("\n" + e.toString()); // NOT
// LOCALIZABLE,
// escape
// code
}
}
if (commaSeparated) {
buffer.flush();
buffer.newLine();
}
}
}
}
/*
* catch (SQLException ex) { if (Debug.isDebug()) ex.printStackTrace();
* nameListMessage.errorMessage = nameListMessage.errorMessage == null ? ex.toString() :
* nameListMessage.errorMessage.concat("\n" + ex.toString()); //NOT LOCALIZABLE, escape
* code }
*/
catch (IOException ex) {
if (Debug.isDebug()) ex.printStackTrace();
nameListMessage.errorMessage =
nameListMessage.errorMessage == null
? ex.toString()
: nameListMessage.errorMessage.concat(
"\n" + ex.toString()); // NOT LOCALIZABLE, escape
// code
}
// close writer: the reading thread terminates.
try {
// WARNING: we must make a pause after closing the buffer
// otherwise
// the nameList variable won't be well initialized...strange!
// [FG]
buffer.close();
Thread.sleep(400);
} catch (IOException ex) {
} catch (InterruptedException ex) {
}
} // end if
} // end parseSqlFile()
/** * Compare the checksums of the hdfs file as well as the local copied file. * * @author [email protected] * @date Fri Jan 27 06:06:00 2012 */ boolean compareChecksums(FileSystem fs, Path p, String sFsPath) { try { // get hdfs file info FileStatus stat = fs.getFileStatus(p); // get HDFS checksum FileChecksum ck = fs.getFileChecksum(p); String sCk, sCkShort; if (ck == null) { sCk = sCkShort = "<null>"; } else { sCk = ck.toString(); sCkShort = sCk.replaceAll("^.*:", ""); } // System.out.println(p.toUri().getPath() + " len=" + stat.getLen() // + " " + stat.getOwner() + "/" + stat.getGroup() // + " checksum=" + sCk); // find the local file File fLocal = new File(sFsPath); if (!fLocal.exists()) { System.out.println("CHECKSUM-ERROR: file does not exist: " + sFsPath); return false; } if (!fLocal.isFile()) { System.out.println("CHECKSUM-ERROR: path is not a file: " + sFsPath); return false; } if (stat.getLen() != fLocal.length()) { System.out.println( "CHECKSUM-ERROR: length mismatch: " + sFsPath + " hdfslen=" + stat.getLen() + " fslen=" + fLocal.length()); return false; } // get local fs checksum FileChecksum ckLocal = getLocalFileChecksum(sFsPath); if (ckLocal == null) { System.out.println("ERROR Failed to get checksum for local file " + sFsPath); return false; } // compare checksums as a string, after stripping the // algorithm name from the beginning String sCkLocal = ckLocal.toString(); String sCkLocalShort = sCkLocal.replaceAll("^.*:", ""); if (false == sCkShort.equals(sCkLocalShort)) { System.out.println( "CHECKSUM-ERROR: checksum mismatch: " + sFsPath + "\nhdfs = " + sCk + "\nlocal= " + sCkLocal); return false; } return true; } catch (IOException e) { System.out.println("CHECKSUM-ERROR: " + sFsPath + " exception " + e.toString()); } return false; }
