public static void main(String[] args) throws IOException {
Path baseDir = null;
String localPath = null;
String preservePath = null;
String sIgnoreTablesFilename = null;
String sNoPreserveFilename = null;
String sDateString = null;
long size = 0;
// UNIX dates for right now
long now = new java.util.Date().getTime() / 1000;
long maxDate = now;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--hdfs-path")) {
baseDir = new Path(args[++i]);
continue;
}
if (args[i].equals("--local-path")) {
localPath = args[++i];
continue;
}
if (args[i].equals("--preserve-path")) {
preservePath = args[++i];
continue;
}
if (args[i].equals("--no-preserve")) {
sNoPreserveFilename = args[++i];
continue;
}
if (args[i].equals("--ignore-tables")) {
sIgnoreTablesFilename = args[++i];
continue;
}
if (args[i].equals("--sleep")) {
try {
m_nSleepSeconds = Integer.parseInt(args[++i]);
} catch (Exception e) {
System.err.println("ERROR: " + e.toString() + "\n");
usage();
}
continue;
}
if (args[i].equals("--dry-run")) {
m_bDryRun = true;
continue;
}
if (args[i].equals("--date")) {
sDateString = args[++i];
continue;
}
if (args[i].equals("--max-date")) {
maxDate = Long.parseLong(args[++i]);
continue;
}
if (args[i].equals("--max-bytes")) {
size = Long.parseLong(args[++i]);
continue;
}
System.err.println("ERROR: unknown arg " + args[i]);
usage();
}
if (baseDir == null || localPath == null || preservePath == null || sDateString == null) {
usage();
}
long minDate;
if ("yesterday".equals(sDateString)) {
// figure out yesterday's dates
Calendar cal = Calendar.getInstance();
cal.roll(Calendar.DAY_OF_YEAR, -1);
// yesterday midnight
cal.set(Calendar.HOUR_OF_DAY, 0);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
minDate = cal.getTimeInMillis() / 1000;
// yesterday end of day
cal.set(Calendar.HOUR_OF_DAY, 23);
cal.set(Calendar.MINUTE, 59);
cal.set(Calendar.SECOND, 59);
cal.set(Calendar.MILLISECOND, 999);
maxDate = cal.getTimeInMillis() / 1000;
} else if ("last-week".equals(sDateString)) {
minDate = maxDate - (7 * 24 * 60 * 60);
} else if ("last-day".equals(sDateString)) {
minDate = maxDate - (24 * 60 * 60);
} else {
// UNIX date since epoch of last backup
minDate = Long.parseLong(sDateString);
}
long tmpDate = 0;
BackupHdfs bak = new BackupHdfs();
// initialize the list of tables to ignore
if (sIgnoreTablesFilename != null) {
bak.initializeTablesToIgnore(sIgnoreTablesFilename);
}
// initialize list of files to not preserve
if (sNoPreserveFilename != null) {
bak.initializeNoPreserve(sNoPreserveFilename);
}
ArrayList<Path> pathList = new ArrayList<Path>(2000);
HashMap<Path, Long> hmTimestamps = new HashMap<Path, Long>();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
// If the HDFS path is a dir continue
if (fs.getFileStatus(baseDir).isDir()) {
Calendar cal = Calendar.getInstance();
System.err.println("");
cal.setTimeInMillis(minDate * 1000);
System.err.println("min date = " + cal.getTime().toString());
cal.setTimeInMillis(maxDate * 1000);
System.err.println("max date = " + cal.getTime().toString());
System.err.println("");
System.err.println("Searching filesystem: " + baseDir.toUri().getPath());
bak.checkDir(fs, minDate, maxDate, baseDir, pathList, hmTimestamps);
System.err.println("");
System.err.println("Skipped " + m_nIgnoredTables + " files due to ignored tables");
System.err.println("");
System.err.println("Number of files to backup = " + pathList.size());
System.err.println("Total bytes to backup = " + prettyPrintBytes(m_nTotalBytes));
System.err.println("");
System.err.println("sorting list of files...");
Collections.sort(pathList, new DateComparator(hmTimestamps));
System.err.println("done");
System.err.println("");
System.err.println("starting backup...");
tmpDate = bak.backupFiles(localPath, preservePath, fs, pathList, size);
bak.closeFiles();
System.err.println("");
System.err.println("backup completed...");
}
if (tmpDate == 0) {
// If not size limit reached print out date for right now
System.out.println(maxDate);
} else {
// Print out date for last file backed up
System.err.println("Size limit reached.");
System.out.println(tmpDate);
}
System.exit(0);
}
// Creates a new thread, runs the program in that thread, and reports any errors as needed.
private void run(String clazz) {
try {
// Makes sure the JVM resets if it's already running.
if (JVMrunning) kill();
// Some String constants for java path and OS-specific separators.
String separator = System.getProperty("file.separator");
String path = System.getProperty("java.home") + separator + "bin" + separator + "java";
// Tries to run compiled code.
ProcessBuilder builder = new ProcessBuilder(path, clazz);
// Should be good now! Everything past this is on you. Don't mess it up.
println(
"Build succeeded on " + java.util.Calendar.getInstance().getTime().toString(), progErr);
println("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~", progErr);
JVM = builder.start();
// Note that as of right now, there is no support for input. Only output.
Reader errorReader = new InputStreamReader(JVM.getErrorStream());
Reader outReader = new InputStreamReader(JVM.getInputStream());
// Writer inReader = new OutputStreamWriter(JVM.getOutputStream());
redirectErr = redirectIOStream(errorReader, err);
redirectOut = redirectIOStream(outReader, out);
// redirectIn = redirectIOStream(null, inReader);
} catch (Exception e) {
// This catches any other errors we might get.
println("Some error thrown", progErr);
logError(e.toString());
displayLog();
return;
}
}
public static void main(String[] args) {
String inputStr = null;
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
try {
while ((inputStr = reader.readLine()) != null) {
inputStr = inputStr.trim();
if (DEST_MATCHER.reset(inputStr).matches()) {
System.out.println("Destination field!\n");
} else if (SOURCE_MATCHER.reset(inputStr).matches()) {
System.out.println("Source field!\n");
} else if (STRING_MATCHER.reset(inputStr).matches()) {
System.out.println("String literal!\n");
} else if (VAR_MATCHER.reset(inputStr).matches()) {
System.out.println("Variable!\n");
} else if (NUM_MATCHER.reset(inputStr).matches()) {
System.out.println("Number!\n");
} else {
System.out.println("Huh???\n");
}
}
} catch (Exception ioe) {
System.out.println("Exception: " + ioe.toString());
ioe.printStackTrace();
}
}
protected String getRallyAPIError(String responseXML) {
String errorMsg = "";
try {
org.jdom.input.SAXBuilder bSAX = new org.jdom.input.SAXBuilder();
org.jdom.Document doc = bSAX.build(new StringReader(responseXML));
Element root = doc.getRootElement();
XPath xpath = XPath.newInstance("//Errors");
List xlist = xpath.selectNodes(root);
Iterator iter = xlist.iterator();
while (iter.hasNext()) {
Element item = (Element) iter.next();
errorMsg = item.getChildText("OperationResultError");
}
} catch (Exception e) {
errorMsg = e.toString();
}
return errorMsg;
}
/** * closes all files * * @author [email protected] * @date Thu Dec 1 22:00:05 2011 */ void closeFiles() { try { m_wrMkdirs.close(); m_wrChmods.close(); } catch (Exception e) { System.err.println("ERROR: failed to close files: " + e.toString()); } }
/** * @author [email protected] * @date Sat Jan 7 05:51:47 2012 */ MD5MD5CRC32FileChecksum getLocalFileChecksum(String sPath) { if (g_checksumGenerator == null) { g_checksumGenerator = new ExternalHDFSChecksumGenerator(); } // copied from checksum generator code long lBlockSize = N_BLOCK_SIZE; int bytesPerCRC = 512; try { return g_checksumGenerator.getLocalFilesystemHDFSStyleChecksum( sPath, bytesPerCRC, lBlockSize); } catch (Exception e) { System.out.println("ERROR getting local checksum: " + e.toString()); e.printStackTrace(); return null; } }