/**
* Method to go though the HDFS filesystem in a DFS to find all files
*
* <p>fs:FileSystem object from HDFS minDate: Oldest date for files to be backed up maxDate:Newest
* date for files to be backed up p:Path in HDFS to look for files pathList:Will be filled with
* all files in p hmTimestamps: hashmap of timestamps for later sorting
*/
public void checkDir(
FileSystem fs,
long minDate,
long maxDate,
Path p,
ArrayList<Path> pathList,
HashMap<Path, Long> hmTimestamps) {
long tmpDate;
FileStatus[] fStat;
try {
String sPath = p.toUri().getPath();
// If this is a directory
if (fs.getFileStatus(p).isDir()) {
// ignore certain directories
if ("dfstmp".equals(p.getName())
|| "tmp".equals(p.getName())
|| "jobtracker".equals(p.getName())
|| sPath.startsWith("/mapred")
|| "ops".equals(p.getName())
|| p.getName().startsWith("_distcp_logs")) {
return;
}
// dump the mkdir and chmod commands for this
// directory -- skip root directory only
{
FileStatus stat = fs.getFileStatus(p);
if (!sPath.equals("/")) {
m_wrMkdirs.println("hadoop fs -mkdir " + sPath);
}
m_wrChmods.println(
"hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath);
Short sh = new Short(stat.getPermission().toShort());
m_wrChmods.println(
"hadoop fs -chmod " + Long.toOctalString(sh.longValue()) + " " + sPath);
}
fStat = fs.listStatus(p);
// Do a recursive call to all elements
for (int i = 0; i < fStat.length; i++) {
checkDir(fs, minDate, maxDate, fStat[i].getPath(), pathList, hmTimestamps);
}
} else {
// If not a directory then we've found a file
// ignore crc files
if (p.getName().endsWith(".crc")) {
return;
}
// ignore other files
if (sPath.startsWith("/user/oozie/etl/workflows/")) {
return;
}
// try to get the table name from the path. There are
// various types of tables, from those replicated from
// another database to regular hive tables to
// partitioned hive tables. We use table names to
// both exclude some from the backup, and for the rest
// to dump out the schema and partition name.
if (m_ignoreTables != null && m_ignoreTables.doIgnoreFile(sPath)) {
m_nIgnoredTables++;
if (m_nIgnoredTables < 5) {
System.out.println("Skipping ignore-table file: " + sPath);
} else if (m_nIgnoredTables == 5) {
System.out.println("(...not showing other skipped tables...)");
}
return;
}
FileStatus stat = fs.getFileStatus(p);
tmpDate = stat.getModificationTime() / 1000;
// store the chmods/chowns for all files
m_wrChmods.println(
"hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath);
m_wrChmods.println("hadoop fs -chmod " + stat.getPermission().toShort() + " " + sPath);
// check dates. is it too young?
if (tmpDate < minDate) {
return;
}
// is the file too recent?
if (tmpDate > maxDate) {
// System.out.println("file too recent: " + sPath);
return;
}
// file timestamp is ok
pathList.add(p);
hmTimestamps.put(p, new Long(tmpDate));
// store info about total bytes neeed to backup
m_nTotalBytes += fs.getContentSummary(p).getLength();
}
} catch (IOException e) {
System.err.println("ERROR: could not open " + p + ": " + e);
// System.exit(1) ;
}
}
private String substitute(String spec, long n) throws IOException {
boolean escaped = false;
byte[] str = spec.getBytes();
StringBuffer sb = new StringBuffer();
for (int i = 0; i < str.length; i++) {
char c = (char) (str[i] & 0xFF);
if (escaped) {
sb.append(c);
escaped = false;
} else if (c == '\\') {
if (i + 1 == str.length) throw new TextParseException("invalid escape character");
escaped = true;
} else if (c == '$') {
boolean negative = false;
long offset = 0;
long width = 0;
long base = 10;
boolean wantUpperCase = false;
if (i + 1 < str.length && str[i + 1] == '$') {
// '$$' == literal '$' for backwards
// compatibility with old versions of BIND.
c = (char) (str[++i] & 0xFF);
sb.append(c);
continue;
} else if (i + 1 < str.length && str[i + 1] == '{') {
// It's a substitution with modifiers.
i++;
if (i + 1 < str.length && str[i + 1] == '-') {
negative = true;
i++;
}
while (i + 1 < str.length) {
c = (char) (str[++i] & 0xFF);
if (c == ',' || c == '}') break;
if (c < '0' || c > '9') throw new TextParseException("invalid offset");
c -= '0';
offset *= 10;
offset += c;
}
if (negative) offset = -offset;
if (c == ',') {
while (i + 1 < str.length) {
c = (char) (str[++i] & 0xFF);
if (c == ',' || c == '}') break;
if (c < '0' || c > '9') throw new TextParseException("invalid width");
c -= '0';
width *= 10;
width += c;
}
}
if (c == ',') {
if (i + 1 == str.length) throw new TextParseException("invalid base");
c = (char) (str[++i] & 0xFF);
if (c == 'o') base = 8;
else if (c == 'x') base = 16;
else if (c == 'X') {
base = 16;
wantUpperCase = true;
} else if (c != 'd') throw new TextParseException("invalid base");
}
if (i + 1 == str.length || str[i + 1] != '}')
throw new TextParseException("invalid modifiers");
i++;
}
long v = n + offset;
if (v < 0) throw new TextParseException("invalid offset expansion");
String number;
if (base == 8) number = Long.toOctalString(v);
else if (base == 16) number = Long.toHexString(v);
else number = Long.toString(v);
if (wantUpperCase) number = number.toUpperCase();
if (width != 0 && width > number.length()) {
int zeros = (int) width - number.length();
while (zeros-- > 0) sb.append('0');
}
sb.append(number);
} else {
sb.append(c);
}
}
return sb.toString();
}