/** * Method to go though the HDFS filesystem in a DFS to find all files * * <p>fs:FileSystem object from HDFS minDate: Oldest date for files to be backed up maxDate:Newest * date for files to be backed up p:Path in HDFS to look for files pathList:Will be filled with * all files in p hmTimestamps: hashmap of timestamps for later sorting */ public void checkDir( FileSystem fs, long minDate, long maxDate, Path p, ArrayList<Path> pathList, HashMap<Path, Long> hmTimestamps) { long tmpDate; FileStatus[] fStat; try { String sPath = p.toUri().getPath(); // If this is a directory if (fs.getFileStatus(p).isDir()) { // ignore certain directories if ("dfstmp".equals(p.getName()) || "tmp".equals(p.getName()) || "jobtracker".equals(p.getName()) || sPath.startsWith("/mapred") || "ops".equals(p.getName()) || p.getName().startsWith("_distcp_logs")) { return; } // dump the mkdir and chmod commands for this // directory -- skip root directory only { FileStatus stat = fs.getFileStatus(p); if (!sPath.equals("/")) { m_wrMkdirs.println("hadoop fs -mkdir " + sPath); } m_wrChmods.println( "hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath); Short sh = new Short(stat.getPermission().toShort()); m_wrChmods.println( "hadoop fs -chmod " + Long.toOctalString(sh.longValue()) + " " + sPath); } fStat = fs.listStatus(p); // Do a recursive call to all elements for (int i = 0; i < fStat.length; i++) { checkDir(fs, minDate, maxDate, fStat[i].getPath(), pathList, hmTimestamps); } } else { // If not a directory then we've found a file // ignore crc files if (p.getName().endsWith(".crc")) { return; } // ignore other files if (sPath.startsWith("/user/oozie/etl/workflows/")) { return; } // try to get the table name from the path. There are // various types of tables, from those replicated from // another database to regular hive tables to // partitioned hive tables. We use table names to // both exclude some from the backup, and for the rest // to dump out the schema and partition name. if (m_ignoreTables != null && m_ignoreTables.doIgnoreFile(sPath)) { m_nIgnoredTables++; if (m_nIgnoredTables < 5) { System.out.println("Skipping ignore-table file: " + sPath); } else if (m_nIgnoredTables == 5) { System.out.println("(...not showing other skipped tables...)"); } return; } FileStatus stat = fs.getFileStatus(p); tmpDate = stat.getModificationTime() / 1000; // store the chmods/chowns for all files m_wrChmods.println( "hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath); m_wrChmods.println("hadoop fs -chmod " + stat.getPermission().toShort() + " " + sPath); // check dates. is it too young? if (tmpDate < minDate) { return; } // is the file too recent? if (tmpDate > maxDate) { // System.out.println("file too recent: " + sPath); return; } // file timestamp is ok pathList.add(p); hmTimestamps.put(p, new Long(tmpDate)); // store info about total bytes neeed to backup m_nTotalBytes += fs.getContentSummary(p).getLength(); } } catch (IOException e) { System.err.println("ERROR: could not open " + p + ": " + e); // System.exit(1) ; } }
private String substitute(String spec, long n) throws IOException { boolean escaped = false; byte[] str = spec.getBytes(); StringBuffer sb = new StringBuffer(); for (int i = 0; i < str.length; i++) { char c = (char) (str[i] & 0xFF); if (escaped) { sb.append(c); escaped = false; } else if (c == '\\') { if (i + 1 == str.length) throw new TextParseException("invalid escape character"); escaped = true; } else if (c == '$') { boolean negative = false; long offset = 0; long width = 0; long base = 10; boolean wantUpperCase = false; if (i + 1 < str.length && str[i + 1] == '$') { // '$$' == literal '$' for backwards // compatibility with old versions of BIND. c = (char) (str[++i] & 0xFF); sb.append(c); continue; } else if (i + 1 < str.length && str[i + 1] == '{') { // It's a substitution with modifiers. i++; if (i + 1 < str.length && str[i + 1] == '-') { negative = true; i++; } while (i + 1 < str.length) { c = (char) (str[++i] & 0xFF); if (c == ',' || c == '}') break; if (c < '0' || c > '9') throw new TextParseException("invalid offset"); c -= '0'; offset *= 10; offset += c; } if (negative) offset = -offset; if (c == ',') { while (i + 1 < str.length) { c = (char) (str[++i] & 0xFF); if (c == ',' || c == '}') break; if (c < '0' || c > '9') throw new TextParseException("invalid width"); c -= '0'; width *= 10; width += c; } } if (c == ',') { if (i + 1 == str.length) throw new TextParseException("invalid base"); c = (char) (str[++i] & 0xFF); if (c == 'o') base = 8; else if (c == 'x') base = 16; else if (c == 'X') { base = 16; wantUpperCase = true; } else if (c != 'd') throw new TextParseException("invalid base"); } if (i + 1 == str.length || str[i + 1] != '}') throw new TextParseException("invalid modifiers"); i++; } long v = n + offset; if (v < 0) throw new TextParseException("invalid offset expansion"); String number; if (base == 8) number = Long.toOctalString(v); else if (base == 16) number = Long.toHexString(v); else number = Long.toString(v); if (wantUpperCase) number = number.toUpperCase(); if (width != 0 && width > number.length()) { int zeros = (int) width - number.length(); while (zeros-- > 0) sb.append('0'); } sb.append(number); } else { sb.append(c); } } return sb.toString(); }