/**
* Method to go though the HDFS filesystem in a DFS to find all files
*
* <p>fs:FileSystem object from HDFS minDate: Oldest date for files to be backed up maxDate:Newest
* date for files to be backed up p:Path in HDFS to look for files pathList:Will be filled with
* all files in p hmTimestamps: hashmap of timestamps for later sorting
*/
public void checkDir(
FileSystem fs,
long minDate,
long maxDate,
Path p,
ArrayList<Path> pathList,
HashMap<Path, Long> hmTimestamps) {
long tmpDate;
FileStatus[] fStat;
try {
String sPath = p.toUri().getPath();
// If this is a directory
if (fs.getFileStatus(p).isDir()) {
// ignore certain directories
if ("dfstmp".equals(p.getName())
|| "tmp".equals(p.getName())
|| "jobtracker".equals(p.getName())
|| sPath.startsWith("/mapred")
|| "ops".equals(p.getName())
|| p.getName().startsWith("_distcp_logs")) {
return;
}
// dump the mkdir and chmod commands for this
// directory -- skip root directory only
{
FileStatus stat = fs.getFileStatus(p);
if (!sPath.equals("/")) {
m_wrMkdirs.println("hadoop fs -mkdir " + sPath);
}
m_wrChmods.println(
"hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath);
Short sh = new Short(stat.getPermission().toShort());
m_wrChmods.println(
"hadoop fs -chmod " + Long.toOctalString(sh.longValue()) + " " + sPath);
}
fStat = fs.listStatus(p);
// Do a recursive call to all elements
for (int i = 0; i < fStat.length; i++) {
checkDir(fs, minDate, maxDate, fStat[i].getPath(), pathList, hmTimestamps);
}
} else {
// If not a directory then we've found a file
// ignore crc files
if (p.getName().endsWith(".crc")) {
return;
}
// ignore other files
if (sPath.startsWith("/user/oozie/etl/workflows/")) {
return;
}
// try to get the table name from the path. There are
// various types of tables, from those replicated from
// another database to regular hive tables to
// partitioned hive tables. We use table names to
// both exclude some from the backup, and for the rest
// to dump out the schema and partition name.
if (m_ignoreTables != null && m_ignoreTables.doIgnoreFile(sPath)) {
m_nIgnoredTables++;
if (m_nIgnoredTables < 5) {
System.out.println("Skipping ignore-table file: " + sPath);
} else if (m_nIgnoredTables == 5) {
System.out.println("(...not showing other skipped tables...)");
}
return;
}
FileStatus stat = fs.getFileStatus(p);
tmpDate = stat.getModificationTime() / 1000;
// store the chmods/chowns for all files
m_wrChmods.println(
"hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath);
m_wrChmods.println("hadoop fs -chmod " + stat.getPermission().toShort() + " " + sPath);
// check dates. is it too young?
if (tmpDate < minDate) {
return;
}
// is the file too recent?
if (tmpDate > maxDate) {
// System.out.println("file too recent: " + sPath);
return;
}
// file timestamp is ok
pathList.add(p);
hmTimestamps.put(p, new Long(tmpDate));
// store info about total bytes neeed to backup
m_nTotalBytes += fs.getContentSummary(p).getLength();
}
} catch (IOException e) {
System.err.println("ERROR: could not open " + p + ": " + e);
// System.exit(1) ;
}
}
@SuppressWarnings("unchecked")
public static Object directBind(
String name, Annotation[] annotations, String value, Class<?> clazz, Type type)
throws Exception {
Logger.trace(
"directBind: value ["
+ value
+ "] annotation ["
+ Utils.join(annotations, " ")
+ "] Class ["
+ clazz
+ "]");
boolean nullOrEmpty = value == null || value.trim().length() == 0;
if (annotations != null) {
for (Annotation annotation : annotations) {
if (annotation.annotationType().equals(As.class)) {
Class<? extends TypeBinder<?>> toInstanciate = ((As) annotation).binder();
if (!(toInstanciate.equals(As.DEFAULT.class))) {
// Instantiate the binder
TypeBinder<?> myInstance = toInstanciate.newInstance();
return myInstance.bind(name, annotations, value, clazz, type);
}
}
}
}
// custom types
for (Class<?> c : supportedTypes.keySet()) {
Logger.trace("directBind: value [" + value + "] c [" + c + "] Class [" + clazz + "]");
if (c.isAssignableFrom(clazz)) {
Logger.trace("directBind: isAssignableFrom is true");
return supportedTypes.get(c).bind(name, annotations, value, clazz, type);
}
}
// application custom types
for (Class<TypeBinder<?>> c : Play.classloader.getAssignableClasses(TypeBinder.class)) {
if (c.isAnnotationPresent(Global.class)) {
Class<?> forType =
(Class) ((ParameterizedType) c.getGenericInterfaces()[0]).getActualTypeArguments()[0];
if (forType.isAssignableFrom(clazz)) {
return c.newInstance().bind(name, annotations, value, clazz, type);
}
}
}
// raw String
if (clazz.equals(String.class)) {
return value;
}
// Enums
if (Enum.class.isAssignableFrom(clazz)) {
if (nullOrEmpty) {
return null;
}
return Enum.valueOf((Class<Enum>) clazz, value);
}
// int or Integer binding
if (clazz.getName().equals("int") || clazz.equals(Integer.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0 : null;
}
return Integer.parseInt(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// long or Long binding
if (clazz.getName().equals("long") || clazz.equals(Long.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0l : null;
}
return Long.parseLong(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// byte or Byte binding
if (clazz.getName().equals("byte") || clazz.equals(Byte.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? (byte) 0 : null;
}
return Byte.parseByte(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// short or Short binding
if (clazz.getName().equals("short") || clazz.equals(Short.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? (short) 0 : null;
}
return Short.parseShort(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// float or Float binding
if (clazz.getName().equals("float") || clazz.equals(Float.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0f : null;
}
return Float.parseFloat(value);
}
// double or Double binding
if (clazz.getName().equals("double") || clazz.equals(Double.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0d : null;
}
return Double.parseDouble(value);
}
// BigDecimal binding
if (clazz.equals(BigDecimal.class)) {
if (nullOrEmpty) {
return null;
}
return new BigDecimal(value);
}
// boolean or Boolean binding
if (clazz.getName().equals("boolean") || clazz.equals(Boolean.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? false : null;
}
if (value.equals("1")
|| value.toLowerCase().equals("on")
|| value.toLowerCase().equals("yes")) {
return true;
}
return Boolean.parseBoolean(value);
}
return null;
}