private Job configureJob(Path secretsPath, Path saltFilePath, Path inputPath, Path outputPath)
     throws Exception {
   Job job = Job.getInstance(getConf());
   FileInputFormat.setInputPaths(job, inputPath);
   FileOutputFormat.setOutputPath(job, outputPath);
   job.getConfiguration().set(ObfuscateMapper.SECRET_WORDS_FILE_KEY, secretsPath.toString());
   job.getConfiguration().set(ObfuscateMapper.SALT_FILE_KEY, saltFilePath.toString());
   job.setInputFormatClass(TextInputFormat.class);
   job.setMapperClass(ObfuscateMapper.class);
   job.setNumReduceTasks(0);
   job.setJarByClass(getClass());
   FileSystem.get(outputPath.toUri(), getConf()).delete(outputPath, true);
   return job;
 }
Esempio n. 2
0
 protected boolean isPathAcceptable(final Path pPath1) {
   String path = pPath1.toString().toLowerCase();
   if (path.startsWith("part-r-")) return true;
   String extension = getExtension();
   if (extension != null && path.endsWith(extension.toLowerCase())) return true;
   if (extension != null && path.endsWith(extension.toLowerCase() + ".gz")) return true;
   //noinspection SimplifiableIfStatement,RedundantIfStatement
   if (extension == null) return true;
   return false;
 }