Ejemplo n.º 1
0
    public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
      FileSplit split = (FileSplit) genericSplit;
      Configuration job = context.getConfiguration();
      m_Sb.setLength(0);
      m_Start = split.getStart();
      m_End = m_Start + split.getLength();
      final Path file = split.getPath();
      compressionCodecs = new CompressionCodecFactory(job);
      final CompressionCodec codec = compressionCodecs.getCodec(file);

      // open the file and seek to the m_Start of the split
      FileSystem fs = file.getFileSystem(job);
      //  getFileStatus fileStatus = fs.getFileStatus(split.getPath());
      //noinspection deprecation
      @SuppressWarnings(value = "deprecated")
      long length = fs.getLength(file);
      FSDataInputStream fileIn = fs.open(split.getPath());
      if (m_Start > 0) fileIn.seek(m_Start);
      if (codec != null) {
        CompressionInputStream inputStream = codec.createInputStream(fileIn);
        m_Input = new BufferedReader(new InputStreamReader(inputStream));
        m_End = length;
      } else {
        m_Input = new BufferedReader(new InputStreamReader(fileIn));
      }
      m_Current = m_Start;
      m_Key = split.getPath().getName();
    }
Ejemplo n.º 2
0
  /**
   * Generate the list of files and make them into FileSplits. This needs to be copied to insert a
   * filter on acceptable data
   */
  @Override
  public List<InputSplit> getSplits(JobContext job) throws IOException {
    long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
    long maxSize = getMaxSplitSize(job);
    long desiredMappers =
        job.getConfiguration().getLong("org.systemsbiology.jxtandem.DesiredXMLInputMappers", 0);

    // generate splits
    List<InputSplit> splits = new ArrayList<InputSplit>();
    List<FileStatus> fileStatuses = listStatus(job);
    boolean forceNumberMappers = fileStatuses.size() == 1;
    for (FileStatus file : fileStatuses) {
      Path path = file.getPath();
      if (!isPathAcceptable(path)) // filter acceptable data
      continue;
      FileSystem fs = path.getFileSystem(job.getConfiguration());
      long length = file.getLen();
      BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length);
      if ((length != 0) && isSplitable(job, path)) {
        long blockSize = file.getBlockSize();
        // use desired mappers to force more splits
        if (forceNumberMappers && desiredMappers > 0)
          maxSize = Math.min(maxSize, (length / desiredMappers));

        long splitSize = computeSplitSize(blockSize, minSize, maxSize);

        long bytesRemaining = length;
        while (withinSlop(splitSize, bytesRemaining)) {
          int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
          splits.add(
              new FileSplit(
                  path, length - bytesRemaining, splitSize, blkLocations[blkIndex].getHosts()));
          bytesRemaining -= splitSize;
        }

        if (bytesRemaining != 0) {
          splits.add(
              new FileSplit(
                  path,
                  length - bytesRemaining,
                  bytesRemaining,
                  blkLocations[blkLocations.length - 1].getHosts()));
        }
      } else if (length != 0) {
        splits.add(new FileSplit(path, 0, length, blkLocations[0].getHosts()));
      } else {
        // Create empty hosts array for zero length files
        splits.add(new FileSplit(path, 0, length, new String[0]));
      }
    }
    System.out.println("Total # of splits: " + splits.size());
    //     LOG.debug("Total # of splits: " + splits.size());
    return splits;
  }
Ejemplo n.º 3
0
 private Job configureJob(Path secretsPath, Path saltFilePath, Path inputPath, Path outputPath)
     throws Exception {
   Job job = Job.getInstance(getConf());
   FileInputFormat.setInputPaths(job, inputPath);
   FileOutputFormat.setOutputPath(job, outputPath);
   job.getConfiguration().set(ObfuscateMapper.SECRET_WORDS_FILE_KEY, secretsPath.toString());
   job.getConfiguration().set(ObfuscateMapper.SALT_FILE_KEY, saltFilePath.toString());
   job.setInputFormatClass(TextInputFormat.class);
   job.setMapperClass(ObfuscateMapper.class);
   job.setNumReduceTasks(0);
   job.setJarByClass(getClass());
   FileSystem.get(outputPath.toUri(), getConf()).delete(outputPath, true);
   return job;
 }
Ejemplo n.º 4
0
 @Override
 protected boolean isSplitable(JobContext context, Path file) {
   String fname = file.getName().toLowerCase();
   //noinspection SimplifiableIfStatementf,RedundantIfStatement
   if (fname.endsWith(".gz")) return false;
   return true;
 }
Ejemplo n.º 5
0
 protected boolean isPathAcceptable(final Path pPath1) {
   String path = pPath1.toString().toLowerCase();
   if (path.startsWith("part-r-")) return true;
   String extension = getExtension();
   if (extension != null && path.endsWith(extension.toLowerCase())) return true;
   if (extension != null && path.endsWith(extension.toLowerCase() + ".gz")) return true;
   //noinspection SimplifiableIfStatement,RedundantIfStatement
   if (extension == null) return true;
   return false;
 }
Ejemplo n.º 6
0
 private int getNumSecrets(Path secretsPath) throws Exception {
   FileSystem fileSystem = FileSystem.get(secretsPath.toUri(), getConf());
   FSDataInputStream inputStream = fileSystem.open(secretsPath);
   BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
   String currentLine;
   int numLines = 0;
   while ((currentLine = reader.readLine()) != null) {
     if (!currentLine.isEmpty()) {
       numLines++;
     }
   }
   reader.close();
   return numLines;
 }
Ejemplo n.º 7
0
 private void generateSaltIfNeeded(Path saltFilePath, Path secretsPath) throws Exception {
   FileSystem fileSystem = FileSystem.get(saltFilePath.toUri(), getConf());
   if (!fileSystem.exists(saltFilePath)) {
     FSDataOutputStream outputStream = fileSystem.create(saltFilePath);
     BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream));
     int numSaltsToGenerate = getNumSecrets(secretsPath);
     System.out.printf("Generating %d salts\n", numSaltsToGenerate);
     for (int i = 0; i < numSaltsToGenerate; i++) {
       writer.write(BCrypt.gensalt());
       writer.newLine();
     }
     writer.close();
     outputStream.close();
   }
 }