Exemple #1
0
  // ==============================================================================
  public final int[] renderGlyph(
      char glyph, Paint paint, android.graphics.Matrix matrix, Rect bounds) {
    Path p = new Path();
    paint.getTextPath(String.valueOf(glyph), 0, 1, 0.0f, 0.0f, p);

    RectF boundsF = new RectF();
    p.computeBounds(boundsF, true);
    matrix.mapRect(boundsF);

    boundsF.roundOut(bounds);
    bounds.left--;
    bounds.right++;

    final int w = bounds.width();
    final int h = Math.max(1, bounds.height());

    Bitmap bm = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888);

    Canvas c = new Canvas(bm);
    matrix.postTranslate(-bounds.left, -bounds.top);
    c.setMatrix(matrix);
    c.drawPath(p, paint);

    final int sizeNeeded = w * h;
    if (cachedRenderArray.length < sizeNeeded) cachedRenderArray = new int[sizeNeeded];

    bm.getPixels(cachedRenderArray, 0, w, 0, 0, w, h);
    bm.recycle();
    return cachedRenderArray;
  }
 /**
  * Interate through all the paths and find the longest one
  *
  * @return longest path distance
  */
 public double getLongestPath() {
   double max = 0;
   for (Path p : getPaths()) {
     if (p.calculateDistance() > max) max = p.calculateDistance();
   }
   return max;
 }
Exemple #3
0
  /** Check that a cancelled key will never be queued */
  static void testCancel(Path dir) throws IOException {
    System.out.println("-- Cancel --");

    try (WatchService watcher = FileSystems.getDefault().newWatchService()) {

      System.out.format("register %s for events\n", dir);
      WatchKey myKey = dir.register(watcher, new WatchEvent.Kind<?>[] {ENTRY_CREATE});
      checkKey(myKey, dir);

      System.out.println("cancel key");
      myKey.cancel();

      // create a file in the directory
      Path file = dir.resolve("mars");
      System.out.format("create: %s\n", file);
      Files.createFile(file);

      // poll for keys - there will be none
      System.out.println("poll...");
      try {
        WatchKey key = watcher.poll(3000, TimeUnit.MILLISECONDS);
        if (key != null) throw new RuntimeException("key should not be queued");
      } catch (InterruptedException x) {
        throw new RuntimeException(x);
      }

      // done
      Files.delete(file);

      System.out.println("OKAY");
    }
  }
  public void testAbort() throws IOException {
    JobConf job = new JobConf();
    setConfForFileOutputCommitter(job);
    JobContext jContext = new JobContextImpl(job, taskID.getJobID());
    TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
    FileOutputCommitter committer = new FileOutputCommitter();
    FileOutputFormat.setWorkOutputPath(job, committer.getTempTaskOutputPath(tContext));

    // do setup
    committer.setupJob(jContext);
    committer.setupTask(tContext);
    String file = "test.txt";

    // A reporter that does nothing
    Reporter reporter = Reporter.NULL;
    // write output
    FileSystem localFs = FileSystem.getLocal(job);
    TextOutputFormat theOutputFormat = new TextOutputFormat();
    RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(localFs, job, file, reporter);
    writeOutput(theRecordWriter, reporter);

    // do abort
    committer.abortTask(tContext);
    File expectedFile =
        new File(new Path(committer.getTempTaskOutputPath(tContext), file).toString());
    assertFalse("task temp dir still exists", expectedFile.exists());

    committer.abortJob(jContext, JobStatus.State.FAILED);
    expectedFile = new File(new Path(outDir, FileOutputCommitter.TEMP_DIR_NAME).toString());
    assertFalse("job temp dir still exists", expectedFile.exists());
    assertEquals("Output directory not empty", 0, new File(outDir.toString()).listFiles().length);
    FileUtil.fullyDelete(new File(outDir.toString()));
  }
Exemple #5
0
  private static IntWritable deduceInputFile(JobConf job) {
    Path[] inputPaths = FileInputFormat.getInputPaths(job);
    Path inputFile = new Path(job.get("map.input.file"));

    // value == one for sort-input; value == two for sort-output
    return (inputFile.getParent().equals(inputPaths[0])) ? sortInput : sortOutput;
  }
Exemple #6
0
  private static void finalize(
      Configuration conf, JobConf jobconf, final Path destPath, String presevedAttributes)
      throws IOException {
    if (presevedAttributes == null) {
      return;
    }
    EnumSet<FileAttribute> preseved = FileAttribute.parse(presevedAttributes);
    if (!preseved.contains(FileAttribute.USER)
        && !preseved.contains(FileAttribute.GROUP)
        && !preseved.contains(FileAttribute.PERMISSION)) {
      return;
    }

    FileSystem dstfs = destPath.getFileSystem(conf);
    Path dstdirlist = new Path(jobconf.get(DST_DIR_LIST_LABEL));
    SequenceFile.Reader in = null;
    try {
      in = new SequenceFile.Reader(dstdirlist.getFileSystem(jobconf), dstdirlist, jobconf);
      Text dsttext = new Text();
      FilePair pair = new FilePair();
      for (; in.next(dsttext, pair); ) {
        Path absdst = new Path(destPath, pair.output);
        updatePermissions(pair.input, dstfs.getFileStatus(absdst), preseved, dstfs);
      }
    } finally {
      checkAndClose(in);
    }
  }
    public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
      FileSplit split = (FileSplit) genericSplit;
      Configuration job = context.getConfiguration();
      m_Sb.setLength(0);
      m_Start = split.getStart();
      m_End = m_Start + split.getLength();
      final Path file = split.getPath();
      compressionCodecs = new CompressionCodecFactory(job);
      final CompressionCodec codec = compressionCodecs.getCodec(file);

      // open the file and seek to the m_Start of the split
      FileSystem fs = file.getFileSystem(job);
      //  getFileStatus fileStatus = fs.getFileStatus(split.getPath());
      //noinspection deprecation
      @SuppressWarnings(value = "deprecated")
      long length = fs.getLength(file);
      FSDataInputStream fileIn = fs.open(split.getPath());
      if (m_Start > 0) fileIn.seek(m_Start);
      if (codec != null) {
        CompressionInputStream inputStream = codec.createInputStream(fileIn);
        m_Input = new BufferedReader(new InputStreamReader(inputStream));
        m_End = length;
      } else {
        m_Input = new BufferedReader(new InputStreamReader(fileIn));
      }
      m_Current = m_Start;
      m_Key = split.getPath().getName();
    }
Exemple #8
0
  public static FileDesc loadFile(Path root, Path file, int blocSize)
      throws NoSuchAlgorithmException, FileNotFoundException, IOException {
    MessageDigest md = MessageDigest.getInstance("SHA-512");
    MessageDigest fileMd = MessageDigest.getInstance("SHA-512");

    FileDesc desc = new FileDesc(file.toString(), null, null);
    List<Bloc> list = new ArrayList<Bloc>();
    try (FileInputStream fis = new FileInputStream(root.resolve(file).toString())) {
      byte[] buf = new byte[blocSize];
      byte[] h;
      int s;
      while ((s = fis.read(buf)) != -1) {
        int c;
        while (s < buf.length && (c = fis.read()) != -1) buf[s++] = (byte) c;
        fileMd.update(buf, 0, s);

        // padding
        byte p = 0;
        while (s < buf.length) buf[s++] = ++p;
        h = md.digest(buf);
        Bloc bloc = new Bloc(RollingChecksum.compute(buf), new Hash(h));
        list.add(bloc);
      }
      h = fileMd.digest();
      desc.fileHash = new Hash(h);
      desc.blocs = list.toArray(new Bloc[0]);
    }
    return desc;
  }
  public int run(String[] args) throws Exception {
    Configuration conf = getConf();
    conf.setLong("mapreduce.task.timeout", 10000 * 60 * 60);

    Path train_file = new Path(args[0]);
    Path test_file = new Path(args[1]);
    conf.set("train_file", train_file.getParent().getName());

    Job job = new Job(conf, "MapTestID");
    job.setJarByClass(MapTestID.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setMapperClass(MapTestID.MapTestIDMap.class);
    job.setReducerClass(MapTestID.MapTestIDReduce.class);

    job.setNumReduceTasks(300);

    FileInputFormat.addInputPath(job, train_file);
    FileInputFormat.addInputPath(job, test_file);
    FileOutputFormat.setOutputPath(job, new Path(args[2]));

    return job.waitForCompletion(true) ? 0 : 1;
  }
  /**
   * Recursively rebuild the tree nodes.
   *
   * @param root The full abstract path to the root saved layout directory.
   * @param local The current directory relative to root (null if none).
   * @param tnode The current parent tree node.
   */
  private void rebuildTreeModel(Path root, Path local, DefaultMutableTreeNode tnode) {
    TreeSet<Path> subdirs = new TreeSet<Path>();
    TreeSet<String> layouts = new TreeSet<String>();
    {
      Path current = new Path(root, local);
      File files[] = current.toFile().listFiles();
      if (files != null) {
        int wk;
        for (wk = 0; wk < files.length; wk++) {
          String name = files[wk].getName();
          if (files[wk].isDirectory()) subdirs.add(new Path(local, name));
          else if (files[wk].isFile()) layouts.add(name);
        }
      }
    }

    for (Path subdir : subdirs) {
      TreeData data = new TreeData(subdir);
      DefaultMutableTreeNode child = new DefaultMutableTreeNode(data, true);
      tnode.add(child);

      rebuildTreeModel(root, subdir, child);
    }

    for (String lname : layouts) {
      TreeData data = new TreeData(new Path(local, lname), lname);
      DefaultMutableTreeNode child = new DefaultMutableTreeNode(data, false);
      tnode.add(child);
    }
  }
  public static void main(String[] args) throws Exception {
    Path dir1 = TestUtil.createTemporaryDirectory();
    try {

      // Same directory
      testCopyFileToFile(dir1, dir1, TestUtil.supportsLinks(dir1));
      testMove(dir1, dir1, TestUtil.supportsLinks(dir1));

      // Different directories. Use test.dir if possible as it might be
      // a different volume/file system and so improve test coverage.
      String testDir = System.getProperty("test.dir", ".");
      Path dir2 = TestUtil.createTemporaryDirectory(testDir);
      try {
        boolean testSymbolicLinks = TestUtil.supportsLinks(dir1) && TestUtil.supportsLinks(dir2);
        testCopyFileToFile(dir1, dir2, testSymbolicLinks);
        testMove(dir1, dir2, testSymbolicLinks);
      } finally {
        TestUtil.removeAll(dir2);
      }

      // Target is location associated with custom provider
      Path dir3 = PassThroughFileSystem.create().getPath(dir1.toString());
      testCopyFileToFile(dir1, dir3, false);
      testMove(dir1, dir3, false);

      // Test copy(InputStream,Path) and copy(Path,OutputStream)
      testCopyInputStreamToFile();
      testCopyFileToOuputStream();

    } finally {
      TestUtil.removeAll(dir1);
    }
  }
Exemple #12
0
  private void uploadFile(String relPath, Path fullPath) throws DropboxException, IOException {
    if (!fullPath.toFile().exists()) return;

    /*if(api.metadata(relPath, 1, null, false, null).rev == MetaHandler.getRev(relPath)){
    	logger.debug("File "+relPath+" not changed");
    	return;
    }*/

    VOSync.debug("Uploading " + relPath);

    String rev = MetaHandler.getRev(relPath);

    InputStream inp = new FileInputStream(fullPath.toFile());
    Entry fileEntry = api.putFile(relPath, inp, fullPath.toFile().length(), rev, null);
    inp.close();

    Path destFilePath =
        FileSystems.getDefault()
            .getPath(fullPath.toFile().getParentFile().getPath(), fileEntry.fileName());

    MetaHandler.setFile(relPath, destFilePath.toFile(), fileEntry.rev);

    logger.debug(relPath + " put to db");

    // if the file was renamed, move the file on disk and download the current from server
    if (!fileEntry.fileName().equals(fullPath.toFile().getName())) {
      logger.error(fileEntry.fileName() + " != " + fullPath.toFile().getName());
      fullPath.toFile().renameTo(destFilePath.toFile());
    }
  }
Exemple #13
0
  @Override
  public Item item(final QueryContext qc, final InputInfo ii) throws QueryException {
    checkCreate(qc);

    final Path path = toPath(0, qc);
    final B64 archive = toB64(exprs[1], qc, false);
    final TokenSet hs = entries(2, qc);

    try (ArchiveIn in = ArchiveIn.get(archive.input(info), info)) {
      while (in.more()) {
        final ZipEntry ze = in.entry();
        final String name = ze.getName();
        if (hs == null || hs.delete(token(name)) != 0) {
          final Path file = path.resolve(name);
          if (ze.isDirectory()) {
            Files.createDirectories(file);
          } else {
            Files.createDirectories(file.getParent());
            Files.write(file, in.read());
          }
        }
      }
    } catch (final IOException ex) {
      throw ARCH_FAIL_X.get(info, ex);
    }
    return null;
  }
Exemple #14
0
  // Mostly for setting up the symlinks. Note that when we setup the distributed
  // cache, we didn't create the symlinks. This is done on a per task basis
  // by the currently executing task.
  public static void setupWorkDir(JobConf conf) throws IOException {
    File workDir = new File(".").getAbsoluteFile();
    FileUtil.fullyDelete(workDir);
    if (DistributedCache.getSymlink(conf)) {
      URI[] archives = DistributedCache.getCacheArchives(conf);
      URI[] files = DistributedCache.getCacheFiles(conf);
      Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
      Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
      if (archives != null) {
        for (int i = 0; i < archives.length; i++) {
          String link = archives[i].getFragment();
          if (link != null) {
            link = workDir.toString() + Path.SEPARATOR + link;
            File flink = new File(link);
            if (!flink.exists()) {
              FileUtil.symLink(localArchives[i].toString(), link);
            }
          }
        }
      }
      if (files != null) {
        for (int i = 0; i < files.length; i++) {
          String link = files[i].getFragment();
          if (link != null) {
            link = workDir.toString() + Path.SEPARATOR + link;
            File flink = new File(link);
            if (!flink.exists()) {
              FileUtil.symLink(localFiles[i].toString(), link);
            }
          }
        }
      }
    }
    File jobCacheDir = null;
    if (conf.getJar() != null) {
      jobCacheDir = new File(new Path(conf.getJar()).getParent().toString());
    }

    // create symlinks for all the files in job cache dir in current
    // workingdir for streaming
    try {
      DistributedCache.createAllSymlink(conf, jobCacheDir, workDir);
    } catch (IOException ie) {
      // Do not exit even if symlinks have not been created.
      LOG.warn(StringUtils.stringifyException(ie));
    }
    // add java.io.tmpdir given by mapred.child.tmp
    String tmp = conf.get("mapred.child.tmp", "./tmp");
    Path tmpDir = new Path(tmp);

    // if temp directory path is not absolute
    // prepend it with workDir.
    if (!tmpDir.isAbsolute()) {
      tmpDir = new Path(workDir.toString(), tmp);
      FileSystem localFs = FileSystem.getLocal(conf);
      if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
        throw new IOException("Mkdirs failed to create " + tmpDir.toString());
      }
    }
  }
Exemple #15
0
  private void directory_to_directory(
      String source_base_name, String destination_base_name, String directory_name)
      throws IOException, InterruptedException {
    // E.g.
    // source_base_name:        /foo/bar
    // destination_base_name:   /x/y
    // directory_name:     abc
    //
    // - Create /x/y/abc
    // - Copy /foo/bar/abc/* to /x/y/abc/*

    // Get source directory.
    String source_directory_name = Path.concatenate(source_base_name, directory_name);
    File source_directory = File.create(source_directory_name);
    Assertion.check(source_directory.isDirectory());

    // Get destination directory, creating it if
    // necessary.
    String destination_directory_name = Path.concatenate(destination_base_name, directory_name);
    File destination_directory = File.create(destination_directory_name);
    destination_directory.mkdirs();

    // Get files in source directory
    String[] source_file_names = source_directory.list();
    Vector source_files = new Vector();
    for (int i = 0; i < source_file_names.length; i++)
      source_files.addElement(File.create(source_directory_name, source_file_names[i]));

    // Copy
    to_directory(source_files, destination_directory);
  }
Exemple #16
0
  private static void addFolder2(
      FileSystem fs, Path p, ArrayList<String> keys, ArrayList<String> failed) {
    try {
      if (fs == null) return;

      Futures futures = new Futures();
      for (FileStatus file : fs.listStatus(p)) {
        Path pfs = file.getPath();
        if (file.isDir()) {
          addFolder2(fs, pfs, keys, failed);
        } else {
          long size = file.getLen();
          Key res;
          if (pfs.getName().endsWith(Extensions.JSON)) {
            throw H2O.unimpl();
          } else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
            throw H2O.unimpl();
          } else {
            Key k = null;
            keys.add((k = HdfsFileVec.make(file, futures)).toString());
            Log.info("PersistHdfs: DKV.put(" + k + ")");
          }
        }
      }
    } catch (Exception e) {
      Log.err(e);
      failed.add(p.toString());
    }
  }
Exemple #17
0
  private InputStream OpenMultiplePartsWithOffset(FileSystem fs, Path pt, long offset)
      throws IOException {
    RemoteIterator<LocatedFileStatus> rit = fs.listFiles(pt, false);
    Vector<FSDataInputStream> fileHandleList = new Vector<FSDataInputStream>();
    while (rit.hasNext()) {
      Path path = rit.next().getPath();
      String filename =
          path.toString().substring(path.getParent().toString().length(), path.toString().length());

      if (filename.startsWith("/part-")) {
        long filesize = fs.getFileStatus(path).getLen();
        if (offset < filesize) {
          FSDataInputStream handle = fs.open(path);
          if (offset > 0) {
            handle.seek(offset);
          }
          fileHandleList.add(handle);
        }
        offset -= filesize;
      }
    }
    if (fileHandleList.size() == 1) return fileHandleList.get(0);
    else if (fileHandleList.size() > 1) {
      Enumeration<FSDataInputStream> enu = fileHandleList.elements();
      return new SequenceInputStream(enu);
    } else {
      System.err.println("Error, no source file loaded. run genSeedDataset.sh fisrt!");
      return null;
    }
  }
Exemple #18
0
  /*
   * Fetch a file that is in a Hadoop file system. Return a local File.
   * Interruptible.
   */
  private File hdfsFetch(Path fromPath, Reporter reporter)
      throws IOException, InterruptedException {
    UUID uniqueId = UUID.randomUUID();
    File toFile = new File(tempDir, uniqueId.toString() + "/" + fromPath.getName());
    File toDir = new File(toFile.getParent());
    if (toDir.exists()) {
      FileUtils.deleteDirectory(toDir);
    }
    toDir.mkdirs();
    Path toPath = new Path(toFile.getCanonicalPath());

    FileSystem fS = fromPath.getFileSystem(hadoopConf);
    FileSystem tofS = FileSystem.getLocal(hadoopConf);

    Throttler throttler = new Throttler((double) bytesPerSecThrottle);
    try {
      for (FileStatus fStatus : fS.globStatus(fromPath)) {
        log.info("Copying " + fStatus.getPath() + " to " + toPath);
        long bytesSoFar = 0;

        FSDataInputStream iS = fS.open(fStatus.getPath());
        FSDataOutputStream oS = tofS.create(toPath);

        byte[] buffer = new byte[downloadBufferSize];

        int nRead;
        while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) {
          // Needed to being able to be interrupted at any moment.
          if (Thread.interrupted()) {
            iS.close();
            oS.close();
            cleanDirNoExceptions(toDir);
            throw new InterruptedException();
          }
          bytesSoFar += nRead;
          oS.write(buffer, 0, nRead);
          throttler.incrementAndThrottle(nRead);
          if (bytesSoFar >= bytesToReportProgress) {
            reporter.progress(bytesSoFar);
            bytesSoFar = 0l;
          }
        }

        if (reporter != null) {
          reporter.progress(bytesSoFar);
        }

        oS.close();
        iS.close();
      }

      return toDir;
    } catch (ClosedByInterruptException e) {
      // This can be thrown by the method read.
      cleanDirNoExceptions(toDir);
      throw new InterruptedIOException();
    }
  }
  // copy source to target with verification
  static void copyAndVerify(Path source, Path target, CopyOption... options) throws IOException {
    Path result = copy(source, target, options);
    assertTrue(result == target);

    // get attributes of source and target file to verify copy
    boolean followLinks = true;
    LinkOption[] linkOptions = new LinkOption[0];
    boolean copyAttributes = false;
    for (CopyOption opt : options) {
      if (opt == NOFOLLOW_LINKS) {
        followLinks = false;
        linkOptions = new LinkOption[] {NOFOLLOW_LINKS};
      }
      if (opt == COPY_ATTRIBUTES) copyAttributes = true;
    }
    BasicFileAttributes basicAttributes =
        readAttributes(source, BasicFileAttributes.class, linkOptions);

    // check hash if regular file
    if (basicAttributes.isRegularFile()) assertTrue(computeHash(source) == computeHash(target));

    // check link target if symbolic link
    if (basicAttributes.isSymbolicLink())
      assert (readSymbolicLink(source).equals(readSymbolicLink(target)));

    // check that attributes are copied
    if (copyAttributes && followLinks) {
      checkBasicAttributes(
          basicAttributes, readAttributes(source, BasicFileAttributes.class, linkOptions));

      // verify other attributes when same provider
      if (source.getFileSystem().provider() == target.getFileSystem().provider()) {

        // check POSIX attributes are copied
        String os = System.getProperty("os.name");
        if ((os.equals("SunOS") || os.equals("Linux")) && testPosixAttributes) {
          checkPosixAttributes(
              readAttributes(source, PosixFileAttributes.class, linkOptions),
              readAttributes(target, PosixFileAttributes.class, linkOptions));
        }

        // check DOS attributes are copied
        if (os.startsWith("Windows")) {
          checkDosAttributes(
              readAttributes(source, DosFileAttributes.class, linkOptions),
              readAttributes(target, DosFileAttributes.class, linkOptions));
        }

        // check named attributes are copied
        if (followLinks
            && getFileStore(source).supportsFileAttributeView("xattr")
            && getFileStore(target).supportsFileAttributeView("xattr")) {
          checkUserDefinedFileAttributes(
              readUserDefinedFileAttributes(source), readUserDefinedFileAttributes(target));
        }
      }
    }
  }
Exemple #20
0
 public static void addFolder2(Path p, ArrayList<String> keys, ArrayList<String> failed)
     throws IOException {
   FileSystem fs = FileSystem.get(p.toUri(), PersistHdfs.CONF);
   if (!fs.exists(p)) {
     failed.add("Path does not exist: '" + p.toString() + "'");
     return;
   }
   addFolder2(fs, p, keys, failed);
 }
Exemple #21
0
    private ArrayList<Path> holeLaufwerkeWindows() {
      ArrayList<Path> laufwerksRoot = new ArrayList<>();

      Iterable<Path> rootDirectories = FileSystems.getDefault().getRootDirectories();

      for (Path path : rootDirectories) {
        laufwerksRoot.add(path.getRoot());
      }
      return laufwerksRoot;
    }
 public static String absolutePath(Path p) {
   if (p == null) return "";
   StringBuilder sb = new StringBuilder();
   Path parentPath = p.getParent();
   if (parentPath == null) return "/";
   sb.append(absolutePath(parentPath));
   if (sb.length() > 1) sb.append("/");
   sb.append(p.getName());
   return sb.toString();
 }
 public void testKryoSerializationRoundtrip() throws IOException {
   Path testDir = Files.createTempDirectory("test");
   try {
     SliceManager sliceManager =
         new SliceManager(new KryoSliceSerializer(), new FileStorageManager(testDir));
     doSerializationRoundtrip("kryo", sliceManager);
   } finally {
     FileHelper.delete(testDir.toFile());
   }
 }
  /**
   * Generate the list of files and make them into FileSplits. This needs to be copied to insert a
   * filter on acceptable data
   */
  @Override
  public List<InputSplit> getSplits(JobContext job) throws IOException {
    long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
    long maxSize = getMaxSplitSize(job);
    long desiredMappers =
        job.getConfiguration().getLong("org.systemsbiology.jxtandem.DesiredXMLInputMappers", 0);

    // generate splits
    List<InputSplit> splits = new ArrayList<InputSplit>();
    List<FileStatus> fileStatuses = listStatus(job);
    boolean forceNumberMappers = fileStatuses.size() == 1;
    for (FileStatus file : fileStatuses) {
      Path path = file.getPath();
      if (!isPathAcceptable(path)) // filter acceptable data
      continue;
      FileSystem fs = path.getFileSystem(job.getConfiguration());
      long length = file.getLen();
      BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length);
      if ((length != 0) && isSplitable(job, path)) {
        long blockSize = file.getBlockSize();
        // use desired mappers to force more splits
        if (forceNumberMappers && desiredMappers > 0)
          maxSize = Math.min(maxSize, (length / desiredMappers));

        long splitSize = computeSplitSize(blockSize, minSize, maxSize);

        long bytesRemaining = length;
        while (withinSlop(splitSize, bytesRemaining)) {
          int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
          splits.add(
              new FileSplit(
                  path, length - bytesRemaining, splitSize, blkLocations[blkIndex].getHosts()));
          bytesRemaining -= splitSize;
        }

        if (bytesRemaining != 0) {
          splits.add(
              new FileSplit(
                  path,
                  length - bytesRemaining,
                  bytesRemaining,
                  blkLocations[blkLocations.length - 1].getHosts()));
        }
      } else if (length != 0) {
        splits.add(new FileSplit(path, 0, length, blkLocations[0].getHosts()));
      } else {
        // Create empty hosts array for zero length files
        splits.add(new FileSplit(path, 0, length, new String[0]));
      }
    }
    System.out.println("Total # of splits: " + splits.size());
    //     LOG.debug("Total # of splits: " + splits.size());
    return splits;
  }
Exemple #25
0
 /**
  * Test the layout of a directory against the sieve.
  *
  * @param dir the directory to test
  * @return true if the directory structure matches the sieve's configuration.
  */
 public boolean matches(File dir) {
   for (String str : includes) {
     File f = new File(dir, Path.translateFile(str));
     if (!f.exists()) return false;
   }
   for (String str : excludes) {
     File f = new File(dir, Path.translateFile(str));
     if (f.exists()) return false;
   }
   return true;
 }
  /**
   * Add an file path to the current set of classpath entries It adds the file to cache as well.
   *
   * @param file Path of the file to be added
   * @param conf Configuration that contains the classpath setting
   */
  public static void addFileToClassPath(Path file, Configuration conf) throws IOException {
    String classpath = conf.get("mapred.job.classpath.files");
    conf.set(
        "mapred.job.classpath.files",
        classpath == null
            ? file.toString()
            : classpath + System.getProperty("path.separator") + file.toString());
    URI uri = file.makeQualified(file.getFileSystem(conf)).toUri();

    addCacheFile(uri, conf);
  }
Exemple #27
0
 public static void addFolder(Path p, JsonArray succeeded, JsonArray failed) throws IOException {
   FileSystem fs = FileSystem.get(p.toUri(), PersistHdfs.CONF);
   if (!fs.exists(p)) {
     JsonObject o = new JsonObject();
     o.addProperty(Constants.FILE, p.toString());
     o.addProperty(Constants.ERROR, "Path does not exist!");
     failed.add(o);
     return;
   }
   addFolder(fs, p, succeeded, failed);
 }
Exemple #28
0
 /** Sanity check for srcPath */
 private static void checkSrcPath(Configuration conf, List<Path> srcPaths) throws IOException {
   List<IOException> rslt = new ArrayList<IOException>();
   for (Path p : srcPaths) {
     FileSystem fs = p.getFileSystem(conf);
     if (!fs.exists(p)) {
       rslt.add(new IOException("Input source " + p + " does not exist."));
     }
   }
   if (!rslt.isEmpty()) {
     throw new InvalidInputException(rslt);
   }
 }
  /**
   * Construct a {@link SubProcessHeavy SubProcessHeavy} instance which when executed will fulfill
   * the given action agenda.
   *
   * <p>
   *
   * @param agenda The agenda to be accomplished by the action.
   * @param outFile The file to which all STDOUT output is redirected.
   * @param errFile The file to which all STDERR output is redirected.
   * @return The SubProcess which will fulfill the agenda.
   * @throws PipelineException If unable to prepare a SubProcess due to illegal, missing or
   *     imcompatable information in the action agenda or a general failure of the prep method code.
   */
  public SubProcessHeavy prep(ActionAgenda agenda, File outFile, File errFile)
      throws PipelineException {
    NodeID nodeID = agenda.getNodeID();

    /* sanity checks */
    Path fromPath = null;
    FileSeq fromSeq = null;
    FileSeq toSeq = null;
    {
      ArrayList<String> formats = new ArrayList<String>();
      formats.add("xml");

      String sname = (String) getSingleParamValue("XmlSource");

      if (sname == null) throw new PipelineException("The Image Source was not set!");

      String xmlpath = null;
      if (sname != null) {
        FileSeq fxml = agenda.getPrimarySource(sname);
        xmlpath = fxml.toString();
      }

      NodeID snodeID = new NodeID(nodeID, sname);

      ArrayList<String> suffixes = new ArrayList<String>();
      suffixes.add("html");
      suffixes.add("htm");
      Path targetPath = getPrimaryTargetPath(agenda, suffixes, "Html fil");

      // ********************************************************************
      // Build the mencoder command
      // ********************************************************************
      ArrayList<String> args = new ArrayList<String>();

      args.add("/intranet/jpt/phpCliPrograms/Edit2pipe.php");
      args.add("none");
      args.add(xmlpath);
      args.add(xmlpath);

      args.add(targetPath.toOsString());

      // Create sub process
      return createSubProcess(
          agenda,
          "php5",
          args,
          agenda.getEnvironment(),
          agenda.getTargetPath().toFile(),
          outFile,
          errFile);
    }
  }
 public void setClassPath(final Path classpath) {
   this.pathComponents.removeAllElements();
   if (classpath != null) {
     final Path actualClasspath = classpath.concatSystemClasspath("ignore");
     final String[] pathElements = actualClasspath.list();
     for (int i = 0; i < pathElements.length; ++i) {
       try {
         this.addPathElement(pathElements[i]);
       } catch (BuildException ex) {
       }
     }
   }
 }