Пример #1
0
  void test(String[] opts, String className) throws Exception {
    count++;
    System.err.println("Test " + count + " " + Arrays.asList(opts) + " " + className);
    Path testSrcDir = Paths.get(System.getProperty("test.src"));
    Path testClassesDir = Paths.get(System.getProperty("test.classes"));
    Path classes = Paths.get("classes." + count);
    classes.createDirectory();

    Context ctx = new Context();
    PathFileManager fm = new JavacPathFileManager(ctx, true, null);
    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
    List<String> options = new ArrayList<String>();
    options.addAll(Arrays.asList(opts));
    options.addAll(Arrays.asList("-verbose", "-XDverboseCompilePolicy", "-d", classes.toString()));
    Iterable<? extends JavaFileObject> compilationUnits =
        fm.getJavaFileObjects(testSrcDir.resolve(className + ".java"));
    StringWriter sw = new StringWriter();
    PrintWriter out = new PrintWriter(sw);
    JavaCompiler.CompilationTask t =
        compiler.getTask(out, fm, null, options, null, compilationUnits);
    boolean ok = t.call();
    System.err.println(sw.toString());
    if (!ok) {
      throw new Exception("compilation failed");
    }

    File expect = new File("classes." + count + "/" + className + ".class");
    if (!expect.exists()) throw new Exception("expected file not found: " + expect);
    long expectedSize = new File(testClassesDir.toString(), className + ".class").length();
    long actualSize = expect.length();
    if (expectedSize != actualSize)
      throw new Exception("wrong size found: " + actualSize + "; expected: " + expectedSize);
  }
Пример #2
0
  public void testAbort() throws IOException {
    JobConf job = new JobConf();
    setConfForFileOutputCommitter(job);
    JobContext jContext = new JobContextImpl(job, taskID.getJobID());
    TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
    FileOutputCommitter committer = new FileOutputCommitter();
    FileOutputFormat.setWorkOutputPath(job, committer.getTempTaskOutputPath(tContext));

    // do setup
    committer.setupJob(jContext);
    committer.setupTask(tContext);
    String file = "test.txt";

    // A reporter that does nothing
    Reporter reporter = Reporter.NULL;
    // write output
    FileSystem localFs = FileSystem.getLocal(job);
    TextOutputFormat theOutputFormat = new TextOutputFormat();
    RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(localFs, job, file, reporter);
    writeOutput(theRecordWriter, reporter);

    // do abort
    committer.abortTask(tContext);
    File expectedFile =
        new File(new Path(committer.getTempTaskOutputPath(tContext), file).toString());
    assertFalse("task temp dir still exists", expectedFile.exists());

    committer.abortJob(jContext, JobStatus.State.FAILED);
    expectedFile = new File(new Path(outDir, FileOutputCommitter.TEMP_DIR_NAME).toString());
    assertFalse("job temp dir still exists", expectedFile.exists());
    assertEquals("Output directory not empty", 0, new File(outDir.toString()).listFiles().length);
    FileUtil.fullyDelete(new File(outDir.toString()));
  }
Пример #3
0
  private InputStream OpenMultiplePartsWithOffset(FileSystem fs, Path pt, long offset)
      throws IOException {
    RemoteIterator<LocatedFileStatus> rit = fs.listFiles(pt, false);
    Vector<FSDataInputStream> fileHandleList = new Vector<FSDataInputStream>();
    while (rit.hasNext()) {
      Path path = rit.next().getPath();
      String filename =
          path.toString().substring(path.getParent().toString().length(), path.toString().length());

      if (filename.startsWith("/part-")) {
        long filesize = fs.getFileStatus(path).getLen();
        if (offset < filesize) {
          FSDataInputStream handle = fs.open(path);
          if (offset > 0) {
            handle.seek(offset);
          }
          fileHandleList.add(handle);
        }
        offset -= filesize;
      }
    }
    if (fileHandleList.size() == 1) return fileHandleList.get(0);
    else if (fileHandleList.size() > 1) {
      Enumeration<FSDataInputStream> enu = fileHandleList.elements();
      return new SequenceInputStream(enu);
    } else {
      System.err.println("Error, no source file loaded. run genSeedDataset.sh fisrt!");
      return null;
    }
  }
Пример #4
0
 private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
   try {
     if (fs == null) return;
     for (FileStatus file : fs.listStatus(p)) {
       Path pfs = file.getPath();
       if (file.isDir()) {
         addFolder(fs, pfs, succeeded, failed);
       } else {
         Key k = Key.make(pfs.toString());
         long size = file.getLen();
         Value val = null;
         if (pfs.getName().endsWith(Extensions.JSON)) {
           JsonParser parser = new JsonParser();
           JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
           JsonElement v = json.get(Constants.VERSION);
           if (v == null) throw new RuntimeException("Missing version");
           JsonElement type = json.get(Constants.TYPE);
           if (type == null) throw new RuntimeException("Missing type");
           Class c = Class.forName(type.getAsString());
           OldModel model = (OldModel) c.newInstance();
           model.fromJson(json);
         } else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
           FSDataInputStream s = fs.open(pfs);
           int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
           byte[] mem = MemoryManager.malloc1(sz);
           s.readFully(mem);
           // Convert to a ValueArray (hope it fits in 1Meg!)
           ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
           val = new Value(k, ary, Value.HDFS);
         } else if (size >= 2 * ValueArray.CHUNK_SZ) {
           val =
               new Value(
                   k,
                   new ValueArray(k, size),
                   Value.HDFS); // ValueArray byte wrapper over a large file
         } else {
           val = new Value(k, (int) size, Value.HDFS); // Plain Value
           val.setdsk();
         }
         DKV.put(k, val);
         Log.info("PersistHdfs: DKV.put(" + k + ")");
         JsonObject o = new JsonObject();
         o.addProperty(Constants.KEY, k.toString());
         o.addProperty(Constants.FILE, pfs.toString());
         o.addProperty(Constants.VALUE_SIZE, file.getLen());
         succeeded.add(o);
       }
     }
   } catch (Exception e) {
     Log.err(e);
     JsonObject o = new JsonObject();
     o.addProperty(Constants.FILE, p.toString());
     o.addProperty(Constants.ERROR, e.getMessage());
     failed.add(o);
   }
 }
Пример #5
0
  /**
   * Add an file path to the current set of classpath entries It adds the file to cache as well.
   *
   * @param file Path of the file to be added
   * @param conf Configuration that contains the classpath setting
   */
  public static void addFileToClassPath(Path file, Configuration conf) throws IOException {
    String classpath = conf.get("mapred.job.classpath.files");
    conf.set(
        "mapred.job.classpath.files",
        classpath == null
            ? file.toString()
            : classpath + System.getProperty("path.separator") + file.toString());
    URI uri = file.makeQualified(file.getFileSystem(conf)).toUri();

    addCacheFile(uri, conf);
  }
Пример #6
0
  // the method which actually copies the caches locally and unjars/unzips them
  // and does chmod for the files
  private static Path localizeCache(
      Configuration conf, URI cache, long confFileStamp, CacheStatus cacheStatus, boolean isArchive)
      throws IOException {
    FileSystem fs = getFileSystem(cache, conf);
    FileSystem localFs = FileSystem.getLocal(conf);
    Path parchive = null;

    if (isArchive) {
      parchive =
          new Path(
              cacheStatus.localizedLoadPath, new Path(cacheStatus.localizedLoadPath.getName()));
    } else {
      parchive = cacheStatus.localizedLoadPath;
    }
    if (!localFs.mkdirs(parchive.getParent())) {
      throw new IOException(
          "Mkdirs failed to create directory " + cacheStatus.localizedLoadPath.toString());
    }
    String cacheId = cache.getPath();

    fs.copyToLocalFile(new Path(cacheId), parchive);
    if (isArchive) {
      String tmpArchive = parchive.toString().toLowerCase();
      File srcFile = new File(parchive.toString());
      File destDir = new File(parchive.getParent().toString());
      if (tmpArchive.endsWith(".jar")) {
        RunJar.unJar(srcFile, destDir);
      } else if (tmpArchive.endsWith(".zip")) {
        FileUtil.unZip(srcFile, destDir);
      } else if (isTarFile(tmpArchive)) {
        FileUtil.unTar(srcFile, destDir);
      }
      // else will not do anyhting
      // and copy the file into the dir as it is
    }
    long cacheSize = FileUtil.getDU(new File(parchive.getParent().toString()));
    cacheStatus.size = cacheSize;
    addCacheInfoUpdate(cacheStatus);

    // do chmod here
    try {
      // Setting recursive permission to grant everyone read and execute
      Path localDir = new Path(cacheStatus.localizedBaseDir, cacheStatus.uniqueParentDir);
      LOG.info("Doing chmod on localdir :" + localDir);
      FileUtil.chmod(localDir.toString(), "ugo+rx", true);
    } catch (InterruptedException e) {
      LOG.warn("Exception in chmod" + e.toString());
    }

    // update cacheStatus to reflect the newly cached file
    cacheStatus.mtime = getTimestamp(conf, cache);
    return cacheStatus.localizedLoadPath;
  }
Пример #7
0
 /** Copy FileSystem files to local files. */
 public static boolean copy(
     FileSystem srcFS, Path src, File dst, boolean deleteSource, Configuration conf)
     throws IOException {
   if (srcFS.getFileStatus(src).isDir()) {
     if (!dst.mkdirs()) {
       return false;
     }
     FileStatus contents[] = srcFS.listStatus(src);
     for (int i = 0; i < contents.length; i++) {
       copy(
           srcFS,
           contents[i].getPath(),
           new File(dst, contents[i].getPath().getName()),
           deleteSource,
           conf);
     }
   } else if (srcFS.isFile(src)) {
     InputStream in = srcFS.open(src);
     IOUtils.copyBytes(in, new FileOutputStream(dst), conf);
   } else {
     throw new IOException(src.toString() + ": No such file or directory");
   }
   if (deleteSource) {
     return srcFS.delete(src, true);
   } else {
     return true;
   }
 }
Пример #8
0
  // Mostly for setting up the symlinks. Note that when we setup the distributed
  // cache, we didn't create the symlinks. This is done on a per task basis
  // by the currently executing task.
  public static void setupWorkDir(JobConf conf) throws IOException {
    File workDir = new File(".").getAbsoluteFile();
    FileUtil.fullyDelete(workDir);
    if (DistributedCache.getSymlink(conf)) {
      URI[] archives = DistributedCache.getCacheArchives(conf);
      URI[] files = DistributedCache.getCacheFiles(conf);
      Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
      Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
      if (archives != null) {
        for (int i = 0; i < archives.length; i++) {
          String link = archives[i].getFragment();
          if (link != null) {
            link = workDir.toString() + Path.SEPARATOR + link;
            File flink = new File(link);
            if (!flink.exists()) {
              FileUtil.symLink(localArchives[i].toString(), link);
            }
          }
        }
      }
      if (files != null) {
        for (int i = 0; i < files.length; i++) {
          String link = files[i].getFragment();
          if (link != null) {
            link = workDir.toString() + Path.SEPARATOR + link;
            File flink = new File(link);
            if (!flink.exists()) {
              FileUtil.symLink(localFiles[i].toString(), link);
            }
          }
        }
      }
    }
    File jobCacheDir = null;
    if (conf.getJar() != null) {
      jobCacheDir = new File(new Path(conf.getJar()).getParent().toString());
    }

    // create symlinks for all the files in job cache dir in current
    // workingdir for streaming
    try {
      DistributedCache.createAllSymlink(conf, jobCacheDir, workDir);
    } catch (IOException ie) {
      // Do not exit even if symlinks have not been created.
      LOG.warn(StringUtils.stringifyException(ie));
    }
    // add java.io.tmpdir given by mapred.child.tmp
    String tmp = conf.get("mapred.child.tmp", "./tmp");
    Path tmpDir = new Path(tmp);

    // if temp directory path is not absolute
    // prepend it with workDir.
    if (!tmpDir.isAbsolute()) {
      tmpDir = new Path(workDir.toString(), tmp);
      FileSystem localFs = FileSystem.getLocal(conf);
      if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
        throw new IOException("Mkdirs failed to create " + tmpDir.toString());
      }
    }
  }
Пример #9
0
  public static void main(String[] args) throws Exception {
    Path dir1 = TestUtil.createTemporaryDirectory();
    try {

      // Same directory
      testCopyFileToFile(dir1, dir1, TestUtil.supportsLinks(dir1));
      testMove(dir1, dir1, TestUtil.supportsLinks(dir1));

      // Different directories. Use test.dir if possible as it might be
      // a different volume/file system and so improve test coverage.
      String testDir = System.getProperty("test.dir", ".");
      Path dir2 = TestUtil.createTemporaryDirectory(testDir);
      try {
        boolean testSymbolicLinks = TestUtil.supportsLinks(dir1) && TestUtil.supportsLinks(dir2);
        testCopyFileToFile(dir1, dir2, testSymbolicLinks);
        testMove(dir1, dir2, testSymbolicLinks);
      } finally {
        TestUtil.removeAll(dir2);
      }

      // Target is location associated with custom provider
      Path dir3 = PassThroughFileSystem.create().getPath(dir1.toString());
      testCopyFileToFile(dir1, dir3, false);
      testMove(dir1, dir3, false);

      // Test copy(InputStream,Path) and copy(Path,OutputStream)
      testCopyInputStreamToFile();
      testCopyFileToOuputStream();

    } finally {
      TestUtil.removeAll(dir1);
    }
  }
Пример #10
0
  private static void addFolder2(
      FileSystem fs, Path p, ArrayList<String> keys, ArrayList<String> failed) {
    try {
      if (fs == null) return;

      Futures futures = new Futures();
      for (FileStatus file : fs.listStatus(p)) {
        Path pfs = file.getPath();
        if (file.isDir()) {
          addFolder2(fs, pfs, keys, failed);
        } else {
          long size = file.getLen();
          Key res;
          if (pfs.getName().endsWith(Extensions.JSON)) {
            throw H2O.unimpl();
          } else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
            throw H2O.unimpl();
          } else {
            Key k = null;
            keys.add((k = HdfsFileVec.make(file, futures)).toString());
            Log.info("PersistHdfs: DKV.put(" + k + ")");
          }
        }
      }
    } catch (Exception e) {
      Log.err(e);
      failed.add(p.toString());
    }
  }
Пример #11
0
  public static FileDesc loadFile(Path root, Path file, int blocSize)
      throws NoSuchAlgorithmException, FileNotFoundException, IOException {
    MessageDigest md = MessageDigest.getInstance("SHA-512");
    MessageDigest fileMd = MessageDigest.getInstance("SHA-512");

    FileDesc desc = new FileDesc(file.toString(), null, null);
    List<Bloc> list = new ArrayList<Bloc>();
    try (FileInputStream fis = new FileInputStream(root.resolve(file).toString())) {
      byte[] buf = new byte[blocSize];
      byte[] h;
      int s;
      while ((s = fis.read(buf)) != -1) {
        int c;
        while (s < buf.length && (c = fis.read()) != -1) buf[s++] = (byte) c;
        fileMd.update(buf, 0, s);

        // padding
        byte p = 0;
        while (s < buf.length) buf[s++] = ++p;
        h = md.digest(buf);
        Bloc bloc = new Bloc(RollingChecksum.compute(buf), new Hash(h));
        list.add(bloc);
      }
      h = fileMd.digest();
      desc.fileHash = new Hash(h);
      desc.blocs = list.toArray(new Bloc[0]);
    }
    return desc;
  }
  /**
   * Check that the user has properly selected a target node for this tool
   *
   * <p>
   *
   * @return The phase progress message or <CODE>null</CODE> to abort early.
   * @throws PipelineException If unable to validate the given user input.
   */
  public synchronized String collectPhaseInput() throws PipelineException {
    if ((pPrimary == null) || (pSelected.size() != 1))
      throw new PipelineException("Please selected one node only.");

    // folder = nPath.getParent();
    // assetName = nPath.getName();
    Path nPath = new Path(pPrimary);
    nPath = nPath.getParentPath().getParentPath();

    if (!nPath.toString().matches(assetPattern))
      throw new PipelineException(
          "This tool only works on assets." + nPath.toString() + " " + nPath.toOsString());

    // if(!pPrimary.matches(matPattern))
    // throw new PipelineException("This tool will only work on a lgt node!");

    File errFile;
    try {
      errFile = File.createTempFile("MaterialsCheckOut", ".err", PackageInfo.sTempPath.toFile());
      err = new PrintWriter(errFile);
      FileCleaner.add(errFile);
    } catch (IOException e) {
      e.printStackTrace();
    }

    NodeStatus status = pSelected.get(pPrimary);
    NodeID nodeID = status.getNodeID();
    pUser = nodeID.getAuthor();
    pView = nodeID.getView();

    OverallNodeState state = status.getHeavyDetails().getOverallNodeState();
    JToolDialog tool = new JToolDialog("MaterialsCheckOut", new JPanel(), "Continue");
    if (!state.equals(OverallNodeState.Identical)) {
      JConfirmDialog dialog =
          new JConfirmDialog(
              tool,
              "This node is different from the checked in node. Do you want to continue with this check out?");
      dialog.setVisible(true);
      if (!dialog.wasConfirmed()) {
        return null;
      } // end if
    } // end if

    return "...Sing Hallelujah!";
  }
Пример #13
0
 public static void addFolder2(Path p, ArrayList<String> keys, ArrayList<String> failed)
     throws IOException {
   FileSystem fs = FileSystem.get(p.toUri(), PersistHdfs.CONF);
   if (!fs.exists(p)) {
     failed.add("Path does not exist: '" + p.toString() + "'");
     return;
   }
   addFolder2(fs, p, keys, failed);
 }
Пример #14
0
 protected boolean isPathAcceptable(final Path pPath1) {
   String path = pPath1.toString().toLowerCase();
   if (path.startsWith("part-r-")) return true;
   String extension = getExtension();
   if (extension != null && path.endsWith(extension.toLowerCase())) return true;
   if (extension != null && path.endsWith(extension.toLowerCase() + ".gz")) return true;
   //noinspection SimplifiableIfStatement,RedundantIfStatement
   if (extension == null) return true;
   return false;
 }
Пример #15
0
  public static String readChild(Path parentDir, String childFileName) {
    try {

      final Path newFilePath = path(parentDir.toString(), childFileName);

      return read(newFilePath);
    } catch (Exception ex) {
      return Exceptions.handle(String.class, ex);
    }
  }
Пример #16
0
  public static void writeChild(Path parentDir, String childFileName, String childContents) {

    try {

      final Path newFilePath = path(parentDir.toString(), childFileName);

      write(newFilePath, childContents);
    } catch (Exception ex) {
      Exceptions.handle(ex);
    }
  }
Пример #17
0
 public static void addFolder(Path p, JsonArray succeeded, JsonArray failed) throws IOException {
   FileSystem fs = FileSystem.get(p.toUri(), PersistHdfs.CONF);
   if (!fs.exists(p)) {
     JsonObject o = new JsonObject();
     o.addProperty(Constants.FILE, p.toString());
     o.addProperty(Constants.ERROR, "Path does not exist!");
     failed.add(o);
     return;
   }
   addFolder(fs, p, succeeded, failed);
 }
Пример #18
0
 public static URL toURL(String str) {
   try {
     if (!Path.uriSchemeSpecified(str)) {
       Path cur = currentPath();
       Path path = cur.resolve(str);
       if (path.isAbsolute()) return path.toURL();
       str = path.toString();
     }
     return new URL(str);
   } catch (Throwable ex) {
     throw WrappedException.wrapIfNeeded(ex);
   }
 }
 public static void recursePath(Configuration conf, Path path, Job job) {
   try {
     FileSystem fs = path.getFileSystem(conf);
     FileStatus[] fstats = fs.listStatus(path);
     if (fstats != null) {
       for (FileStatus f : fstats) {
         Path p = f.getPath();
         ;
         if (fs.isFile(p)) {
           // connection times out otherwise
           System.err.println("file:" + p.toString());
           FileInputFormat.addInputPath(job, p);
         } else {
           System.err.println("dir:" + p.toString());
           recursePath(conf, p, job);
         }
       }
     }
   } catch (IOException e) {
     // shouldn't be here
     throw new RuntimeException(e);
   }
 }
Пример #20
0
  /** Copy files between FileSystems. */
  public static boolean copy(
      FileSystem srcFS,
      Path src,
      FileSystem dstFS,
      Path dst,
      boolean deleteSource,
      boolean overwrite,
      Configuration conf)
      throws IOException {
    dst = checkDest(src.getName(), dstFS, dst, overwrite);

    if (srcFS.getFileStatus(src).isDir()) {
      checkDependencies(srcFS, src, dstFS, dst);
      if (!dstFS.mkdirs(dst)) {
        return false;
      }
      FileStatus contents[] = srcFS.listStatus(src);
      for (int i = 0; i < contents.length; i++) {
        copy(
            srcFS,
            contents[i].getPath(),
            dstFS,
            new Path(dst, contents[i].getPath().getName()),
            deleteSource,
            overwrite,
            conf);
      }
    } else if (srcFS.isFile(src)) {
      InputStream in = null;
      OutputStream out = null;
      try {
        in = srcFS.open(src);
        out = dstFS.create(dst, overwrite);
        IOUtils.copyBytes(in, out, conf, true);
      } catch (IOException e) {
        IOUtils.closeStream(out);
        IOUtils.closeStream(in);
        throw e;
      }
    } else {
      throw new IOException(src.toString() + ": No such file or directory");
    }
    if (deleteSource) {
      return srcFS.delete(src, true);
    } else {
      return true;
    }
  }
Пример #21
0
  public static Path createChildDirectory(Path parentDir, String childDir) {

    try {

      final Path newDir = path(parentDir.toString(), childDir);

      if (!Files.exists(newDir)) {
        Files.createDirectory(newDir);
      }

      return newDir;

    } catch (Exception ex) {
      return Exceptions.handle(Path.class, ex);
    }
  }
Пример #22
0
 @Override
 public void run() {
   for (CacheStatus lcacheStatus : toBeDeletedCache) {
     synchronized (lcacheStatus) {
       Path fullUniqueParentDir =
           new Path(lcacheStatus.localizedBaseDir, lcacheStatus.uniqueParentDir);
       try {
         LOG.info("Deleting local cached path: " + fullUniqueParentDir.toString());
         deleteLocalPath(asyncDiskService, fs, fullUniqueParentDir);
         // decrement the size of the cache from baseDirSize
         deleteCacheInfoUpdate(lcacheStatus);
         LOG.info("Removed cache " + lcacheStatus.localizedLoadPath);
       } catch (IOException e) {
         LOG.warn("Error when deleting " + fullUniqueParentDir, e);
       }
     }
   }
 }
Пример #23
0
  private void laden(Path saveName) throws IOException {
    Properties prop = new Properties();

    FileInputStream in = new FileInputStream(saveName.toString());
    prop.load(in);

    for (int i = 0; prop.containsKey(String.format("quellMenu%d", i)); i++)
      quellListModel.addElement(
          new ListItem(
              Paths.get(prop.getProperty(String.format("quellMenu%d", i))),
              Paths.get(prop.getProperty(String.format("quellMenu%d", i)))));
    for (int i = 0; prop.containsKey(String.format("zielMenu%d", i)); i++)
      zielListModel.addElement(
          new ListItem(
              Paths.get(prop.getProperty(String.format("zielMenu%d", i))),
              Paths.get(prop.getProperty(String.format("zielMenu%d", i)))));

    in.close();
  }
Пример #24
0
    private int getNumberOfItems(Path quellOrdner) {
      int retValue = 0;

      try {
        DirectoryStream<Path> qstream = Files.newDirectoryStream(quellOrdner);
        for (Path qfile : qstream) {
          if (Files.isDirectory(qfile)) {
            getNumberOfItems(Paths.get(quellOrdner.toString() + "/" + qfile.getFileName()));
          }
          i++;
        }
        qstream.close();
      } catch (IOException e) {
        e.printStackTrace();
      }

      retValue = i;
      return retValue;
    }
Пример #25
0
 private static void createSymlink(
     Configuration conf,
     URI cache,
     CacheStatus cacheStatus,
     boolean isArchive,
     Path currentWorkDir,
     boolean honorSymLinkConf)
     throws IOException {
   boolean doSymlink = honorSymLinkConf && DistributedCache.getSymlink(conf);
   if (cache.getFragment() == null) {
     doSymlink = false;
   }
   String link = currentWorkDir.toString() + Path.SEPARATOR + cache.getFragment();
   File flink = new File(link);
   if (doSymlink) {
     if (!flink.exists()) {
       FileUtil.symLink(cacheStatus.localizedLoadPath.toString(), link);
     }
   }
 }
Пример #26
0
  private static List<String> listFromDefaultClassLoader(String s) {
    List<String> result = new ArrayList<>();

    String newPath = s;

    final List<Path> resources = Classpaths.resources(IO.class, newPath);

    for (Path resourcePath : resources) {
      if (Files.isDirectory(resourcePath)) {
        result.addAll(IO.list(resourcePath));
      } else {
        result.add(resourcePath.toString());
      }
    }

    //        for ( int index = 0; index < result.size(); index++ ) {
    //            result.set( index, "classpath:" + result.get( index ) );
    //        }

    return result;
  }
 /**
  * open a file for reading
  *
  * @param hdfsPath !null path - probably of an existing file
  * @return !null stream
  */
 @Override
 public InputStream openFileForRead(Path src) {
   if (isRunningAsUser()) {
     return super.openFileForRead(src);
   }
   String hdfsPath = src.toString();
   if (isFileNameLocal(hdfsPath)) {
     try {
       return new FileInputStream(hdfsPath); // better be local
     } catch (FileNotFoundException e) {
       throw new RuntimeException(e);
     }
   }
   if (true) throw new UnsupportedOperationException("Fix This"); // ToDo
   final FileSystem fs = getDFS();
   try {
     return fs.open(src);
   } catch (IOException e) {
     throw new RuntimeException(e);
   }
 }
Пример #28
0
  @SuppressWarnings("unchecked")
  public void testCommitter() throws Exception {
    JobConf job = new JobConf();
    setConfForFileOutputCommitter(job);
    JobContext jContext = new JobContextImpl(job, taskID.getJobID());
    TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
    FileOutputCommitter committer = new FileOutputCommitter();
    FileOutputFormat.setWorkOutputPath(job, committer.getTempTaskOutputPath(tContext));

    committer.setupJob(jContext);
    committer.setupTask(tContext);
    String file = "test.txt";

    // A reporter that does nothing
    Reporter reporter = Reporter.NULL;
    // write output
    FileSystem localFs = FileSystem.getLocal(job);
    TextOutputFormat theOutputFormat = new TextOutputFormat();
    RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(localFs, job, file, reporter);
    writeOutput(theRecordWriter, reporter);

    // do commit
    committer.commitTask(tContext);
    committer.commitJob(jContext);

    // validate output
    File expectedFile = new File(new Path(outDir, file).toString());
    StringBuffer expectedOutput = new StringBuffer();
    expectedOutput.append(key1).append('\t').append(val1).append("\n");
    expectedOutput.append(val1).append("\n");
    expectedOutput.append(val2).append("\n");
    expectedOutput.append(key2).append("\n");
    expectedOutput.append(key1).append("\n");
    expectedOutput.append(key2).append('\t').append(val2).append("\n");
    String output = UtilsForTests.slurp(expectedFile);
    assertEquals(output, expectedOutput.toString());

    FileUtil.fullyDelete(new File(outDir.toString()));
  }
Пример #29
0
  private void speichern(Path saveName) {
    Properties prop = new Properties();

    if (!quellListModel.isEmpty())
      for (int i = 0; i < quellListModel.getSize(); i++)
        prop.setProperty(
            String.format("quellMenu%d", i),
            quellListModel.getElementAt(i).getValueMember().toString());

    if (!zielListModel.isEmpty())
      for (int i = 0; i < zielListModel.getSize(); i++)
        prop.setProperty(
            String.format("zielMenu%d", i),
            zielListModel.getElementAt(i).getValueMember().toString());

    try {
      FileOutputStream out = new FileOutputStream(saveName.toString());
      prop.store(out, null);
      out.close();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Пример #30
0
  public void downloadFile(String relPath) {
    VOSync.debug("Downloading file from storage: " + relPath);
    Path filePath = FileSystems.getDefault().getPath(startDir.toString(), relPath.substring(1));
    try {
      WatchKey key =
          filePath.getParent().register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
      keys.remove(key);
      key.cancel();

      FileOutputStream outp = new FileOutputStream(filePath.toFile());
      DropboxFileInfo info = api.getFile(relPath, null, outp, null);
      outp.close();

      key = filePath.getParent().register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
      keys.put(key, filePath.getParent());

      MetaHandler.setFile(relPath, filePath.toFile(), info.getMetadata().rev);
    } catch (IOException ex) {
      logger.error("Error downloading file " + relPath + ": " + ex.getMessage());
    } catch (DropboxException ex) {
      ex.printStackTrace();
      logger.error("Error downloading file " + relPath + ": " + ex.getMessage());
    }
  }