Beispiel #1
0
 private String copyFromLocal(FileSystem fs, Path basePath, String[] files) throws IOException {
   StringBuilder csv = new StringBuilder(files.length * (basePath.toString().length() + 16));
   for (String localFile : files) {
     Path src = new Path(localFile);
     String filename = src.getName();
     Path dst = new Path(basePath, filename);
     URI localFileURI = null;
     try {
       localFileURI = new URI(localFile);
     } catch (URISyntaxException e) {
       throw new IOException(e);
     }
     if (localFileURI.getScheme() == null || localFileURI.getScheme().startsWith("file")) {
       LOG.info("Copy {} from local filesystem to {}", localFile, dst);
       fs.copyFromLocalFile(false, true, src, dst);
     } else {
       LOG.info("Copy {} from DFS to {}", localFile, dst);
       FileUtil.copy(fs, src, fs, dst, false, true, conf);
     }
     if (csv.length() > 0) {
       csv.append(LIB_JARS_SEP);
     }
     csv.append(dst.toString());
   }
   return csv.toString();
 }
Beispiel #2
0
  void test(String[] opts, String className) throws Exception {
    count++;
    System.err.println("Test " + count + " " + Arrays.asList(opts) + " " + className);
    Path testSrcDir = Paths.get(System.getProperty("test.src"));
    Path testClassesDir = Paths.get(System.getProperty("test.classes"));
    Path classes = Paths.get("classes." + count);
    classes.createDirectory();

    Context ctx = new Context();
    PathFileManager fm = new JavacPathFileManager(ctx, true, null);
    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
    List<String> options = new ArrayList<String>();
    options.addAll(Arrays.asList(opts));
    options.addAll(Arrays.asList("-verbose", "-XDverboseCompilePolicy", "-d", classes.toString()));
    Iterable<? extends JavaFileObject> compilationUnits =
        fm.getJavaFileObjects(testSrcDir.resolve(className + ".java"));
    StringWriter sw = new StringWriter();
    PrintWriter out = new PrintWriter(sw);
    JavaCompiler.CompilationTask t =
        compiler.getTask(out, fm, null, options, null, compilationUnits);
    boolean ok = t.call();
    System.err.println(sw.toString());
    if (!ok) {
      throw new Exception("compilation failed");
    }

    File expect = new File("classes." + count + "/" + className + ".class");
    if (!expect.exists()) throw new Exception("expected file not found: " + expect);
    long expectedSize = new File(testClassesDir.toString(), className + ".class").length();
    long actualSize = expect.length();
    if (expectedSize != actualSize)
      throw new Exception("wrong size found: " + actualSize + "; expected: " + expectedSize);
  }
  /**
   * Add an file path to the current set of classpath entries It adds the file to cache as well.
   *
   * @param file Path of the file to be added
   * @param conf Configuration that contains the classpath setting
   */
  public static void addFileToClassPath(Path file, Configuration conf) throws IOException {
    String classpath = conf.get("mapred.job.classpath.files");
    conf.set(
        "mapred.job.classpath.files",
        classpath == null
            ? file.toString()
            : classpath + System.getProperty("path.separator") + file.toString());
    URI uri = file.makeQualified(file.getFileSystem(conf)).toUri();

    addCacheFile(uri, conf);
  }
 @Override
 public String toString() {
   StringBuilder sb = new StringBuilder();
   for (Path path : getPathContents().keySet()) {
     sb.append(String.format("    A %s%n", path.toString()));
   }
   for (Path path : getPathsToDelete()) {
     sb.append(String.format("    D %s%n", path.toString()));
   }
   return sb.toString();
 }
  // the method which actually copies the caches locally and unjars/unzips them
  // and does chmod for the files
  private static Path localizeCache(
      Configuration conf, URI cache, long confFileStamp, CacheStatus cacheStatus, boolean isArchive)
      throws IOException {
    FileSystem fs = getFileSystem(cache, conf);
    FileSystem localFs = FileSystem.getLocal(conf);
    Path parchive = null;

    if (isArchive) {
      parchive =
          new Path(
              cacheStatus.localizedLoadPath, new Path(cacheStatus.localizedLoadPath.getName()));
    } else {
      parchive = cacheStatus.localizedLoadPath;
    }
    if (!localFs.mkdirs(parchive.getParent())) {
      throw new IOException(
          "Mkdirs failed to create directory " + cacheStatus.localizedLoadPath.toString());
    }
    String cacheId = cache.getPath();

    fs.copyToLocalFile(new Path(cacheId), parchive);
    if (isArchive) {
      String tmpArchive = parchive.toString().toLowerCase();
      File srcFile = new File(parchive.toString());
      File destDir = new File(parchive.getParent().toString());
      if (tmpArchive.endsWith(".jar")) {
        RunJar.unJar(srcFile, destDir);
      } else if (tmpArchive.endsWith(".zip")) {
        FileUtil.unZip(srcFile, destDir);
      } else if (isTarFile(tmpArchive)) {
        FileUtil.unTar(srcFile, destDir);
      }
      // else will not do anyhting
      // and copy the file into the dir as it is
    }
    long cacheSize = FileUtil.getDU(new File(parchive.getParent().toString()));
    cacheStatus.size = cacheSize;
    addCacheInfoUpdate(cacheStatus);

    // do chmod here
    try {
      // Setting recursive permission to grant everyone read and execute
      Path localDir = new Path(cacheStatus.localizedBaseDir, cacheStatus.uniqueParentDir);
      LOG.info("Doing chmod on localdir :" + localDir);
      FileUtil.chmod(localDir.toString(), "ugo+rx", true);
    } catch (InterruptedException e) {
      LOG.warn("Exception in chmod" + e.toString());
    }

    // update cacheStatus to reflect the newly cached file
    cacheStatus.mtime = getTimestamp(conf, cache);
    return cacheStatus.localizedLoadPath;
  }
Beispiel #6
0
  public static FileDesc loadFile(Path root, Path file, int blocSize)
      throws NoSuchAlgorithmException, FileNotFoundException, IOException {
    MessageDigest md = MessageDigest.getInstance("SHA-512");
    MessageDigest fileMd = MessageDigest.getInstance("SHA-512");

    FileDesc desc = new FileDesc(file.toString(), null, null);
    List<Bloc> list = new ArrayList<Bloc>();
    try (FileInputStream fis = new FileInputStream(root.resolve(file).toString())) {
      byte[] buf = new byte[blocSize];
      byte[] h;
      int s;
      while ((s = fis.read(buf)) != -1) {
        int c;
        while (s < buf.length && (c = fis.read()) != -1) buf[s++] = (byte) c;
        fileMd.update(buf, 0, s);

        // padding
        byte p = 0;
        while (s < buf.length) buf[s++] = ++p;
        h = md.digest(buf);
        Bloc bloc = new Bloc(RollingChecksum.compute(buf), new Hash(h));
        list.add(bloc);
      }
      h = fileMd.digest();
      desc.fileHash = new Hash(h);
      desc.blocs = list.toArray(new Bloc[0]);
    }
    return desc;
  }
  public static void main(String[] args) throws Exception {
    Path dir1 = TestUtil.createTemporaryDirectory();
    try {

      // Same directory
      testCopyFileToFile(dir1, dir1, TestUtil.supportsLinks(dir1));
      testMove(dir1, dir1, TestUtil.supportsLinks(dir1));

      // Different directories. Use test.dir if possible as it might be
      // a different volume/file system and so improve test coverage.
      String testDir = System.getProperty("test.dir", ".");
      Path dir2 = TestUtil.createTemporaryDirectory(testDir);
      try {
        boolean testSymbolicLinks = TestUtil.supportsLinks(dir1) && TestUtil.supportsLinks(dir2);
        testCopyFileToFile(dir1, dir2, testSymbolicLinks);
        testMove(dir1, dir2, testSymbolicLinks);
      } finally {
        TestUtil.removeAll(dir2);
      }

      // Target is location associated with custom provider
      Path dir3 = PassThroughFileSystem.create().getPath(dir1.toString());
      testCopyFileToFile(dir1, dir3, false);
      testMove(dir1, dir3, false);

      // Test copy(InputStream,Path) and copy(Path,OutputStream)
      testCopyInputStreamToFile();
      testCopyFileToOuputStream();

    } finally {
      TestUtil.removeAll(dir1);
    }
  }
Beispiel #8
0
  public void removePlugin(String name, Terminal terminal) throws IOException {
    if (name == null) {
      throw new IllegalArgumentException("plugin name must be supplied with remove [name].");
    }
    PluginHandle pluginHandle = PluginHandle.parse(name);
    boolean removed = false;

    checkForForbiddenName(pluginHandle.name);
    Path pluginToDelete = pluginHandle.extractedDir(environment);
    if (Files.exists(pluginToDelete)) {
      terminal.println(VERBOSE, "Removing: %s", pluginToDelete);
      try {
        IOUtils.rm(pluginToDelete);
      } catch (IOException ex) {
        throw new IOException(
            "Unable to remove "
                + pluginHandle.name
                + ". Check file permissions on "
                + pluginToDelete.toString(),
            ex);
      }
      removed = true;
    }
    Path binLocation = pluginHandle.binDir(environment);
    if (Files.exists(binLocation)) {
      terminal.println(VERBOSE, "Removing: %s", binLocation);
      try {
        IOUtils.rm(binLocation);
      } catch (IOException ex) {
        throw new IOException(
            "Unable to remove "
                + pluginHandle.name
                + ". Check file permissions on "
                + binLocation.toString(),
            ex);
      }
      removed = true;
    }

    if (removed) {
      terminal.println("Removed %s", name);
    } else {
      terminal.println(
          "Plugin %s not found. Run \"plugin list\" to get list of installed plugins.", name);
    }
  }
  /**
   * Check that the user has properly selected a target node for this tool
   *
   * <p>
   *
   * @return The phase progress message or <CODE>null</CODE> to abort early.
   * @throws PipelineException If unable to validate the given user input.
   */
  public synchronized String collectPhaseInput() throws PipelineException {
    if ((pPrimary == null) || (pSelected.size() != 1))
      throw new PipelineException("Please selected one node only.");

    // folder = nPath.getParent();
    // assetName = nPath.getName();
    Path nPath = new Path(pPrimary);
    nPath = nPath.getParentPath().getParentPath();

    if (!nPath.toString().matches(assetPattern))
      throw new PipelineException(
          "This tool only works on assets." + nPath.toString() + " " + nPath.toOsString());

    // if(!pPrimary.matches(matPattern))
    // throw new PipelineException("This tool will only work on a lgt node!");

    File errFile;
    try {
      errFile = File.createTempFile("MaterialsCheckOut", ".err", PackageInfo.sTempPath.toFile());
      err = new PrintWriter(errFile);
      FileCleaner.add(errFile);
    } catch (IOException e) {
      e.printStackTrace();
    }

    NodeStatus status = pSelected.get(pPrimary);
    NodeID nodeID = status.getNodeID();
    pUser = nodeID.getAuthor();
    pView = nodeID.getView();

    OverallNodeState state = status.getHeavyDetails().getOverallNodeState();
    JToolDialog tool = new JToolDialog("MaterialsCheckOut", new JPanel(), "Continue");
    if (!state.equals(OverallNodeState.Identical)) {
      JConfirmDialog dialog =
          new JConfirmDialog(
              tool,
              "This node is different from the checked in node. Do you want to continue with this check out?");
      dialog.setVisible(true);
      if (!dialog.wasConfirmed()) {
        return null;
      } // end if
    } // end if

    return "...Sing Hallelujah!";
  }
Beispiel #10
0
  public void copyInitialState(Path origAppDir) throws IOException {
    // locate previous snapshot
    String newAppDir = this.dag.assertAppPath();

    FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(origAppDir.toString(), conf);
    // read snapshot against new dependencies
    Object snapshot = recoveryHandler.restore();
    if (snapshot == null) {
      throw new IllegalArgumentException("No previous application state found in " + origAppDir);
    }
    InputStream logIs = recoveryHandler.getLog();

    // modify snapshot state to switch app id
    ((StreamingContainerManager.CheckpointState) snapshot).setApplicationId(this.dag, conf);
    Path checkpointPath = new Path(newAppDir, LogicalPlan.SUBDIR_CHECKPOINTS);

    FileSystem fs = FileSystem.newInstance(origAppDir.toUri(), conf);
    // remove the path that was created by the storage agent during deserialization and replacement
    fs.delete(checkpointPath, true);

    // write snapshot to new location
    recoveryHandler = new FSRecoveryHandler(newAppDir, conf);
    recoveryHandler.save(snapshot);
    OutputStream logOs = recoveryHandler.rotateLog();
    IOUtils.copy(logIs, logOs);
    logOs.flush();
    logOs.close();
    logIs.close();

    // copy sub directories that are not present in target
    FileStatus[] lFiles = fs.listStatus(origAppDir);
    for (FileStatus f : lFiles) {
      if (f.isDirectory()) {
        String targetPath = f.getPath().toString().replace(origAppDir.toString(), newAppDir);
        if (!fs.exists(new Path(targetPath))) {
          LOG.debug("Copying {} to {}", f.getPath(), targetPath);
          FileUtil.copy(fs, f.getPath(), fs, new Path(targetPath), false, conf);
          // FSUtil.copy(fs, f, fs, new Path(targetPath), false, false, conf);
        } else {
          LOG.debug("Ignoring {} as it already exists under {}", f.getPath(), targetPath);
          // FSUtil.setPermission(fs, new Path(targetPath), new FsPermission((short)0777));
        }
      }
    }
  }
 protected boolean isPathAcceptable(final Path pPath1) {
   String path = pPath1.toString().toLowerCase();
   if (path.startsWith("part-r-")) return true;
   String extension = getExtension();
   if (extension != null && path.endsWith(extension.toLowerCase())) return true;
   if (extension != null && path.endsWith(extension.toLowerCase() + ".gz")) return true;
   //noinspection SimplifiableIfStatement,RedundantIfStatement
   if (extension == null) return true;
   return false;
 }
Beispiel #12
0
 /*
  * find the parent path that is the
  * archive path in the path. The last
  * path segment that ends with .har is
  * the path that will be returned.
  */
 private Path archivePath(Path p) {
   Path retPath = null;
   Path tmp = p;
   for (int i = 0; i < p.depth(); i++) {
     if (tmp.toString().endsWith(".har")) {
       retPath = tmp;
       break;
     }
     tmp = tmp.getParent();
   }
   return retPath;
 }
Beispiel #13
0
 // the relative path of p. basically
 // getting rid of /. Parsing and doing
 // string manipulation is not good - so
 // just use the path api to do it.
 private Path makeRelative(String initial, Path p) {
   String scheme = this.uri.getScheme();
   String authority = this.uri.getAuthority();
   Path root = new Path(Path.SEPARATOR);
   if (root.compareTo(p) == 0) return new Path(scheme, authority, initial);
   Path retPath = new Path(p.getName());
   Path parent = p.getParent();
   for (int i = 0; i < p.depth() - 1; i++) {
     retPath = new Path(parent.getName(), retPath);
     parent = parent.getParent();
   }
   return new Path(new Path(scheme, authority, initial), retPath.toString());
 }
Beispiel #14
0
 private static boolean writeChecksum(Path localPath, String checksum) {
   try {
     java.io.BufferedWriter writer =
         Files.newBufferedWriter(
             java.nio.file.Paths.get(localPath.toString() + ".crc"), Charset.forName("UTF-8"));
     writer.write(checksum);
     writer.flush();
     writer.close();
     return true;
   } catch (Exception e) {
     logger.warn("Unable to write CRC file!");
     return false;
   }
 }
 public static void recursePath(Configuration conf, Path path, Job job) {
   try {
     FileSystem fs = path.getFileSystem(conf);
     FileStatus[] fstats = fs.listStatus(path);
     if (fstats != null) {
       for (FileStatus f : fstats) {
         Path p = f.getPath();
         ;
         if (fs.isFile(p)) {
           // connection times out otherwise
           System.err.println("file:" + p.toString());
           FileInputFormat.addInputPath(job, p);
         } else {
           System.err.println("dir:" + p.toString());
           recursePath(conf, p, job);
         }
       }
     }
   } catch (IOException e) {
     // shouldn't be here
     throw new RuntimeException(e);
   }
 }
Beispiel #16
0
  /**
   * Convert a value to an Oak markup representation.
   *
   * @param value The value to represent.
   * @param tb The buffer in which to write the markup.
   */
  public void toMarkup(Object value, Text t) {
    if (value == null) {
      t.append("null");
    } else {
      switch (this) {
        case z:
        case Z:
        case f:
        case F:
        case bool:
          t.append(value.toString());
          break;

        case cardinality:
          ((Cardinality) value).toString(t);
          break;

        case text:
          escapeForOak((String) value, true, t);
          break;

        case identifier:
          t.append(value.toString());
          break;

        case path:
          ((Path) value).toString(t);
          break;

        case datetime:
          t.append('@');
          t.append(DateU.formatStandardDatetime((LocalDateTime) value));
          break;

        case date:
          t.append('@');
          t.append(DateU.formatStandardDate((LocalDate) value));
          break;

        case time:
          t.append('@');
          t.append(DateU.formatStandardTime((LocalTime) value));
          break;

        case any:
        default:
          throw new UnexpectedException("asString: " + this);
      }
    }
  }
Beispiel #17
0
 private static void extractAndLoadNativeLib(String nativeLibName, Path target) {
   //		System.err.println("loading "+nativeLibName);
   final Path path = Paths.get(target.toString(), nativeLibName);
   if (!path.toFile().exists()) {
     try (InputStream is =
         CudaEngine.class
             .getClassLoader()
             .getResourceAsStream("/lib/" + nativeLibName)) { // TODO TK property for lib dir
       Files.copy(is, path);
     } catch (IOException e) {
       e.printStackTrace();
     } catch (NullPointerException e) { // TODO find a way to do it instead of eclipse
       final Path eclipsePath = FileSystems.getDefault().getPath("lib", nativeLibName);
       try {
         Files.copy(eclipsePath, path);
       } catch (IOException e1) {
         // TODO Auto-generated catch block
         e1.printStackTrace();
       }
     }
   }
   System.load(path.toString());
   //		System.load(nativeLibName);
 }
Beispiel #18
0
 /**
  * this method returns the path inside the har filesystem. this is relative path inside the har
  * filesystem.
  *
  * @param path the fully qualified path in the har filesystem.
  * @return relative path in the filesystem.
  */
 private Path getPathInHar(Path path) {
   Path harPath = new Path(path.toUri().getPath());
   if (archivePath.compareTo(harPath) == 0) return new Path(Path.SEPARATOR);
   Path tmp = new Path(harPath.getName());
   Path parent = harPath.getParent();
   while (!(parent.compareTo(archivePath) == 0)) {
     if (parent.toString().equals(Path.SEPARATOR)) {
       tmp = null;
       break;
     }
     tmp = new Path(parent.getName(), tmp);
     parent = parent.getParent();
   }
   if (tmp != null) tmp = new Path(Path.SEPARATOR, tmp);
   return tmp;
 }
Beispiel #19
0
 private static boolean checksumLocalTest(Path localPath, String checksum) {
   String crcPath = localPath.toString() + ".crc";
   if (Files.isReadable(java.nio.file.Paths.get(crcPath))) {
     try {
       List<String> lines =
           Files.readAllLines(java.nio.file.Paths.get(crcPath), Charset.forName("UTF-8"));
       for (String line : lines) {
         if (line.equals(checksum)) {
           return true;
         }
       }
     } catch (Exception e) {
       return false;
     }
   }
   return false;
 }
 @Override
 public void run() {
   for (CacheStatus lcacheStatus : toBeDeletedCache) {
     synchronized (lcacheStatus) {
       Path fullUniqueParentDir =
           new Path(lcacheStatus.localizedBaseDir, lcacheStatus.uniqueParentDir);
       try {
         LOG.info("Deleting local cached path: " + fullUniqueParentDir.toString());
         deleteLocalPath(asyncDiskService, fs, fullUniqueParentDir);
         // decrement the size of the cache from baseDirSize
         deleteCacheInfoUpdate(lcacheStatus);
         LOG.info("Removed cache " + lcacheStatus.localizedLoadPath);
       } catch (IOException e) {
         LOG.warn("Error when deleting " + fullUniqueParentDir, e);
       }
     }
   }
 }
Beispiel #21
0
  private void laden(Path saveName) throws IOException {
    Properties prop = new Properties();

    FileInputStream in = new FileInputStream(saveName.toString());
    prop.load(in);

    for (int i = 0; prop.containsKey(String.format("quellMenu%d", i)); i++)
      quellListModel.addElement(
          new ListItem(
              Paths.get(prop.getProperty(String.format("quellMenu%d", i))),
              Paths.get(prop.getProperty(String.format("quellMenu%d", i)))));
    for (int i = 0; prop.containsKey(String.format("zielMenu%d", i)); i++)
      zielListModel.addElement(
          new ListItem(
              Paths.get(prop.getProperty(String.format("zielMenu%d", i))),
              Paths.get(prop.getProperty(String.format("zielMenu%d", i)))));

    in.close();
  }
Beispiel #22
0
    private int getNumberOfItems(Path quellOrdner) {
      int retValue = 0;

      try {
        DirectoryStream<Path> qstream = Files.newDirectoryStream(quellOrdner);
        for (Path qfile : qstream) {
          if (Files.isDirectory(qfile)) {
            getNumberOfItems(Paths.get(quellOrdner.toString() + "/" + qfile.getFileName()));
          }
          i++;
        }
        qstream.close();
      } catch (IOException e) {
        e.printStackTrace();
      }

      retValue = i;
      return retValue;
    }
Beispiel #23
0
 private static void extractAndLoadNativeLibs() throws IOException {
   Path target = Paths.get(ioTmpDir, "/tklib");
   if (!target.toFile().exists()) {
     Files.createDirectories(target);
   }
   final boolean windows = System.getProperty("os.name").equalsIgnoreCase("windows");
   String fileExtension = windows ? "dll" : "so";
   String prefix = windows ? "" : "lib";
   String libPattern = fileExtension.equals("dll") ? "-windows" : "-linux" + "-x86";
   if (System.getProperty("sun.arch.data.model").equals("64")) {
     libPattern += "_64";
   }
   libPattern += "." + fileExtension;
   //		System.err.println(libPattern);
   System.setProperty("java.library.path", target.toString());
   //		System.err.println(System.getProperty("java.library.path"));
   extractAndLoadNativeLib(prefix + "JCudaDriver" + libPattern, target);
   extractAndLoadNativeLib(prefix + "JCudaRuntime" + libPattern, target);
   extractAndLoadNativeLib(prefix + "JCurand" + libPattern, target);
 }
 private static void createSymlink(
     Configuration conf,
     URI cache,
     CacheStatus cacheStatus,
     boolean isArchive,
     Path currentWorkDir,
     boolean honorSymLinkConf)
     throws IOException {
   boolean doSymlink = honorSymLinkConf && DistributedCache.getSymlink(conf);
   if (cache.getFragment() == null) {
     doSymlink = false;
   }
   String link = currentWorkDir.toString() + Path.SEPARATOR + cache.getFragment();
   File flink = new File(link);
   if (doSymlink) {
     if (!flink.exists()) {
       FileUtil.symLink(cacheStatus.localizedLoadPath.toString(), link);
     }
   }
 }
 /**
  * open a file for reading
  *
  * @param hdfsPath !null path - probably of an existing file
  * @return !null stream
  */
 @Override
 public InputStream openFileForRead(Path src) {
   if (isRunningAsUser()) {
     return super.openFileForRead(src);
   }
   String hdfsPath = src.toString();
   if (isFileNameLocal(hdfsPath)) {
     try {
       return new FileInputStream(hdfsPath); // better be local
     } catch (FileNotFoundException e) {
       throw new RuntimeException(e);
     }
   }
   if (true) throw new UnsupportedOperationException("Fix This"); // ToDo
   final FileSystem fs = getDFS();
   try {
     return fs.open(src);
   } catch (IOException e) {
     throw new RuntimeException(e);
   }
 }
Beispiel #26
0
 // void compileKernelsPtx()
 static void compileKernelsPtx() throws IOException {
   if (!new File(ioTmpDir, PHEROMONES_CU + ".ptx").exists()) { // TODO externalize
     try (InputStream is =
         CudaEngine.class.getResourceAsStream(
             "/turtlekit/cuda/kernels/" + PHEROMONES_CU + ".cu")) {
       final Path path = Paths.get(ioTmpDir, PHEROMONES_CU + ".cu");
       try {
         Files.copy(is, path);
       } catch (FileAlreadyExistsException e) {
       }
       System.err.println("--------------- Compiling ptx ----------------------");
       KernelLauncher.create(
           path.toString(),
           Kernel.DIFFUSION_TO_TMP.name(),
           false,
           "--use_fast_math",
           "--prec-div=false"); // ,"--gpu-architecture=sm_20");
     } catch (IOException e) {
       throw e;
     }
   }
 }
Beispiel #27
0
  private void speichern(Path saveName) {
    Properties prop = new Properties();

    if (!quellListModel.isEmpty())
      for (int i = 0; i < quellListModel.getSize(); i++)
        prop.setProperty(
            String.format("quellMenu%d", i),
            quellListModel.getElementAt(i).getValueMember().toString());

    if (!zielListModel.isEmpty())
      for (int i = 0; i < zielListModel.getSize(); i++)
        prop.setProperty(
            String.format("zielMenu%d", i),
            zielListModel.getElementAt(i).getValueMember().toString());

    try {
      FileOutputStream out = new FileOutputStream(saveName.toString());
      prop.store(out, null);
      out.close();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Beispiel #28
0
  public void downloadFile(String relPath) {
    VOSync.debug("Downloading file from storage: " + relPath);
    Path filePath = FileSystems.getDefault().getPath(startDir.toString(), relPath.substring(1));
    try {
      WatchKey key =
          filePath.getParent().register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
      keys.remove(key);
      key.cancel();

      FileOutputStream outp = new FileOutputStream(filePath.toFile());
      DropboxFileInfo info = api.getFile(relPath, null, outp, null);
      outp.close();

      key = filePath.getParent().register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
      keys.put(key, filePath.getParent());

      MetaHandler.setFile(relPath, filePath.toFile(), info.getMetadata().rev);
    } catch (IOException ex) {
      logger.error("Error downloading file " + relPath + ": " + ex.getMessage());
    } catch (DropboxException ex) {
      ex.printStackTrace();
      logger.error("Error downloading file " + relPath + ": " + ex.getMessage());
    }
  }
Beispiel #29
0
  @Override
  public void run() {
    logger.debug("Register root " + startDir.toString());

    try {
      registerAll(startDir);
    } catch (IOException ex) {
      logger.error(ex.getMessage());
      return;
    }

    if (isInterrupted()) return;

    VOSync.debug("Sync local db with drive");

    DbPool.goSql(
        "Synching the local db with drive",
        "select NAME from FILES",
        new SqlWorker<Boolean>() {
          @Override
          public Boolean go(Connection conn, PreparedStatement stmt) throws SQLException {
            ResultSet resSet = stmt.executeQuery();
            while (resSet.next()) {
              try {
                String fileName = resSet.getString(1);
                Path filePath =
                    FileSystems.getDefault().getPath(startDir.toString(), fileName.substring(1));
                if (!filePath.toFile().exists()) {
                  logger.debug(
                      "Deleting file " + fileName + " existing in DB and not present on disk");
                  api.delete(fileName);
                  MetaHandler.delete(fileName);
                }
              } catch (DropboxException ex) {
              }
            }
            resSet.close();
            return true;
          }
        });

    if (isInterrupted()) return;

    VOSync.debug("Sync storage");

    syncStorage();

    logger.debug("Start watching");

    while (!isInterrupted()) {
      WatchKey key;
      try {
        key = watcher.take();
      } catch (InterruptedException x) {
        return;
      }

      Path dir = keys.get(key);
      if (dir == null) {
        System.err.println("WatchKey " + key.toString() + " not recognized!");
        continue;
      }

      for (WatchEvent<?> event : key.pollEvents()) {
        Kind<?> kind = event.kind();

        // TBD - provide example of how OVERFLOW event is handled
        if (kind == OVERFLOW) {
          continue;
        }

        // Context for directory entry event is the file name of entry
        WatchEvent<Path> ev = cast(event);
        Path name = ev.context();
        Path child = dir.resolve(name);
        Path relativeDir = startDir.relativize(child);
        String fileRelPath = "/" + fixPath(relativeDir.toString());

        // print out event
        logger.debug(event.kind().name() + ":" + child + " " + name + " " + key);

        try {
          if (Files.exists(child, new LinkOption[] {}) && Files.isHidden(child)) {
            logger.error(
                "Skipping hidden file " + child.getFileName()); // skip OS generated catalog files
          } else {
            if (event.kind() == ENTRY_CREATE) {
              if (Files.isRegularFile(child, NOFOLLOW_LINKS)) { // file modified
                uploadFile(fileRelPath, child);
              } else if (Files.isDirectory(child, NOFOLLOW_LINKS)) { // directory contents changed
                registerAll(child);
              }
            } else if (event.kind() == ENTRY_DELETE) {
              logger.debug("Deleting " + fileRelPath);
              api.delete(fileRelPath);
              MetaHandler.delete(fileRelPath);
              logger.debug("Deleted!");
            } else if (event.kind() == ENTRY_MODIFY) {
              if (Files.isRegularFile(child, NOFOLLOW_LINKS)) { // file modified
                uploadFile(fileRelPath, child);
              } else if (Files.isDirectory(child, NOFOLLOW_LINKS)) { // directory contents changed
                // logger.debug("Renewing dir: "+relativeDir.toString());
                // TODO update folder date
                // MetaHandler.setFile(fileRelPath, child, rev);
              }
            }
          }
        } catch (IOException ex) {
          ex.printStackTrace();
          logger.error(ex.getMessage());
        } catch (DropboxException ex) {
          ex.printStackTrace();
          logger.error(ex.getMessage());
        }
      }

      boolean valid = key.reset();

      if (!valid) keys.remove(key);
    }
  }
Beispiel #30
0
  /**
   * Launch application for the dag represented by this client.
   *
   * @throws YarnException
   * @throws IOException
   */
  public void startApplication() throws YarnException, IOException {
    Class<?>[] defaultClasses;

    if (applicationType.equals(YARN_APPLICATION_TYPE)) {
      // TODO restrict the security check to only check if security is enabled for webservices.
      if (UserGroupInformation.isSecurityEnabled()) {
        defaultClasses = DATATORRENT_SECURITY_CLASSES;
      } else {
        defaultClasses = DATATORRENT_CLASSES;
      }
    } else {
      throw new IllegalStateException(applicationType + " is not a valid application type.");
    }

    LinkedHashSet<String> localJarFiles = findJars(dag, defaultClasses);

    if (resources != null) {
      localJarFiles.addAll(resources);
    }

    YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
    LOG.info(
        "Got Cluster metric info from ASM"
            + ", numNodeManagers="
            + clusterMetrics.getNumNodeManagers());

    // GetClusterNodesRequest clusterNodesReq = Records.newRecord(GetClusterNodesRequest.class);
    // GetClusterNodesResponse clusterNodesResp =
    // rmClient.clientRM.getClusterNodes(clusterNodesReq);
    // LOG.info("Got Cluster node info from ASM");
    // for (NodeReport node : clusterNodesResp.getNodeReports()) {
    //  LOG.info("Got node report from ASM for"
    //           + ", nodeId=" + node.getNodeId()
    //           + ", nodeAddress" + node.getHttpAddress()
    //           + ", nodeRackName" + node.getRackName()
    //           + ", nodeNumContainers" + node.getNumContainers()
    //           + ", nodeHealthStatus" + node.getHealthReport());
    // }
    List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
    for (QueueUserACLInfo aclInfo : listAclInfo) {
      for (QueueACL userAcl : aclInfo.getUserAcls()) {
        LOG.info(
            "User ACL Info for Queue"
                + ", queueName="
                + aclInfo.getQueueName()
                + ", userAcl="
                + userAcl.name());
      }
    }

    // Get a new application id
    YarnClientApplication newApp = yarnClient.createApplication();
    appId = newApp.getNewApplicationResponse().getApplicationId();

    // Dump out information about cluster capability as seen by the resource manager
    int maxMem = newApp.getNewApplicationResponse().getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
    int amMemory = dag.getMasterMemoryMB();
    if (amMemory > maxMem) {
      LOG.info(
          "AM memory specified above max threshold of cluster. Using max value."
              + ", specified="
              + amMemory
              + ", max="
              + maxMem);
      amMemory = maxMem;
    }

    if (dag.getAttributes().get(LogicalPlan.APPLICATION_ID) == null) {
      dag.setAttribute(LogicalPlan.APPLICATION_ID, appId.toString());
    }

    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);
    // set the application name
    appContext.setApplicationName(dag.getValue(LogicalPlan.APPLICATION_NAME));
    appContext.setApplicationType(this.applicationType);
    if (YARN_APPLICATION_TYPE.equals(this.applicationType)) {
      // appContext.setMaxAppAttempts(1); // no retries until Stram is HA
    }

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    // Setup security tokens
    // If security is enabled get ResourceManager and NameNode delegation tokens.
    // Set these tokens on the container so that they are sent as part of application submission.
    // This also sets them up for renewal by ResourceManager. The NameNode delegation rmToken
    // is also used by ResourceManager to fetch the jars from HDFS and set them up for the
    // application master launch.
    if (UserGroupInformation.isSecurityEnabled()) {
      Credentials credentials = new Credentials();
      String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
      if (tokenRenewer == null || tokenRenewer.length() == 0) {
        throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
      }

      // For now, only getting tokens for the default file-system.
      FileSystem fs = StramClientUtils.newFileSystemInstance(conf);
      try {
        final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
        if (tokens != null) {
          for (Token<?> token : tokens) {
            LOG.info("Got dt for " + fs.getUri() + "; " + token);
          }
        }
      } finally {
        fs.close();
      }

      addRMDelegationToken(tokenRenewer, credentials);

      DataOutputBuffer dob = new DataOutputBuffer();
      credentials.writeTokenStorageToStream(dob);
      ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
      amContainer.setTokens(fsTokens);
    }

    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    // copy required jar files to dfs, to be localized for containers
    FileSystem fs = StramClientUtils.newFileSystemInstance(conf);
    try {
      Path appsBasePath =
          new Path(StramClientUtils.getDTDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
      Path appPath = new Path(appsBasePath, appId.toString());

      String libJarsCsv = copyFromLocal(fs, appPath, localJarFiles.toArray(new String[] {}));

      LOG.info("libjars: {}", libJarsCsv);
      dag.getAttributes().put(LogicalPlan.LIBRARY_JARS, libJarsCsv);
      LaunchContainerRunnable.addFilesToLocalResources(
          LocalResourceType.FILE, libJarsCsv, localResources, fs);

      if (archives != null) {
        String[] localFiles = archives.split(",");
        String archivesCsv = copyFromLocal(fs, appPath, localFiles);
        LOG.info("archives: {}", archivesCsv);
        dag.getAttributes().put(LogicalPlan.ARCHIVES, archivesCsv);
        LaunchContainerRunnable.addFilesToLocalResources(
            LocalResourceType.ARCHIVE, archivesCsv, localResources, fs);
      }

      if (files != null) {
        String[] localFiles = files.split(",");
        String filesCsv = copyFromLocal(fs, appPath, localFiles);
        LOG.info("files: {}", filesCsv);
        dag.getAttributes().put(LogicalPlan.FILES, filesCsv);
        LaunchContainerRunnable.addFilesToLocalResources(
            LocalResourceType.FILE, filesCsv, localResources, fs);
      }

      dag.getAttributes().put(LogicalPlan.APPLICATION_PATH, appPath.toString());
      if (dag.getAttributes().get(OperatorContext.STORAGE_AGENT) == null) {
          /* which would be the most likely case */
        Path checkpointPath = new Path(appPath, LogicalPlan.SUBDIR_CHECKPOINTS);
        // use conf client side to pickup any proxy settings from dt-site.xml
        dag.setAttribute(
            OperatorContext.STORAGE_AGENT, new FSStorageAgent(checkpointPath.toString(), conf));
      }
      if (dag.getAttributes().get(LogicalPlan.CONTAINER_OPTS_CONFIGURATOR) == null) {
        dag.setAttribute(
            LogicalPlan.CONTAINER_OPTS_CONFIGURATOR, new BasicContainerOptConfigurator());
      }

      // Set the log4j properties if needed
      if (!log4jPropFile.isEmpty()) {
        Path log4jSrc = new Path(log4jPropFile);
        Path log4jDst = new Path(appPath, "log4j.props");
        fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
        FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
        LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
        log4jRsrc.setType(LocalResourceType.FILE);
        log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
        log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
        log4jRsrc.setSize(log4jFileStatus.getLen());
        localResources.put("log4j.properties", log4jRsrc);
      }

      if (originalAppId != null) {
        Path origAppPath = new Path(appsBasePath, this.originalAppId);
        LOG.info("Restart from {}", origAppPath);
        copyInitialState(origAppPath);
      }

      // push logical plan to DFS location
      Path cfgDst = new Path(appPath, LogicalPlan.SER_FILE_NAME);
      FSDataOutputStream outStream = fs.create(cfgDst, true);
      LogicalPlan.write(this.dag, outStream);
      outStream.close();

      Path launchConfigDst = new Path(appPath, LogicalPlan.LAUNCH_CONFIG_FILE_NAME);
      outStream = fs.create(launchConfigDst, true);
      conf.writeXml(outStream);
      outStream.close();

      FileStatus topologyFileStatus = fs.getFileStatus(cfgDst);
      LocalResource topologyRsrc = Records.newRecord(LocalResource.class);
      topologyRsrc.setType(LocalResourceType.FILE);
      topologyRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
      topologyRsrc.setResource(ConverterUtils.getYarnUrlFromURI(cfgDst.toUri()));
      topologyRsrc.setTimestamp(topologyFileStatus.getModificationTime());
      topologyRsrc.setSize(topologyFileStatus.getLen());
      localResources.put(LogicalPlan.SER_FILE_NAME, topologyRsrc);

      // Set local resource info into app master container launch context
      amContainer.setLocalResources(localResources);

      // Set the necessary security tokens as needed
      // amContainer.setContainerTokens(containerToken);
      // Set the env variables to be setup in the env where the application master will be run
      LOG.info("Set the environment for the application master");
      Map<String, String> env = new HashMap<String, String>();

      // Add application jar(s) location to classpath
      // At some point we should not be required to add
      // the hadoop specific classpaths to the env.
      // It should be provided out of the box.
      // For now setting all required classpaths including
      // the classpath to "." for the application jar(s)
      // including ${CLASSPATH} will duplicate the class path in app master, removing it for now
      // StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*");
      StringBuilder classPathEnv = new StringBuilder("./*");
      String classpath = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
      for (String c :
          StringUtils.isBlank(classpath)
              ? YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH
              : classpath.split(",")) {
        if (c.equals("$HADOOP_CLIENT_CONF_DIR")) {
          // SPOI-2501
          continue;
        }
        classPathEnv.append(':');
        classPathEnv.append(c.trim());
      }
      env.put("CLASSPATH", classPathEnv.toString());
      // propagate to replace node managers user name (effective in non-secure mode)
      env.put("HADOOP_USER_NAME", UserGroupInformation.getLoginUser().getUserName());

      amContainer.setEnvironment(env);

      // Set the necessary command to execute the application master
      ArrayList<CharSequence> vargs = new ArrayList<CharSequence>(30);

      // Set java executable command
      LOG.info("Setting up app master command");
      vargs.add(javaCmd);
      if (dag.isDebug()) {
        vargs.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=n");
      }
      // Set Xmx based on am memory size
      // default heap size 75% of total memory
      vargs.add("-Xmx" + (amMemory * 3 / 4) + "m");
      vargs.add("-XX:+HeapDumpOnOutOfMemoryError");
      vargs.add("-XX:HeapDumpPath=/tmp/dt-heap-" + appId.getId() + ".bin");
      vargs.add("-Dhadoop.root.logger=" + (dag.isDebug() ? "DEBUG" : "INFO") + ",RFA");
      vargs.add("-Dhadoop.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
      vargs.add(String.format("-D%s=%s", StreamingContainer.PROP_APP_PATH, dag.assertAppPath()));
      if (dag.isDebug()) {
        vargs.add("-Dlog4j.debug=true");
      }

      String loggersLevel = conf.get(DTLoggerFactory.DT_LOGGERS_LEVEL);
      if (loggersLevel != null) {
        vargs.add(String.format("-D%s=%s", DTLoggerFactory.DT_LOGGERS_LEVEL, loggersLevel));
      }
      vargs.add(StreamingAppMaster.class.getName());
      vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
      vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");

      // Get final command
      StringBuilder command = new StringBuilder(9 * vargs.size());
      for (CharSequence str : vargs) {
        command.append(str).append(" ");
      }

      LOG.info("Completed setting up app master command " + command.toString());
      List<String> commands = new ArrayList<String>();
      commands.add(command.toString());
      amContainer.setCommands(commands);

      // Set up resource type requirements
      // For now, only memory is supported so we set memory requirements
      Resource capability = Records.newRecord(Resource.class);
      capability.setMemory(amMemory);
      appContext.setResource(capability);

      // Service data is a binary blob that can be passed to the application
      // Not needed in this scenario
      // amContainer.setServiceData(serviceData);
      appContext.setAMContainerSpec(amContainer);

      // Set the priority for the application master
      Priority pri = Records.newRecord(Priority.class);
      pri.setPriority(amPriority);
      appContext.setPriority(pri);
      // Set the queue to which this application is to be submitted in the RM
      appContext.setQueue(queueName);

      // Submit the application to the applications manager
      // SubmitApplicationResponse submitResp = rmClient.submitApplication(appRequest);
      // Ignore the response as either a valid response object is returned on success
      // or an exception thrown to denote some form of a failure
      String specStr =
          Objects.toStringHelper("Submitting application: ")
              .add("name", appContext.getApplicationName())
              .add("queue", appContext.getQueue())
              .add("user", UserGroupInformation.getLoginUser())
              .add("resource", appContext.getResource())
              .toString();
      LOG.info(specStr);
      if (dag.isDebug()) {
        // LOG.info("Full submission context: " + appContext);
      }
      yarnClient.submitApplication(appContext);
    } finally {
      fs.close();
    }
  }