private String copyFromLocal(FileSystem fs, Path basePath, String[] files) throws IOException {
StringBuilder csv = new StringBuilder(files.length * (basePath.toString().length() + 16));
for (String localFile : files) {
Path src = new Path(localFile);
String filename = src.getName();
Path dst = new Path(basePath, filename);
URI localFileURI = null;
try {
localFileURI = new URI(localFile);
} catch (URISyntaxException e) {
throw new IOException(e);
}
if (localFileURI.getScheme() == null || localFileURI.getScheme().startsWith("file")) {
LOG.info("Copy {} from local filesystem to {}", localFile, dst);
fs.copyFromLocalFile(false, true, src, dst);
} else {
LOG.info("Copy {} from DFS to {}", localFile, dst);
FileUtil.copy(fs, src, fs, dst, false, true, conf);
}
if (csv.length() > 0) {
csv.append(LIB_JARS_SEP);
}
csv.append(dst.toString());
}
return csv.toString();
}
void test(String[] opts, String className) throws Exception {
count++;
System.err.println("Test " + count + " " + Arrays.asList(opts) + " " + className);
Path testSrcDir = Paths.get(System.getProperty("test.src"));
Path testClassesDir = Paths.get(System.getProperty("test.classes"));
Path classes = Paths.get("classes." + count);
classes.createDirectory();
Context ctx = new Context();
PathFileManager fm = new JavacPathFileManager(ctx, true, null);
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
List<String> options = new ArrayList<String>();
options.addAll(Arrays.asList(opts));
options.addAll(Arrays.asList("-verbose", "-XDverboseCompilePolicy", "-d", classes.toString()));
Iterable<? extends JavaFileObject> compilationUnits =
fm.getJavaFileObjects(testSrcDir.resolve(className + ".java"));
StringWriter sw = new StringWriter();
PrintWriter out = new PrintWriter(sw);
JavaCompiler.CompilationTask t =
compiler.getTask(out, fm, null, options, null, compilationUnits);
boolean ok = t.call();
System.err.println(sw.toString());
if (!ok) {
throw new Exception("compilation failed");
}
File expect = new File("classes." + count + "/" + className + ".class");
if (!expect.exists()) throw new Exception("expected file not found: " + expect);
long expectedSize = new File(testClassesDir.toString(), className + ".class").length();
long actualSize = expect.length();
if (expectedSize != actualSize)
throw new Exception("wrong size found: " + actualSize + "; expected: " + expectedSize);
}
/**
* Add an file path to the current set of classpath entries It adds the file to cache as well.
*
* @param file Path of the file to be added
* @param conf Configuration that contains the classpath setting
*/
public static void addFileToClassPath(Path file, Configuration conf) throws IOException {
String classpath = conf.get("mapred.job.classpath.files");
conf.set(
"mapred.job.classpath.files",
classpath == null
? file.toString()
: classpath + System.getProperty("path.separator") + file.toString());
URI uri = file.makeQualified(file.getFileSystem(conf)).toUri();
addCacheFile(uri, conf);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
for (Path path : getPathContents().keySet()) {
sb.append(String.format(" A %s%n", path.toString()));
}
for (Path path : getPathsToDelete()) {
sb.append(String.format(" D %s%n", path.toString()));
}
return sb.toString();
}
// the method which actually copies the caches locally and unjars/unzips them
// and does chmod for the files
private static Path localizeCache(
Configuration conf, URI cache, long confFileStamp, CacheStatus cacheStatus, boolean isArchive)
throws IOException {
FileSystem fs = getFileSystem(cache, conf);
FileSystem localFs = FileSystem.getLocal(conf);
Path parchive = null;
if (isArchive) {
parchive =
new Path(
cacheStatus.localizedLoadPath, new Path(cacheStatus.localizedLoadPath.getName()));
} else {
parchive = cacheStatus.localizedLoadPath;
}
if (!localFs.mkdirs(parchive.getParent())) {
throw new IOException(
"Mkdirs failed to create directory " + cacheStatus.localizedLoadPath.toString());
}
String cacheId = cache.getPath();
fs.copyToLocalFile(new Path(cacheId), parchive);
if (isArchive) {
String tmpArchive = parchive.toString().toLowerCase();
File srcFile = new File(parchive.toString());
File destDir = new File(parchive.getParent().toString());
if (tmpArchive.endsWith(".jar")) {
RunJar.unJar(srcFile, destDir);
} else if (tmpArchive.endsWith(".zip")) {
FileUtil.unZip(srcFile, destDir);
} else if (isTarFile(tmpArchive)) {
FileUtil.unTar(srcFile, destDir);
}
// else will not do anyhting
// and copy the file into the dir as it is
}
long cacheSize = FileUtil.getDU(new File(parchive.getParent().toString()));
cacheStatus.size = cacheSize;
addCacheInfoUpdate(cacheStatus);
// do chmod here
try {
// Setting recursive permission to grant everyone read and execute
Path localDir = new Path(cacheStatus.localizedBaseDir, cacheStatus.uniqueParentDir);
LOG.info("Doing chmod on localdir :" + localDir);
FileUtil.chmod(localDir.toString(), "ugo+rx", true);
} catch (InterruptedException e) {
LOG.warn("Exception in chmod" + e.toString());
}
// update cacheStatus to reflect the newly cached file
cacheStatus.mtime = getTimestamp(conf, cache);
return cacheStatus.localizedLoadPath;
}
public static FileDesc loadFile(Path root, Path file, int blocSize)
throws NoSuchAlgorithmException, FileNotFoundException, IOException {
MessageDigest md = MessageDigest.getInstance("SHA-512");
MessageDigest fileMd = MessageDigest.getInstance("SHA-512");
FileDesc desc = new FileDesc(file.toString(), null, null);
List<Bloc> list = new ArrayList<Bloc>();
try (FileInputStream fis = new FileInputStream(root.resolve(file).toString())) {
byte[] buf = new byte[blocSize];
byte[] h;
int s;
while ((s = fis.read(buf)) != -1) {
int c;
while (s < buf.length && (c = fis.read()) != -1) buf[s++] = (byte) c;
fileMd.update(buf, 0, s);
// padding
byte p = 0;
while (s < buf.length) buf[s++] = ++p;
h = md.digest(buf);
Bloc bloc = new Bloc(RollingChecksum.compute(buf), new Hash(h));
list.add(bloc);
}
h = fileMd.digest();
desc.fileHash = new Hash(h);
desc.blocs = list.toArray(new Bloc[0]);
}
return desc;
}
public static void main(String[] args) throws Exception {
Path dir1 = TestUtil.createTemporaryDirectory();
try {
// Same directory
testCopyFileToFile(dir1, dir1, TestUtil.supportsLinks(dir1));
testMove(dir1, dir1, TestUtil.supportsLinks(dir1));
// Different directories. Use test.dir if possible as it might be
// a different volume/file system and so improve test coverage.
String testDir = System.getProperty("test.dir", ".");
Path dir2 = TestUtil.createTemporaryDirectory(testDir);
try {
boolean testSymbolicLinks = TestUtil.supportsLinks(dir1) && TestUtil.supportsLinks(dir2);
testCopyFileToFile(dir1, dir2, testSymbolicLinks);
testMove(dir1, dir2, testSymbolicLinks);
} finally {
TestUtil.removeAll(dir2);
}
// Target is location associated with custom provider
Path dir3 = PassThroughFileSystem.create().getPath(dir1.toString());
testCopyFileToFile(dir1, dir3, false);
testMove(dir1, dir3, false);
// Test copy(InputStream,Path) and copy(Path,OutputStream)
testCopyInputStreamToFile();
testCopyFileToOuputStream();
} finally {
TestUtil.removeAll(dir1);
}
}
public void removePlugin(String name, Terminal terminal) throws IOException {
if (name == null) {
throw new IllegalArgumentException("plugin name must be supplied with remove [name].");
}
PluginHandle pluginHandle = PluginHandle.parse(name);
boolean removed = false;
checkForForbiddenName(pluginHandle.name);
Path pluginToDelete = pluginHandle.extractedDir(environment);
if (Files.exists(pluginToDelete)) {
terminal.println(VERBOSE, "Removing: %s", pluginToDelete);
try {
IOUtils.rm(pluginToDelete);
} catch (IOException ex) {
throw new IOException(
"Unable to remove "
+ pluginHandle.name
+ ". Check file permissions on "
+ pluginToDelete.toString(),
ex);
}
removed = true;
}
Path binLocation = pluginHandle.binDir(environment);
if (Files.exists(binLocation)) {
terminal.println(VERBOSE, "Removing: %s", binLocation);
try {
IOUtils.rm(binLocation);
} catch (IOException ex) {
throw new IOException(
"Unable to remove "
+ pluginHandle.name
+ ". Check file permissions on "
+ binLocation.toString(),
ex);
}
removed = true;
}
if (removed) {
terminal.println("Removed %s", name);
} else {
terminal.println(
"Plugin %s not found. Run \"plugin list\" to get list of installed plugins.", name);
}
}
/**
* Check that the user has properly selected a target node for this tool
*
* <p>
*
* @return The phase progress message or <CODE>null</CODE> to abort early.
* @throws PipelineException If unable to validate the given user input.
*/
public synchronized String collectPhaseInput() throws PipelineException {
if ((pPrimary == null) || (pSelected.size() != 1))
throw new PipelineException("Please selected one node only.");
// folder = nPath.getParent();
// assetName = nPath.getName();
Path nPath = new Path(pPrimary);
nPath = nPath.getParentPath().getParentPath();
if (!nPath.toString().matches(assetPattern))
throw new PipelineException(
"This tool only works on assets." + nPath.toString() + " " + nPath.toOsString());
// if(!pPrimary.matches(matPattern))
// throw new PipelineException("This tool will only work on a lgt node!");
File errFile;
try {
errFile = File.createTempFile("MaterialsCheckOut", ".err", PackageInfo.sTempPath.toFile());
err = new PrintWriter(errFile);
FileCleaner.add(errFile);
} catch (IOException e) {
e.printStackTrace();
}
NodeStatus status = pSelected.get(pPrimary);
NodeID nodeID = status.getNodeID();
pUser = nodeID.getAuthor();
pView = nodeID.getView();
OverallNodeState state = status.getHeavyDetails().getOverallNodeState();
JToolDialog tool = new JToolDialog("MaterialsCheckOut", new JPanel(), "Continue");
if (!state.equals(OverallNodeState.Identical)) {
JConfirmDialog dialog =
new JConfirmDialog(
tool,
"This node is different from the checked in node. Do you want to continue with this check out?");
dialog.setVisible(true);
if (!dialog.wasConfirmed()) {
return null;
} // end if
} // end if
return "...Sing Hallelujah!";
}
public void copyInitialState(Path origAppDir) throws IOException {
// locate previous snapshot
String newAppDir = this.dag.assertAppPath();
FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(origAppDir.toString(), conf);
// read snapshot against new dependencies
Object snapshot = recoveryHandler.restore();
if (snapshot == null) {
throw new IllegalArgumentException("No previous application state found in " + origAppDir);
}
InputStream logIs = recoveryHandler.getLog();
// modify snapshot state to switch app id
((StreamingContainerManager.CheckpointState) snapshot).setApplicationId(this.dag, conf);
Path checkpointPath = new Path(newAppDir, LogicalPlan.SUBDIR_CHECKPOINTS);
FileSystem fs = FileSystem.newInstance(origAppDir.toUri(), conf);
// remove the path that was created by the storage agent during deserialization and replacement
fs.delete(checkpointPath, true);
// write snapshot to new location
recoveryHandler = new FSRecoveryHandler(newAppDir, conf);
recoveryHandler.save(snapshot);
OutputStream logOs = recoveryHandler.rotateLog();
IOUtils.copy(logIs, logOs);
logOs.flush();
logOs.close();
logIs.close();
// copy sub directories that are not present in target
FileStatus[] lFiles = fs.listStatus(origAppDir);
for (FileStatus f : lFiles) {
if (f.isDirectory()) {
String targetPath = f.getPath().toString().replace(origAppDir.toString(), newAppDir);
if (!fs.exists(new Path(targetPath))) {
LOG.debug("Copying {} to {}", f.getPath(), targetPath);
FileUtil.copy(fs, f.getPath(), fs, new Path(targetPath), false, conf);
// FSUtil.copy(fs, f, fs, new Path(targetPath), false, false, conf);
} else {
LOG.debug("Ignoring {} as it already exists under {}", f.getPath(), targetPath);
// FSUtil.setPermission(fs, new Path(targetPath), new FsPermission((short)0777));
}
}
}
}
protected boolean isPathAcceptable(final Path pPath1) {
String path = pPath1.toString().toLowerCase();
if (path.startsWith("part-r-")) return true;
String extension = getExtension();
if (extension != null && path.endsWith(extension.toLowerCase())) return true;
if (extension != null && path.endsWith(extension.toLowerCase() + ".gz")) return true;
//noinspection SimplifiableIfStatement,RedundantIfStatement
if (extension == null) return true;
return false;
}
/*
* find the parent path that is the
* archive path in the path. The last
* path segment that ends with .har is
* the path that will be returned.
*/
private Path archivePath(Path p) {
Path retPath = null;
Path tmp = p;
for (int i = 0; i < p.depth(); i++) {
if (tmp.toString().endsWith(".har")) {
retPath = tmp;
break;
}
tmp = tmp.getParent();
}
return retPath;
}
// the relative path of p. basically
// getting rid of /. Parsing and doing
// string manipulation is not good - so
// just use the path api to do it.
private Path makeRelative(String initial, Path p) {
String scheme = this.uri.getScheme();
String authority = this.uri.getAuthority();
Path root = new Path(Path.SEPARATOR);
if (root.compareTo(p) == 0) return new Path(scheme, authority, initial);
Path retPath = new Path(p.getName());
Path parent = p.getParent();
for (int i = 0; i < p.depth() - 1; i++) {
retPath = new Path(parent.getName(), retPath);
parent = parent.getParent();
}
return new Path(new Path(scheme, authority, initial), retPath.toString());
}
private static boolean writeChecksum(Path localPath, String checksum) {
try {
java.io.BufferedWriter writer =
Files.newBufferedWriter(
java.nio.file.Paths.get(localPath.toString() + ".crc"), Charset.forName("UTF-8"));
writer.write(checksum);
writer.flush();
writer.close();
return true;
} catch (Exception e) {
logger.warn("Unable to write CRC file!");
return false;
}
}
public static void recursePath(Configuration conf, Path path, Job job) {
try {
FileSystem fs = path.getFileSystem(conf);
FileStatus[] fstats = fs.listStatus(path);
if (fstats != null) {
for (FileStatus f : fstats) {
Path p = f.getPath();
;
if (fs.isFile(p)) {
// connection times out otherwise
System.err.println("file:" + p.toString());
FileInputFormat.addInputPath(job, p);
} else {
System.err.println("dir:" + p.toString());
recursePath(conf, p, job);
}
}
}
} catch (IOException e) {
// shouldn't be here
throw new RuntimeException(e);
}
}
/**
* Convert a value to an Oak markup representation.
*
* @param value The value to represent.
* @param tb The buffer in which to write the markup.
*/
public void toMarkup(Object value, Text t) {
if (value == null) {
t.append("null");
} else {
switch (this) {
case z:
case Z:
case f:
case F:
case bool:
t.append(value.toString());
break;
case cardinality:
((Cardinality) value).toString(t);
break;
case text:
escapeForOak((String) value, true, t);
break;
case identifier:
t.append(value.toString());
break;
case path:
((Path) value).toString(t);
break;
case datetime:
t.append('@');
t.append(DateU.formatStandardDatetime((LocalDateTime) value));
break;
case date:
t.append('@');
t.append(DateU.formatStandardDate((LocalDate) value));
break;
case time:
t.append('@');
t.append(DateU.formatStandardTime((LocalTime) value));
break;
case any:
default:
throw new UnexpectedException("asString: " + this);
}
}
}
private static void extractAndLoadNativeLib(String nativeLibName, Path target) {
// System.err.println("loading "+nativeLibName);
final Path path = Paths.get(target.toString(), nativeLibName);
if (!path.toFile().exists()) {
try (InputStream is =
CudaEngine.class
.getClassLoader()
.getResourceAsStream("/lib/" + nativeLibName)) { // TODO TK property for lib dir
Files.copy(is, path);
} catch (IOException e) {
e.printStackTrace();
} catch (NullPointerException e) { // TODO find a way to do it instead of eclipse
final Path eclipsePath = FileSystems.getDefault().getPath("lib", nativeLibName);
try {
Files.copy(eclipsePath, path);
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
}
System.load(path.toString());
// System.load(nativeLibName);
}
/**
* this method returns the path inside the har filesystem. this is relative path inside the har
* filesystem.
*
* @param path the fully qualified path in the har filesystem.
* @return relative path in the filesystem.
*/
private Path getPathInHar(Path path) {
Path harPath = new Path(path.toUri().getPath());
if (archivePath.compareTo(harPath) == 0) return new Path(Path.SEPARATOR);
Path tmp = new Path(harPath.getName());
Path parent = harPath.getParent();
while (!(parent.compareTo(archivePath) == 0)) {
if (parent.toString().equals(Path.SEPARATOR)) {
tmp = null;
break;
}
tmp = new Path(parent.getName(), tmp);
parent = parent.getParent();
}
if (tmp != null) tmp = new Path(Path.SEPARATOR, tmp);
return tmp;
}
private static boolean checksumLocalTest(Path localPath, String checksum) {
String crcPath = localPath.toString() + ".crc";
if (Files.isReadable(java.nio.file.Paths.get(crcPath))) {
try {
List<String> lines =
Files.readAllLines(java.nio.file.Paths.get(crcPath), Charset.forName("UTF-8"));
for (String line : lines) {
if (line.equals(checksum)) {
return true;
}
}
} catch (Exception e) {
return false;
}
}
return false;
}
@Override
public void run() {
for (CacheStatus lcacheStatus : toBeDeletedCache) {
synchronized (lcacheStatus) {
Path fullUniqueParentDir =
new Path(lcacheStatus.localizedBaseDir, lcacheStatus.uniqueParentDir);
try {
LOG.info("Deleting local cached path: " + fullUniqueParentDir.toString());
deleteLocalPath(asyncDiskService, fs, fullUniqueParentDir);
// decrement the size of the cache from baseDirSize
deleteCacheInfoUpdate(lcacheStatus);
LOG.info("Removed cache " + lcacheStatus.localizedLoadPath);
} catch (IOException e) {
LOG.warn("Error when deleting " + fullUniqueParentDir, e);
}
}
}
}
private void laden(Path saveName) throws IOException {
Properties prop = new Properties();
FileInputStream in = new FileInputStream(saveName.toString());
prop.load(in);
for (int i = 0; prop.containsKey(String.format("quellMenu%d", i)); i++)
quellListModel.addElement(
new ListItem(
Paths.get(prop.getProperty(String.format("quellMenu%d", i))),
Paths.get(prop.getProperty(String.format("quellMenu%d", i)))));
for (int i = 0; prop.containsKey(String.format("zielMenu%d", i)); i++)
zielListModel.addElement(
new ListItem(
Paths.get(prop.getProperty(String.format("zielMenu%d", i))),
Paths.get(prop.getProperty(String.format("zielMenu%d", i)))));
in.close();
}
private int getNumberOfItems(Path quellOrdner) {
int retValue = 0;
try {
DirectoryStream<Path> qstream = Files.newDirectoryStream(quellOrdner);
for (Path qfile : qstream) {
if (Files.isDirectory(qfile)) {
getNumberOfItems(Paths.get(quellOrdner.toString() + "/" + qfile.getFileName()));
}
i++;
}
qstream.close();
} catch (IOException e) {
e.printStackTrace();
}
retValue = i;
return retValue;
}
private static void extractAndLoadNativeLibs() throws IOException {
Path target = Paths.get(ioTmpDir, "/tklib");
if (!target.toFile().exists()) {
Files.createDirectories(target);
}
final boolean windows = System.getProperty("os.name").equalsIgnoreCase("windows");
String fileExtension = windows ? "dll" : "so";
String prefix = windows ? "" : "lib";
String libPattern = fileExtension.equals("dll") ? "-windows" : "-linux" + "-x86";
if (System.getProperty("sun.arch.data.model").equals("64")) {
libPattern += "_64";
}
libPattern += "." + fileExtension;
// System.err.println(libPattern);
System.setProperty("java.library.path", target.toString());
// System.err.println(System.getProperty("java.library.path"));
extractAndLoadNativeLib(prefix + "JCudaDriver" + libPattern, target);
extractAndLoadNativeLib(prefix + "JCudaRuntime" + libPattern, target);
extractAndLoadNativeLib(prefix + "JCurand" + libPattern, target);
}
private static void createSymlink(
Configuration conf,
URI cache,
CacheStatus cacheStatus,
boolean isArchive,
Path currentWorkDir,
boolean honorSymLinkConf)
throws IOException {
boolean doSymlink = honorSymLinkConf && DistributedCache.getSymlink(conf);
if (cache.getFragment() == null) {
doSymlink = false;
}
String link = currentWorkDir.toString() + Path.SEPARATOR + cache.getFragment();
File flink = new File(link);
if (doSymlink) {
if (!flink.exists()) {
FileUtil.symLink(cacheStatus.localizedLoadPath.toString(), link);
}
}
}
/**
* open a file for reading
*
* @param hdfsPath !null path - probably of an existing file
* @return !null stream
*/
@Override
public InputStream openFileForRead(Path src) {
if (isRunningAsUser()) {
return super.openFileForRead(src);
}
String hdfsPath = src.toString();
if (isFileNameLocal(hdfsPath)) {
try {
return new FileInputStream(hdfsPath); // better be local
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
}
if (true) throw new UnsupportedOperationException("Fix This"); // ToDo
final FileSystem fs = getDFS();
try {
return fs.open(src);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
// void compileKernelsPtx()
static void compileKernelsPtx() throws IOException {
if (!new File(ioTmpDir, PHEROMONES_CU + ".ptx").exists()) { // TODO externalize
try (InputStream is =
CudaEngine.class.getResourceAsStream(
"/turtlekit/cuda/kernels/" + PHEROMONES_CU + ".cu")) {
final Path path = Paths.get(ioTmpDir, PHEROMONES_CU + ".cu");
try {
Files.copy(is, path);
} catch (FileAlreadyExistsException e) {
}
System.err.println("--------------- Compiling ptx ----------------------");
KernelLauncher.create(
path.toString(),
Kernel.DIFFUSION_TO_TMP.name(),
false,
"--use_fast_math",
"--prec-div=false"); // ,"--gpu-architecture=sm_20");
} catch (IOException e) {
throw e;
}
}
}
private void speichern(Path saveName) {
Properties prop = new Properties();
if (!quellListModel.isEmpty())
for (int i = 0; i < quellListModel.getSize(); i++)
prop.setProperty(
String.format("quellMenu%d", i),
quellListModel.getElementAt(i).getValueMember().toString());
if (!zielListModel.isEmpty())
for (int i = 0; i < zielListModel.getSize(); i++)
prop.setProperty(
String.format("zielMenu%d", i),
zielListModel.getElementAt(i).getValueMember().toString());
try {
FileOutputStream out = new FileOutputStream(saveName.toString());
prop.store(out, null);
out.close();
} catch (Exception e) {
e.printStackTrace();
}
}
public void downloadFile(String relPath) {
VOSync.debug("Downloading file from storage: " + relPath);
Path filePath = FileSystems.getDefault().getPath(startDir.toString(), relPath.substring(1));
try {
WatchKey key =
filePath.getParent().register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
keys.remove(key);
key.cancel();
FileOutputStream outp = new FileOutputStream(filePath.toFile());
DropboxFileInfo info = api.getFile(relPath, null, outp, null);
outp.close();
key = filePath.getParent().register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
keys.put(key, filePath.getParent());
MetaHandler.setFile(relPath, filePath.toFile(), info.getMetadata().rev);
} catch (IOException ex) {
logger.error("Error downloading file " + relPath + ": " + ex.getMessage());
} catch (DropboxException ex) {
ex.printStackTrace();
logger.error("Error downloading file " + relPath + ": " + ex.getMessage());
}
}
@Override
public void run() {
logger.debug("Register root " + startDir.toString());
try {
registerAll(startDir);
} catch (IOException ex) {
logger.error(ex.getMessage());
return;
}
if (isInterrupted()) return;
VOSync.debug("Sync local db with drive");
DbPool.goSql(
"Synching the local db with drive",
"select NAME from FILES",
new SqlWorker<Boolean>() {
@Override
public Boolean go(Connection conn, PreparedStatement stmt) throws SQLException {
ResultSet resSet = stmt.executeQuery();
while (resSet.next()) {
try {
String fileName = resSet.getString(1);
Path filePath =
FileSystems.getDefault().getPath(startDir.toString(), fileName.substring(1));
if (!filePath.toFile().exists()) {
logger.debug(
"Deleting file " + fileName + " existing in DB and not present on disk");
api.delete(fileName);
MetaHandler.delete(fileName);
}
} catch (DropboxException ex) {
}
}
resSet.close();
return true;
}
});
if (isInterrupted()) return;
VOSync.debug("Sync storage");
syncStorage();
logger.debug("Start watching");
while (!isInterrupted()) {
WatchKey key;
try {
key = watcher.take();
} catch (InterruptedException x) {
return;
}
Path dir = keys.get(key);
if (dir == null) {
System.err.println("WatchKey " + key.toString() + " not recognized!");
continue;
}
for (WatchEvent<?> event : key.pollEvents()) {
Kind<?> kind = event.kind();
// TBD - provide example of how OVERFLOW event is handled
if (kind == OVERFLOW) {
continue;
}
// Context for directory entry event is the file name of entry
WatchEvent<Path> ev = cast(event);
Path name = ev.context();
Path child = dir.resolve(name);
Path relativeDir = startDir.relativize(child);
String fileRelPath = "/" + fixPath(relativeDir.toString());
// print out event
logger.debug(event.kind().name() + ":" + child + " " + name + " " + key);
try {
if (Files.exists(child, new LinkOption[] {}) && Files.isHidden(child)) {
logger.error(
"Skipping hidden file " + child.getFileName()); // skip OS generated catalog files
} else {
if (event.kind() == ENTRY_CREATE) {
if (Files.isRegularFile(child, NOFOLLOW_LINKS)) { // file modified
uploadFile(fileRelPath, child);
} else if (Files.isDirectory(child, NOFOLLOW_LINKS)) { // directory contents changed
registerAll(child);
}
} else if (event.kind() == ENTRY_DELETE) {
logger.debug("Deleting " + fileRelPath);
api.delete(fileRelPath);
MetaHandler.delete(fileRelPath);
logger.debug("Deleted!");
} else if (event.kind() == ENTRY_MODIFY) {
if (Files.isRegularFile(child, NOFOLLOW_LINKS)) { // file modified
uploadFile(fileRelPath, child);
} else if (Files.isDirectory(child, NOFOLLOW_LINKS)) { // directory contents changed
// logger.debug("Renewing dir: "+relativeDir.toString());
// TODO update folder date
// MetaHandler.setFile(fileRelPath, child, rev);
}
}
}
} catch (IOException ex) {
ex.printStackTrace();
logger.error(ex.getMessage());
} catch (DropboxException ex) {
ex.printStackTrace();
logger.error(ex.getMessage());
}
}
boolean valid = key.reset();
if (!valid) keys.remove(key);
}
}
/**
* Launch application for the dag represented by this client.
*
* @throws YarnException
* @throws IOException
*/
public void startApplication() throws YarnException, IOException {
Class<?>[] defaultClasses;
if (applicationType.equals(YARN_APPLICATION_TYPE)) {
// TODO restrict the security check to only check if security is enabled for webservices.
if (UserGroupInformation.isSecurityEnabled()) {
defaultClasses = DATATORRENT_SECURITY_CLASSES;
} else {
defaultClasses = DATATORRENT_CLASSES;
}
} else {
throw new IllegalStateException(applicationType + " is not a valid application type.");
}
LinkedHashSet<String> localJarFiles = findJars(dag, defaultClasses);
if (resources != null) {
localJarFiles.addAll(resources);
}
YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
LOG.info(
"Got Cluster metric info from ASM"
+ ", numNodeManagers="
+ clusterMetrics.getNumNodeManagers());
// GetClusterNodesRequest clusterNodesReq = Records.newRecord(GetClusterNodesRequest.class);
// GetClusterNodesResponse clusterNodesResp =
// rmClient.clientRM.getClusterNodes(clusterNodesReq);
// LOG.info("Got Cluster node info from ASM");
// for (NodeReport node : clusterNodesResp.getNodeReports()) {
// LOG.info("Got node report from ASM for"
// + ", nodeId=" + node.getNodeId()
// + ", nodeAddress" + node.getHttpAddress()
// + ", nodeRackName" + node.getRackName()
// + ", nodeNumContainers" + node.getNumContainers()
// + ", nodeHealthStatus" + node.getHealthReport());
// }
List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
for (QueueUserACLInfo aclInfo : listAclInfo) {
for (QueueACL userAcl : aclInfo.getUserAcls()) {
LOG.info(
"User ACL Info for Queue"
+ ", queueName="
+ aclInfo.getQueueName()
+ ", userAcl="
+ userAcl.name());
}
}
// Get a new application id
YarnClientApplication newApp = yarnClient.createApplication();
appId = newApp.getNewApplicationResponse().getApplicationId();
// Dump out information about cluster capability as seen by the resource manager
int maxMem = newApp.getNewApplicationResponse().getMaximumResourceCapability().getMemory();
LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
int amMemory = dag.getMasterMemoryMB();
if (amMemory > maxMem) {
LOG.info(
"AM memory specified above max threshold of cluster. Using max value."
+ ", specified="
+ amMemory
+ ", max="
+ maxMem);
amMemory = maxMem;
}
if (dag.getAttributes().get(LogicalPlan.APPLICATION_ID) == null) {
dag.setAttribute(LogicalPlan.APPLICATION_ID, appId.toString());
}
// Create launch context for app master
LOG.info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
// set the application id
appContext.setApplicationId(appId);
// set the application name
appContext.setApplicationName(dag.getValue(LogicalPlan.APPLICATION_NAME));
appContext.setApplicationType(this.applicationType);
if (YARN_APPLICATION_TYPE.equals(this.applicationType)) {
// appContext.setMaxAppAttempts(1); // no retries until Stram is HA
}
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
// Setup security tokens
// If security is enabled get ResourceManager and NameNode delegation tokens.
// Set these tokens on the container so that they are sent as part of application submission.
// This also sets them up for renewal by ResourceManager. The NameNode delegation rmToken
// is also used by ResourceManager to fetch the jars from HDFS and set them up for the
// application master launch.
if (UserGroupInformation.isSecurityEnabled()) {
Credentials credentials = new Credentials();
String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
if (tokenRenewer == null || tokenRenewer.length() == 0) {
throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
}
// For now, only getting tokens for the default file-system.
FileSystem fs = StramClientUtils.newFileSystemInstance(conf);
try {
final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
if (tokens != null) {
for (Token<?> token : tokens) {
LOG.info("Got dt for " + fs.getUri() + "; " + token);
}
}
} finally {
fs.close();
}
addRMDelegationToken(tokenRenewer, credentials);
DataOutputBuffer dob = new DataOutputBuffer();
credentials.writeTokenStorageToStream(dob);
ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
amContainer.setTokens(fsTokens);
}
// set local resources for the application master
// local files or archives as needed
// In this scenario, the jar file for the application master is part of the local resources
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
// copy required jar files to dfs, to be localized for containers
FileSystem fs = StramClientUtils.newFileSystemInstance(conf);
try {
Path appsBasePath =
new Path(StramClientUtils.getDTDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
Path appPath = new Path(appsBasePath, appId.toString());
String libJarsCsv = copyFromLocal(fs, appPath, localJarFiles.toArray(new String[] {}));
LOG.info("libjars: {}", libJarsCsv);
dag.getAttributes().put(LogicalPlan.LIBRARY_JARS, libJarsCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.FILE, libJarsCsv, localResources, fs);
if (archives != null) {
String[] localFiles = archives.split(",");
String archivesCsv = copyFromLocal(fs, appPath, localFiles);
LOG.info("archives: {}", archivesCsv);
dag.getAttributes().put(LogicalPlan.ARCHIVES, archivesCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.ARCHIVE, archivesCsv, localResources, fs);
}
if (files != null) {
String[] localFiles = files.split(",");
String filesCsv = copyFromLocal(fs, appPath, localFiles);
LOG.info("files: {}", filesCsv);
dag.getAttributes().put(LogicalPlan.FILES, filesCsv);
LaunchContainerRunnable.addFilesToLocalResources(
LocalResourceType.FILE, filesCsv, localResources, fs);
}
dag.getAttributes().put(LogicalPlan.APPLICATION_PATH, appPath.toString());
if (dag.getAttributes().get(OperatorContext.STORAGE_AGENT) == null) {
/* which would be the most likely case */
Path checkpointPath = new Path(appPath, LogicalPlan.SUBDIR_CHECKPOINTS);
// use conf client side to pickup any proxy settings from dt-site.xml
dag.setAttribute(
OperatorContext.STORAGE_AGENT, new FSStorageAgent(checkpointPath.toString(), conf));
}
if (dag.getAttributes().get(LogicalPlan.CONTAINER_OPTS_CONFIGURATOR) == null) {
dag.setAttribute(
LogicalPlan.CONTAINER_OPTS_CONFIGURATOR, new BasicContainerOptConfigurator());
}
// Set the log4j properties if needed
if (!log4jPropFile.isEmpty()) {
Path log4jSrc = new Path(log4jPropFile);
Path log4jDst = new Path(appPath, "log4j.props");
fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
log4jRsrc.setType(LocalResourceType.FILE);
log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
log4jRsrc.setSize(log4jFileStatus.getLen());
localResources.put("log4j.properties", log4jRsrc);
}
if (originalAppId != null) {
Path origAppPath = new Path(appsBasePath, this.originalAppId);
LOG.info("Restart from {}", origAppPath);
copyInitialState(origAppPath);
}
// push logical plan to DFS location
Path cfgDst = new Path(appPath, LogicalPlan.SER_FILE_NAME);
FSDataOutputStream outStream = fs.create(cfgDst, true);
LogicalPlan.write(this.dag, outStream);
outStream.close();
Path launchConfigDst = new Path(appPath, LogicalPlan.LAUNCH_CONFIG_FILE_NAME);
outStream = fs.create(launchConfigDst, true);
conf.writeXml(outStream);
outStream.close();
FileStatus topologyFileStatus = fs.getFileStatus(cfgDst);
LocalResource topologyRsrc = Records.newRecord(LocalResource.class);
topologyRsrc.setType(LocalResourceType.FILE);
topologyRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
topologyRsrc.setResource(ConverterUtils.getYarnUrlFromURI(cfgDst.toUri()));
topologyRsrc.setTimestamp(topologyFileStatus.getModificationTime());
topologyRsrc.setSize(topologyFileStatus.getLen());
localResources.put(LogicalPlan.SER_FILE_NAME, topologyRsrc);
// Set local resource info into app master container launch context
amContainer.setLocalResources(localResources);
// Set the necessary security tokens as needed
// amContainer.setContainerTokens(containerToken);
// Set the env variables to be setup in the env where the application master will be run
LOG.info("Set the environment for the application master");
Map<String, String> env = new HashMap<String, String>();
// Add application jar(s) location to classpath
// At some point we should not be required to add
// the hadoop specific classpaths to the env.
// It should be provided out of the box.
// For now setting all required classpaths including
// the classpath to "." for the application jar(s)
// including ${CLASSPATH} will duplicate the class path in app master, removing it for now
// StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*");
StringBuilder classPathEnv = new StringBuilder("./*");
String classpath = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
for (String c :
StringUtils.isBlank(classpath)
? YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH
: classpath.split(",")) {
if (c.equals("$HADOOP_CLIENT_CONF_DIR")) {
// SPOI-2501
continue;
}
classPathEnv.append(':');
classPathEnv.append(c.trim());
}
env.put("CLASSPATH", classPathEnv.toString());
// propagate to replace node managers user name (effective in non-secure mode)
env.put("HADOOP_USER_NAME", UserGroupInformation.getLoginUser().getUserName());
amContainer.setEnvironment(env);
// Set the necessary command to execute the application master
ArrayList<CharSequence> vargs = new ArrayList<CharSequence>(30);
// Set java executable command
LOG.info("Setting up app master command");
vargs.add(javaCmd);
if (dag.isDebug()) {
vargs.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=n");
}
// Set Xmx based on am memory size
// default heap size 75% of total memory
vargs.add("-Xmx" + (amMemory * 3 / 4) + "m");
vargs.add("-XX:+HeapDumpOnOutOfMemoryError");
vargs.add("-XX:HeapDumpPath=/tmp/dt-heap-" + appId.getId() + ".bin");
vargs.add("-Dhadoop.root.logger=" + (dag.isDebug() ? "DEBUG" : "INFO") + ",RFA");
vargs.add("-Dhadoop.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
vargs.add(String.format("-D%s=%s", StreamingContainer.PROP_APP_PATH, dag.assertAppPath()));
if (dag.isDebug()) {
vargs.add("-Dlog4j.debug=true");
}
String loggersLevel = conf.get(DTLoggerFactory.DT_LOGGERS_LEVEL);
if (loggersLevel != null) {
vargs.add(String.format("-D%s=%s", DTLoggerFactory.DT_LOGGERS_LEVEL, loggersLevel));
}
vargs.add(StreamingAppMaster.class.getName());
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");
// Get final command
StringBuilder command = new StringBuilder(9 * vargs.size());
for (CharSequence str : vargs) {
command.append(str).append(" ");
}
LOG.info("Completed setting up app master command " + command.toString());
List<String> commands = new ArrayList<String>();
commands.add(command.toString());
amContainer.setCommands(commands);
// Set up resource type requirements
// For now, only memory is supported so we set memory requirements
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(amMemory);
appContext.setResource(capability);
// Service data is a binary blob that can be passed to the application
// Not needed in this scenario
// amContainer.setServiceData(serviceData);
appContext.setAMContainerSpec(amContainer);
// Set the priority for the application master
Priority pri = Records.newRecord(Priority.class);
pri.setPriority(amPriority);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.setQueue(queueName);
// Submit the application to the applications manager
// SubmitApplicationResponse submitResp = rmClient.submitApplication(appRequest);
// Ignore the response as either a valid response object is returned on success
// or an exception thrown to denote some form of a failure
String specStr =
Objects.toStringHelper("Submitting application: ")
.add("name", appContext.getApplicationName())
.add("queue", appContext.getQueue())
.add("user", UserGroupInformation.getLoginUser())
.add("resource", appContext.getResource())
.toString();
LOG.info(specStr);
if (dag.isDebug()) {
// LOG.info("Full submission context: " + appContext);
}
yarnClient.submitApplication(appContext);
} finally {
fs.close();
}
}