/** Test copy from an input stream to a file */
static void testCopyInputStreamToFile() throws IOException {
testCopyInputStreamToFile(0);
for (int i = 0; i < 100; i++) {
testCopyInputStreamToFile(rand.nextInt(32000));
}
// FileAlreadyExistsException
Path target = createTempFile("blah", null);
try {
InputStream in = new ByteArrayInputStream(new byte[0]);
try {
copy(in, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException ignore) {
}
} finally {
delete(target);
}
Path tmpdir = createTempDirectory("blah");
try {
if (TestUtil.supportsLinks(tmpdir)) {
Path link = createSymbolicLink(tmpdir.resolve("link"), tmpdir.resolve("target"));
try {
InputStream in = new ByteArrayInputStream(new byte[0]);
try {
copy(in, link);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException ignore) {
}
} finally {
delete(link);
}
}
} finally {
delete(tmpdir);
}
// nulls
try {
copy((InputStream) null, target);
throw new RuntimeException("NullPointerException expected");
} catch (NullPointerException ignore) {
}
try {
copy(new ByteArrayInputStream(new byte[0]), (Path) null);
throw new RuntimeException("NullPointerException expected");
} catch (NullPointerException ignore) {
}
}
private void unzipPlugin(Path zip, Path target) throws IOException {
Files.createDirectories(target);
try (ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) {
ZipEntry entry;
byte[] buffer = new byte[8192];
while ((entry = zipInput.getNextEntry()) != null) {
Path targetFile = target.resolve(entry.getName());
// be on the safe side: do not rely on that directories are always extracted
// before their children (although this makes sense, but is it guaranteed?)
Files.createDirectories(targetFile.getParent());
if (entry.isDirectory() == false) {
try (OutputStream out = Files.newOutputStream(targetFile)) {
int len;
while ((len = zipInput.read(buffer)) >= 0) {
out.write(buffer, 0, len);
}
}
}
zipInput.closeEntry();
}
}
}
private static void static_files_copy(Path abs_path_from, Path abs_path_to) throws IOException {
Path from = abs_path_from.resolve("static");
Path to = abs_path_to.resolve("static");
java.nio.file.Files.walkFileTree(
from,
EnumSet.of(FileVisitOption.FOLLOW_LINKS),
Integer.MAX_VALUE,
new CopyDirectoryVisitor(from, to));
}
/**
* we check whether we need to remove the top-level folder while extracting sometimes (e.g.
* github) the downloaded archive contains a top-level folder which needs to be removed
*/
private Path findPluginRoot(Path dir) throws IOException {
if (Files.exists(dir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES))) {
return dir;
} else {
final Path[] topLevelFiles = FileSystemUtils.files(dir);
if (topLevelFiles.length == 1 && Files.isDirectory(topLevelFiles[0])) {
Path subdir = topLevelFiles[0];
if (Files.exists(subdir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES))) {
return subdir;
}
}
}
throw new RuntimeException(
"Could not find plugin descriptor '" + PluginInfo.ES_PLUGIN_PROPERTIES + "' in plugin zip");
}
private static boolean exists(Path p) {
Path base = p.getParent();
String fileName = p.getFileName().toString();
return Arrays.asList(extensions)
.stream()
.anyMatch(it -> Files.exists(base.resolve(fileName + it)));
}
// create directory in the given directory
static Path createSourceDirectory(Path dir) throws IOException {
String name = "sourcedir" + Integer.toString(rand.nextInt());
Path subdir = dir.resolve(name);
createDirectory(subdir);
randomizeAttributes(subdir);
return subdir;
}
void test(String[] opts, String className) throws Exception {
count++;
System.err.println("Test " + count + " " + Arrays.asList(opts) + " " + className);
Path testSrcDir = Paths.get(System.getProperty("test.src"));
Path testClassesDir = Paths.get(System.getProperty("test.classes"));
Path classes = Paths.get("classes." + count);
classes.createDirectory();
Context ctx = new Context();
PathFileManager fm = new JavacPathFileManager(ctx, true, null);
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
List<String> options = new ArrayList<String>();
options.addAll(Arrays.asList(opts));
options.addAll(Arrays.asList("-verbose", "-XDverboseCompilePolicy", "-d", classes.toString()));
Iterable<? extends JavaFileObject> compilationUnits =
fm.getJavaFileObjects(testSrcDir.resolve(className + ".java"));
StringWriter sw = new StringWriter();
PrintWriter out = new PrintWriter(sw);
JavaCompiler.CompilationTask t =
compiler.getTask(out, fm, null, options, null, compilationUnits);
boolean ok = t.call();
System.err.println(sw.toString());
if (!ok) {
throw new Exception("compilation failed");
}
File expect = new File("classes." + count + "/" + className + ".class");
if (!expect.exists()) throw new Exception("expected file not found: " + expect);
long expectedSize = new File(testClassesDir.toString(), className + ".class").length();
long actualSize = expect.length();
if (expectedSize != actualSize)
throw new Exception("wrong size found: " + actualSize + "; expected: " + expectedSize);
}
public static FileDesc loadFile(Path root, Path file, int blocSize)
throws NoSuchAlgorithmException, FileNotFoundException, IOException {
MessageDigest md = MessageDigest.getInstance("SHA-512");
MessageDigest fileMd = MessageDigest.getInstance("SHA-512");
FileDesc desc = new FileDesc(file.toString(), null, null);
List<Bloc> list = new ArrayList<Bloc>();
try (FileInputStream fis = new FileInputStream(root.resolve(file).toString())) {
byte[] buf = new byte[blocSize];
byte[] h;
int s;
while ((s = fis.read(buf)) != -1) {
int c;
while (s < buf.length && (c = fis.read()) != -1) buf[s++] = (byte) c;
fileMd.update(buf, 0, s);
// padding
byte p = 0;
while (s < buf.length) buf[s++] = ++p;
h = md.digest(buf);
Bloc bloc = new Bloc(RollingChecksum.compute(buf), new Hash(h));
list.add(bloc);
}
h = fileMd.digest();
desc.fileHash = new Hash(h);
desc.blocs = list.toArray(new Bloc[0]);
}
return desc;
}
/** Check that a cancelled key will never be queued */
static void testCancel(Path dir) throws IOException {
System.out.println("-- Cancel --");
try (WatchService watcher = FileSystems.getDefault().newWatchService()) {
System.out.format("register %s for events\n", dir);
WatchKey myKey = dir.register(watcher, new WatchEvent.Kind<?>[] {ENTRY_CREATE});
checkKey(myKey, dir);
System.out.println("cancel key");
myKey.cancel();
// create a file in the directory
Path file = dir.resolve("mars");
System.out.format("create: %s\n", file);
Files.createFile(file);
// poll for keys - there will be none
System.out.println("poll...");
try {
WatchKey key = watcher.poll(3000, TimeUnit.MILLISECONDS);
if (key != null) throw new RuntimeException("key should not be queued");
} catch (InterruptedException x) {
throw new RuntimeException(x);
}
// done
Files.delete(file);
System.out.println("OKAY");
}
}
public static void main(String[] args) throws IOException {
// open file but do not close it. Its existance will be checked by
// the calling script.
Paths.get(args[0]).newByteChannel(READ, WRITE, DELETE_ON_CLOSE);
// check temporary file has been deleted after closing it
Path file = File.createTempFile("blah", "tmp").toPath();
file.newByteChannel(READ, WRITE, DELETE_ON_CLOSE).close();
if (file.exists()) throw new RuntimeException("Temporary file was not deleted");
Path dir = TestUtil.createTemporaryDirectory();
try {
// check that DELETE_ON_CLOSE fails when file is a sym link
if (TestUtil.supportsLinks(dir)) {
file = dir.resolve("foo").createFile();
Path link = dir.resolve("link").createSymbolicLink(file);
try {
link.newByteChannel(READ, WRITE, DELETE_ON_CLOSE);
throw new RuntimeException("IOException expected");
} catch (IOException ignore) {
}
}
// check that DELETE_ON_CLOSE works with files created via open
// directories
DirectoryStream stream = dir.newDirectoryStream();
try {
if (stream instanceof SecureDirectoryStream) {
SecureDirectoryStream secure = (SecureDirectoryStream) stream;
file = Paths.get("foo");
Set<OpenOption> opts = new HashSet<OpenOption>();
opts.add(WRITE);
opts.add(DELETE_ON_CLOSE);
secure.newByteChannel(file, opts).close();
if (dir.resolve(file).exists()) throw new RuntimeException("File not deleted");
}
} finally {
stream.close();
}
} finally {
TestUtil.removeAll(dir);
}
}
private static String pick_template(Path abs_path_from, String name) {
String template = name + ".vm";
String val = "default.vm";
if (java.nio.file.Files.exists(abs_path_from.resolve("templates").resolve(template))) {
val = template;
}
return val;
}
public NodePath(Path path, Environment environment) throws IOException {
this.path = path;
this.indicesPath = path.resolve(INDICES_FOLDER);
this.fileStore = environment.getFileStore(path);
if (fileStore.supportsFileAttributeView("lucene")) {
this.spins = (Boolean) fileStore.getAttribute("lucene:spins");
} else {
this.spins = null;
}
}
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
throws IOException {
Path targetPath = toPath.resolve(fromPath.relativize(dir));
if (!Files.exists(targetPath)) {
targetPath.toFile().mkdirs();
// Files.createDirectory(targetPath);
}
return FileVisitResult.CONTINUE;
}
// create file of random size in given directory
static Path createSourceFile(Path dir) throws IOException {
String name = "source" + Integer.toString(rand.nextInt());
Path file = dir.resolve(name);
createFile(file);
byte[] bytes = new byte[rand.nextInt(128 * 1024)];
rand.nextBytes(bytes);
try (OutputStream out = newOutputStream(file)) {
out.write(bytes);
}
randomizeAttributes(file);
return file;
}
@Override
public void parse() throws IOException {
if (conservedRegionPath == null
|| !Files.exists(conservedRegionPath)
|| !Files.isDirectory(conservedRegionPath)) {
throw new IOException(
"Conservation directory whether does not exist, is not a directory or cannot be read");
}
Map<String, Path> files = new HashMap<>();
String chromosome;
Set<String> chromosomes = new HashSet<>();
// Reading all files in phastCons folder
DirectoryStream<Path> directoryStream =
Files.newDirectoryStream(conservedRegionPath.resolve("phastCons"));
for (Path path : directoryStream) {
chromosome = path.getFileName().toString().split("\\.")[0].replace("chr", "");
chromosomes.add(chromosome);
files.put(chromosome + "phastCons", path);
}
// Reading all files in phylop folder
directoryStream = Files.newDirectoryStream(conservedRegionPath.resolve("phylop"));
for (Path path : directoryStream) {
chromosome = path.getFileName().toString().split("\\.")[0].replace("chr", "");
chromosomes.add(chromosome);
files.put(chromosome + "phylop", path);
}
/** Now we can iterate over all the chromosomes found and process the files */
logger.debug("Chromosomes found {}", chromosomes.toString());
for (String chr : chromosomes) {
logger.debug("Processing chromosome {}, file {}", chr, files.get(chr + "phastCons"));
processFile(files.get(chr + "phastCons"), "phastCons");
logger.debug("Processing chromosome {}, file {}", chr, files.get(chr + "phylop"));
processFile(files.get(chr + "phylop"), "phylop");
}
}
/**
* Compile takes an absolute path of the app and returns an absolute path of the resulting
* Vertabrae.
*
* @param abs_path_from the absolute path where the resources of an application lives.
* @param abs_path_to the absolute path to where the application should be built to and served
* from.
* @return the absolute path to serve the application from
*/
public static String compile(Path abs_path_from, Path abs_path_to) {
System.out.println("[SPINE] Template path: " + abs_path_from);
velocity_engine.setProperty(
RuntimeConstants.FILE_RESOURCE_LOADER_PATH, abs_path_from.resolve("templates").toString());
velocity_engine.init();
System.out.println("[SPINE] From path: " + abs_path_from);
System.out.println("[SPINE] To path: " + abs_path_to);
try {
Path dir = abs_path_from.resolve("content");
Map<String, Vertabrae> file_names = extract_files(dir);
generate_markup(abs_path_from, abs_path_to, file_names);
static_files_copy(abs_path_from, abs_path_to);
} catch (IOException e) {
e.printStackTrace();
}
return abs_path_to.toAbsolutePath().toString();
}
/** Обработчик всех событий помещенных в очередь */
void processEvents() {
for (; ; ) {
WatchKey key;
try {
key = watcher.take();
} catch (InterruptedException x) {
LOG.log(Level.SEVERE, x.getMessage());
return;
}
Path dir = keys.get(key);
if (dir == null) {
LOG.log(Level.SEVERE, "Входной каталог не найден!");
continue;
}
for (WatchEvent<?> event : key.pollEvents()) {
WatchEvent.Kind kind = event.kind();
// TODO - подумать над обработчиком события OVERFLOW
if (kind == OVERFLOW) {
continue;
}
WatchEvent<Path> ev = cast(event);
Path name = ev.context();
Path child = dir.resolve(name);
// логируем событие
if (kind == ENTRY_CREATE) {
LOG.log(Level.FINEST, "{0}: {1}", new Object[] {event.kind().name(), child});
Runnable worker = new WorkerThread(child);
executor.execute(worker);
}
}
boolean valid = key.reset();
if (!valid) {
keys.remove(key);
if (keys.isEmpty()) {
break;
}
}
}
}
static void testCopyInputStreamToFile(int size) throws IOException {
Path tmpdir = createTempDirectory("blah");
Path source = tmpdir.resolve("source");
Path target = tmpdir.resolve("target");
try {
boolean testReplaceExisting = rand.nextBoolean();
// create source file
byte[] b = new byte[size];
rand.nextBytes(b);
write(source, b);
// target file might already exist
if (testReplaceExisting && rand.nextBoolean()) {
write(target, new byte[rand.nextInt(512)]);
}
// copy from stream to file
InputStream in = new FileInputStream(source.toFile());
try {
long n;
if (testReplaceExisting) {
n = copy(in, target, StandardCopyOption.REPLACE_EXISTING);
} else {
n = copy(in, target);
}
assertTrue(in.read() == -1); // EOF
assertTrue(n == size);
assertTrue(size(target) == size);
} finally {
in.close();
}
// check file
byte[] read = readAllBytes(target);
assertTrue(Arrays.equals(read, b));
} finally {
deleteIfExists(source);
deleteIfExists(target);
delete(tmpdir);
}
}
/** Check that deleting a registered directory causes the key to be cancelled and queued. */
static void testAutomaticCancel(Path dir) throws IOException {
System.out.println("-- Automatic Cancel --");
Path subdir = Files.createDirectory(dir.resolve("bar"));
try (WatchService watcher = FileSystems.getDefault().newWatchService()) {
System.out.format("register %s for events\n", subdir);
WatchKey myKey =
subdir.register(
watcher, new WatchEvent.Kind<?>[] {ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY});
System.out.format("delete: %s\n", subdir);
Files.delete(subdir);
takeExpectedKey(watcher, myKey);
System.out.println("reset key");
if (myKey.reset()) throw new RuntimeException("Key was not cancelled");
if (myKey.isValid()) throw new RuntimeException("Key is still valid");
System.out.println("OKAY");
}
}
@Test
public void testTransform() throws Exception {
Path output = folder.newFolder("output").toPath();
JavaTransformer transformer = new JavaTransformer();
val targetClass = this.getClass().getName();
BooleanHolder holder = new BooleanHolder(false);
transformer.addTransformer(
c -> {
System.out.println(
"Transforming class: " + c.getName() + " of type " + c.getClass().getSimpleName());
if (c.getName().equals(targetClass)) {
holder.value = true;
}
});
transformer.transform(input, output);
Assert.assertTrue("Transformer must process " + targetClass, holder.value);
Assert.assertTrue(exists(output.resolve("me/nallar/javatransformer/api/JavaTransformerTest.")));
}
/** Simple test of each of the standard events */
static void testEvents(Path dir) throws IOException {
System.out.println("-- Standard Events --");
FileSystem fs = FileSystems.getDefault();
Path name = fs.getPath("foo");
try (WatchService watcher = fs.newWatchService()) {
// --- ENTRY_CREATE ---
// register for event
System.out.format("register %s for ENTRY_CREATE\n", dir);
WatchKey myKey = dir.register(watcher, new WatchEvent.Kind<?>[] {ENTRY_CREATE});
checkKey(myKey, dir);
// create file
Path file = dir.resolve("foo");
System.out.format("create %s\n", file);
Files.createFile(file);
// remove key and check that we got the ENTRY_CREATE event
takeExpectedKey(watcher, myKey);
checkExpectedEvent(myKey.pollEvents(), StandardWatchEventKinds.ENTRY_CREATE, name);
System.out.println("reset key");
if (!myKey.reset()) throw new RuntimeException("key has been cancalled");
System.out.println("OKAY");
// --- ENTRY_DELETE ---
System.out.format("register %s for ENTRY_DELETE\n", dir);
WatchKey deleteKey = dir.register(watcher, new WatchEvent.Kind<?>[] {ENTRY_DELETE});
if (deleteKey != myKey) throw new RuntimeException("register did not return existing key");
checkKey(deleteKey, dir);
System.out.format("delete %s\n", file);
Files.delete(file);
takeExpectedKey(watcher, myKey);
checkExpectedEvent(myKey.pollEvents(), StandardWatchEventKinds.ENTRY_DELETE, name);
System.out.println("reset key");
if (!myKey.reset()) throw new RuntimeException("key has been cancalled");
System.out.println("OKAY");
// create the file for the next test
Files.createFile(file);
// --- ENTRY_MODIFY ---
System.out.format("register %s for ENTRY_MODIFY\n", dir);
WatchKey newKey = dir.register(watcher, new WatchEvent.Kind<?>[] {ENTRY_MODIFY});
if (newKey != myKey) throw new RuntimeException("register did not return existing key");
checkKey(newKey, dir);
System.out.format("update: %s\n", file);
try (OutputStream out = Files.newOutputStream(file, StandardOpenOption.APPEND)) {
out.write("I am a small file".getBytes("UTF-8"));
}
// remove key and check that we got the ENTRY_MODIFY event
takeExpectedKey(watcher, myKey);
checkExpectedEvent(myKey.pollEvents(), StandardWatchEventKinds.ENTRY_MODIFY, name);
System.out.println("OKAY");
// done
Files.delete(file);
}
}
private void extract(PluginHandle pluginHandle, Terminal terminal, Path pluginFile)
throws IOException {
// unzip plugin to a staging temp dir, named for the plugin
Path tmp = Files.createTempDirectory(environment.tmpFile(), null);
Path root = tmp.resolve(pluginHandle.name);
unzipPlugin(pluginFile, root);
// find the actual root (in case its unzipped with extra directory wrapping)
root = findPluginRoot(root);
// read and validate the plugin descriptor
PluginInfo info = PluginInfo.readFromProperties(root);
terminal.println(VERBOSE, "%s", info);
// update name in handle based on 'name' property found in descriptor file
pluginHandle = new PluginHandle(info.getName(), pluginHandle.version, pluginHandle.user);
final Path extractLocation = pluginHandle.extractedDir(environment);
if (Files.exists(extractLocation)) {
throw new IOException(
"plugin directory "
+ extractLocation.toAbsolutePath()
+ " already exists. To update the plugin, uninstall it first using 'remove "
+ pluginHandle.name
+ "' command");
}
// check for jar hell before any copying
if (info.isJvm()) {
jarHellCheck(root, info.isIsolated());
}
// install plugin
FileSystemUtils.copyDirectoryRecursively(root, extractLocation);
terminal.println("Installed %s into %s", pluginHandle.name, extractLocation.toAbsolutePath());
// cleanup
tryToDeletePath(terminal, tmp, pluginFile);
// take care of bin/ by moving and applying permissions if needed
Path sourcePluginBinDirectory = extractLocation.resolve("bin");
Path destPluginBinDirectory = pluginHandle.binDir(environment);
boolean needToCopyBinDirectory = Files.exists(sourcePluginBinDirectory);
if (needToCopyBinDirectory) {
if (Files.exists(destPluginBinDirectory) && !Files.isDirectory(destPluginBinDirectory)) {
tryToDeletePath(terminal, extractLocation);
throw new IOException(
"plugin bin directory " + destPluginBinDirectory + " is not a directory");
}
try {
copyBinDirectory(
sourcePluginBinDirectory, destPluginBinDirectory, pluginHandle.name, terminal);
} catch (IOException e) {
// rollback and remove potentially before installed leftovers
terminal.printError(
"Error copying bin directory [%s] to [%s], cleaning up, reason: %s",
sourcePluginBinDirectory, destPluginBinDirectory, ExceptionsHelper.detailedMessage(e));
tryToDeletePath(terminal, extractLocation, pluginHandle.binDir(environment));
throw e;
}
}
Path sourceConfigDirectory = extractLocation.resolve("config");
Path destConfigDirectory = pluginHandle.configDir(environment);
boolean needToCopyConfigDirectory = Files.exists(sourceConfigDirectory);
if (needToCopyConfigDirectory) {
if (Files.exists(destConfigDirectory) && !Files.isDirectory(destConfigDirectory)) {
tryToDeletePath(terminal, extractLocation, destPluginBinDirectory);
throw new IOException(
"plugin config directory " + destConfigDirectory + " is not a directory");
}
try {
terminal.println(
VERBOSE, "Found config, moving to %s", destConfigDirectory.toAbsolutePath());
moveFilesWithoutOverwriting(sourceConfigDirectory, destConfigDirectory, ".new");
if (Environment.getFileStore(destConfigDirectory)
.supportsFileAttributeView(PosixFileAttributeView.class)) {
// We copy owner, group and permissions from the parent ES_CONFIG directory, assuming they
// were properly set depending
// on how es was installed in the first place: can be root:elasticsearch (750) if es was
// installed from rpm/deb packages
// or most likely elasticsearch:elasticsearch if installed from tar/zip. As for
// permissions we don't rely on umask.
final PosixFileAttributes parentDirAttributes =
Files.getFileAttributeView(
destConfigDirectory.getParent(), PosixFileAttributeView.class)
.readAttributes();
// for files though, we make sure not to copy execute permissions from the parent dir and
// leave them untouched
final Set<PosixFilePermission> baseFilePermissions = new HashSet<>();
for (PosixFilePermission posixFilePermission : parentDirAttributes.permissions()) {
switch (posixFilePermission) {
case OWNER_EXECUTE:
case GROUP_EXECUTE:
case OTHERS_EXECUTE:
break;
default:
baseFilePermissions.add(posixFilePermission);
}
}
Files.walkFileTree(
destConfigDirectory,
new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
if (attrs.isRegularFile()) {
Set<PosixFilePermission> newFilePermissions =
new HashSet<>(baseFilePermissions);
Set<PosixFilePermission> currentFilePermissions =
Files.getPosixFilePermissions(file);
for (PosixFilePermission posixFilePermission : currentFilePermissions) {
switch (posixFilePermission) {
case OWNER_EXECUTE:
case GROUP_EXECUTE:
case OTHERS_EXECUTE:
newFilePermissions.add(posixFilePermission);
}
}
setPosixFileAttributes(
file,
parentDirAttributes.owner(),
parentDirAttributes.group(),
newFilePermissions);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
throws IOException {
setPosixFileAttributes(
dir,
parentDirAttributes.owner(),
parentDirAttributes.group(),
parentDirAttributes.permissions());
return FileVisitResult.CONTINUE;
}
});
} else {
terminal.println(
VERBOSE, "Skipping posix permissions - filestore doesn't support posix permission");
}
terminal.println(
VERBOSE,
"Installed %s into %s",
pluginHandle.name,
destConfigDirectory.toAbsolutePath());
} catch (IOException e) {
terminal.printError(
"Error copying config directory [%s] to [%s], cleaning up, reason: %s",
sourceConfigDirectory, destConfigDirectory, ExceptionsHelper.detailedMessage(e));
tryToDeletePath(terminal, extractLocation, destPluginBinDirectory, destConfigDirectory);
throw e;
}
}
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.copy(file, toPath.resolve(fromPath.relativize(file)), copyOption);
return FileVisitResult.CONTINUE;
}
/** Tests all possible ways to invoke move */
static void testMove(Path dir1, Path dir2, boolean supportsLinks) throws IOException {
Path source, target, entry;
boolean sameDevice = getFileStore(dir1).equals(getFileStore(dir2));
// -- regular file --
/** Test: move regular file, target does not exist */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
moveAndVerify(source, target);
delete(target);
/** Test: move regular file, target exists */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
createFile(target);
try {
moveAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(target);
createDirectory(target);
try {
moveAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(source);
delete(target);
/** Test: move regular file, target does not exist */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
moveAndVerify(source, target, REPLACE_EXISTING);
delete(target);
/** Test: move regular file, target exists */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
createFile(target);
moveAndVerify(source, target, REPLACE_EXISTING);
delete(target);
/** Test: move regular file, target exists and is empty directory */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
createDirectory(target);
moveAndVerify(source, target, REPLACE_EXISTING);
delete(target);
/** Test: move regular file, target exists and is non-empty directory */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
createDirectory(target);
entry = target.resolve("foo");
createFile(entry);
try {
moveAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(entry);
delete(source);
delete(target);
/** Test atomic move of regular file (same file store) */
source = createSourceFile(dir1);
target = getTargetFile(dir1);
moveAndVerify(source, target, ATOMIC_MOVE);
delete(target);
/** Test atomic move of regular file (different file store) */
if (!sameDevice) {
source = createSourceFile(dir1);
target = getTargetFile(dir2);
try {
moveAndVerify(source, target, ATOMIC_MOVE);
throw new RuntimeException("AtomicMoveNotSupportedException expected");
} catch (AtomicMoveNotSupportedException x) {
}
delete(source);
}
// -- directories --
/*
* Test: move empty directory, target does not exist
*/
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
moveAndVerify(source, target);
delete(target);
/** Test: move empty directory, target exists */
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
createFile(target);
try {
moveAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(target);
createDirectory(target);
try {
moveAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(source);
delete(target);
/** Test: move empty directory, target does not exist */
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
moveAndVerify(source, target, REPLACE_EXISTING);
delete(target);
/** Test: move empty directory, target exists */
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
createFile(target);
moveAndVerify(source, target, REPLACE_EXISTING);
delete(target);
/** Test: move empty, target exists and is empty directory */
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
createDirectory(target);
moveAndVerify(source, target, REPLACE_EXISTING);
delete(target);
/** Test: move empty directory, target exists and is non-empty directory */
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
createDirectory(target);
entry = target.resolve("foo");
createFile(entry);
try {
moveAndVerify(source, target, REPLACE_EXISTING);
throw new RuntimeException("DirectoryNotEmptyException expected");
} catch (DirectoryNotEmptyException x) {
}
delete(entry);
delete(source);
delete(target);
/** Test: move non-empty directory (same file system) */
source = createSourceDirectory(dir1);
createFile(source.resolve("foo"));
target = getTargetFile(dir1);
moveAndVerify(source, target);
delete(target.resolve("foo"));
delete(target);
/** Test: move non-empty directory (different file store) */
if (!sameDevice) {
source = createSourceDirectory(dir1);
createFile(source.resolve("foo"));
target = getTargetFile(dir2);
try {
moveAndVerify(source, target);
throw new RuntimeException("IOException expected");
} catch (IOException x) {
}
delete(source.resolve("foo"));
delete(source);
}
/** Test atomic move of directory (same file store) */
source = createSourceDirectory(dir1);
createFile(source.resolve("foo"));
target = getTargetFile(dir1);
moveAndVerify(source, target, ATOMIC_MOVE);
delete(target.resolve("foo"));
delete(target);
// -- symbolic links --
/** Test: Move symbolic link to file, target does not exist */
if (supportsLinks) {
Path tmp = createSourceFile(dir1);
source = dir1.resolve("link");
createSymbolicLink(source, tmp);
target = getTargetFile(dir2);
moveAndVerify(source, target);
delete(target);
delete(tmp);
}
/** Test: Move symbolic link to directory, target does not exist */
if (supportsLinks) {
source = dir1.resolve("link");
createSymbolicLink(source, dir2);
target = getTargetFile(dir2);
moveAndVerify(source, target);
delete(target);
}
/** Test: Move broken symbolic link, target does not exists */
if (supportsLinks) {
Path tmp = Paths.get("doesnotexist");
source = dir1.resolve("link");
createSymbolicLink(source, tmp);
target = getTargetFile(dir2);
moveAndVerify(source, target);
delete(target);
}
/** Test: Move symbolic link, target exists */
if (supportsLinks) {
source = dir1.resolve("link");
createSymbolicLink(source, dir2);
target = getTargetFile(dir2);
createFile(target);
try {
moveAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(source);
delete(target);
}
/** Test: Move regular file, target exists */
if (supportsLinks) {
source = dir1.resolve("link");
createSymbolicLink(source, dir2);
target = getTargetFile(dir2);
createFile(target);
moveAndVerify(source, target, REPLACE_EXISTING);
delete(target);
}
/** Test: move symbolic link, target exists and is empty directory */
if (supportsLinks) {
source = dir1.resolve("link");
createSymbolicLink(source, dir2);
target = getTargetFile(dir2);
createDirectory(target);
moveAndVerify(source, target, REPLACE_EXISTING);
delete(target);
}
/** Test: symbolic link, target exists and is non-empty directory */
if (supportsLinks) {
source = dir1.resolve("link");
createSymbolicLink(source, dir2);
target = getTargetFile(dir2);
createDirectory(target);
entry = target.resolve("foo");
createFile(entry);
try {
moveAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(entry);
delete(source);
delete(target);
}
/** Test atomic move of symbolic link (same file store) */
if (supportsLinks) {
source = dir1.resolve("link");
createSymbolicLink(source, dir1);
target = getTargetFile(dir2);
createFile(target);
moveAndVerify(source, target, REPLACE_EXISTING);
delete(target);
}
// -- misc. tests --
/** Test nulls */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
try {
move(null, target);
throw new RuntimeException("NullPointerException expected");
} catch (NullPointerException x) {
}
try {
move(source, null);
throw new RuntimeException("NullPointerException expected");
} catch (NullPointerException x) {
}
try {
move(source, target, (CopyOption[]) null);
throw new RuntimeException("NullPointerException expected");
} catch (NullPointerException x) {
}
try {
CopyOption[] opts = {REPLACE_EXISTING, null};
move(source, target, opts);
throw new RuntimeException("NullPointerException expected");
} catch (NullPointerException x) {
}
delete(source);
/** Test UOE */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
try {
move(source, target, new CopyOption() {});
} catch (UnsupportedOperationException x) {
}
try {
move(source, target, REPLACE_EXISTING, new CopyOption() {});
} catch (UnsupportedOperationException x) {
}
delete(source);
}
/** Process all events for keys queued to the watcher */
void processEvents() {
for (; ; ) {
// wait for key to be signalled
WatchKey key;
try {
key = watcher.take();
} catch (InterruptedException x) {
return;
}
Path dir = keys.get(key);
if (dir == null) {
System.err.println("WatchKey not recognized!!");
continue;
}
for (WatchEvent<?> event : key.pollEvents()) {
WatchEvent.Kind kind = event.kind();
// TBD - provide example of how OVERFLOW event is handled
if (kind == OVERFLOW) {
continue;
}
// Context for directory entry event is the file name of entry
WatchEvent<Path> ev = cast(event);
Path name = ev.context();
Path child = dir.resolve(name);
// print out event
System.out.format("%s: %s\n", event.kind().name(), child);
// Printing manager
if (event.kind().name().equals("ENTRY_MODIFY")
&& (child.endsWith("baseFerremundoPointer.csv")
|| child.endsWith("baseFerremundoPointer2.csv"))) {
File file = new File(child.toString());
file.renameTo(new File("/home/dios/FERREMUNDO/BD/baseFerremundoPointer_.csv"));
InvoicePrintingManager manager = new InvoicePrintingManager();
manager.manage();
}
// if directory is created, and watching recursively, then
// register it and its sub-directories
if (recursive && (kind == ENTRY_CREATE)) {
try {
if (Files.isDirectory(child, NOFOLLOW_LINKS)) {
registerAll(child);
}
} catch (IOException x) {
// ignore to keep sample readbale
}
}
}
// reset key and remove from set if directory no longer accessible
boolean valid = key.reset();
if (!valid) {
keys.remove(key);
// all directories are inaccessible
if (keys.isEmpty()) {
break;
}
}
}
}
// create name for file in given directory
static Path getTargetFile(Path dir) throws IOException {
String name = "target" + Integer.toString(rand.nextInt());
return dir.resolve(name);
}
/** Resolves the given indexes directory against this NodePath */
public Path resolve(Index index) {
return indicesPath.resolve(index.name());
}
@Override
public void run() {
logger.debug("Register root " + startDir.toString());
try {
registerAll(startDir);
} catch (IOException ex) {
logger.error(ex.getMessage());
return;
}
if (isInterrupted()) return;
VOSync.debug("Sync local db with drive");
DbPool.goSql(
"Synching the local db with drive",
"select NAME from FILES",
new SqlWorker<Boolean>() {
@Override
public Boolean go(Connection conn, PreparedStatement stmt) throws SQLException {
ResultSet resSet = stmt.executeQuery();
while (resSet.next()) {
try {
String fileName = resSet.getString(1);
Path filePath =
FileSystems.getDefault().getPath(startDir.toString(), fileName.substring(1));
if (!filePath.toFile().exists()) {
logger.debug(
"Deleting file " + fileName + " existing in DB and not present on disk");
api.delete(fileName);
MetaHandler.delete(fileName);
}
} catch (DropboxException ex) {
}
}
resSet.close();
return true;
}
});
if (isInterrupted()) return;
VOSync.debug("Sync storage");
syncStorage();
logger.debug("Start watching");
while (!isInterrupted()) {
WatchKey key;
try {
key = watcher.take();
} catch (InterruptedException x) {
return;
}
Path dir = keys.get(key);
if (dir == null) {
System.err.println("WatchKey " + key.toString() + " not recognized!");
continue;
}
for (WatchEvent<?> event : key.pollEvents()) {
Kind<?> kind = event.kind();
// TBD - provide example of how OVERFLOW event is handled
if (kind == OVERFLOW) {
continue;
}
// Context for directory entry event is the file name of entry
WatchEvent<Path> ev = cast(event);
Path name = ev.context();
Path child = dir.resolve(name);
Path relativeDir = startDir.relativize(child);
String fileRelPath = "/" + fixPath(relativeDir.toString());
// print out event
logger.debug(event.kind().name() + ":" + child + " " + name + " " + key);
try {
if (Files.exists(child, new LinkOption[] {}) && Files.isHidden(child)) {
logger.error(
"Skipping hidden file " + child.getFileName()); // skip OS generated catalog files
} else {
if (event.kind() == ENTRY_CREATE) {
if (Files.isRegularFile(child, NOFOLLOW_LINKS)) { // file modified
uploadFile(fileRelPath, child);
} else if (Files.isDirectory(child, NOFOLLOW_LINKS)) { // directory contents changed
registerAll(child);
}
} else if (event.kind() == ENTRY_DELETE) {
logger.debug("Deleting " + fileRelPath);
api.delete(fileRelPath);
MetaHandler.delete(fileRelPath);
logger.debug("Deleted!");
} else if (event.kind() == ENTRY_MODIFY) {
if (Files.isRegularFile(child, NOFOLLOW_LINKS)) { // file modified
uploadFile(fileRelPath, child);
} else if (Files.isDirectory(child, NOFOLLOW_LINKS)) { // directory contents changed
// logger.debug("Renewing dir: "+relativeDir.toString());
// TODO update folder date
// MetaHandler.setFile(fileRelPath, child, rev);
}
}
}
} catch (IOException ex) {
ex.printStackTrace();
logger.error(ex.getMessage());
} catch (DropboxException ex) {
ex.printStackTrace();
logger.error(ex.getMessage());
}
}
boolean valid = key.reset();
if (!valid) keys.remove(key);
}
}
// void processEvents() {
public void run() {
System.out.println("WatchDir Thread INFO: priority=" + Thread.currentThread().getPriority());
for (; ; ) {
// wait for key to be signalled
System.out.println("WatchDir INFO: restarting loop...acquiring new key");
WatchKey key;
try {
key = watcher.take();
} catch (InterruptedException x) {
return;
}
Path dir = keys.get(key);
if (dir == null) {
System.err.println("WatchKey not recognized!!");
continue;
}
for (WatchEvent<?> event : key.pollEvents()) {
WatchEvent.Kind kind = event.kind();
// TBD - provide example of how OVERFLOW event is handled
if (kind == OVERFLOW) {
System.out.println("Encountered OVERFLOW Event - " + event);
continue;
}
// Context for directory entry event is the file name of entry
WatchEvent<Path> ev = cast(event);
Path name = ev.context();
Path child = dir.resolve(name);
// print out event
System.out.format("[WatchDir] %s: %s\n", event.kind().name(), child);
// if directory is created, and watching recursively, then
// register it and its sub-directories
if (recursive && (kind == ENTRY_CREATE)) {
try {
if (Files.isDirectory(child, NOFOLLOW_LINKS)) {
registerAll(child);
}
} catch (IOException x) {
// ignore to keep sample readbale
}
}
long t = System.currentTimeMillis();
if (!Folder.dontWatch.contains(Folder.getInternalPath(child))) {
Thread.currentThread().setPriority(Thread.NORM_PRIORITY);
System.out.println(
"WatchDir#"
+ key
+ " INFO: path="
+ child
+ ", internal="
+ Folder.getInternalPath(child)
+ " is NOT in don't watch list. Forwarding it to other peers. @"
+ Main.timeToString(t)); // DEBUG
forwardToItopic(kind, child);
} else {
Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
System.out.println(
"WatchDir#"
+ key
+ " INFO: path="
+ child
+ ", internal="
+ Folder.getInternalPath(child)
+ " IS in the don't watch list. NOT forwarding. @"
+ Main.timeToString(t)); // DEBUG
// try{
// Thread.sleep(minDelayBtwnWatchEvents);
// } catch(InterruptedException ex) {
// System.err.println("Exception:"+ex+" while trying to sleep WatchDir thread");
// ex.printStackTrace();
// }
}
}
// reset key and remove from set if directory no longer accessible
boolean valid = key.reset();
if (!valid) {
keys.remove(key);
// all directories are inaccessible
if (keys.isEmpty()) {
break;
}
}
}
}
/** Tests all possible ways to invoke copy to copy a file to a file */
static void testCopyFileToFile(Path dir1, Path dir2, boolean supportsLinks) throws IOException {
Path source, target, link, entry;
// -- regular file --
/** Test: move regular file, target does not exist */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
copyAndVerify(source, target);
delete(source);
delete(target);
/** Test: copy regular file, target exists */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
createFile(target);
try {
copyAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(target);
createDirectory(target);
try {
copyAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(source);
delete(target);
/** Test: copy regular file, target does not exist */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
copyAndVerify(source, target, REPLACE_EXISTING);
delete(source);
delete(target);
/** Test: copy regular file, target exists */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
createFile(target);
copyAndVerify(source, target, REPLACE_EXISTING);
delete(source);
delete(target);
/** Test: copy regular file, target exists and is empty directory */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
createDirectory(target);
copyAndVerify(source, target, REPLACE_EXISTING);
delete(source);
delete(target);
/** Test: copy regular file, target exists and is non-empty directory */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
createDirectory(target);
entry = target.resolve("foo");
createFile(entry);
try {
copyAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(entry);
delete(source);
delete(target);
/** Test: copy regular file + attributes */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
copyAndVerify(source, target, COPY_ATTRIBUTES);
delete(source);
delete(target);
// -- directory --
/*
* Test: copy directory, target does not exist
*/
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
copyAndVerify(source, target);
delete(source);
delete(target);
/** Test: copy directory, target exists */
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
createFile(target);
try {
copyAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(target);
createDirectory(target);
try {
copyAndVerify(source, target);
throw new RuntimeException("FileAlreadyExistsException expected");
} catch (FileAlreadyExistsException x) {
}
delete(source);
delete(target);
/** Test: copy directory, target does not exist */
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
copyAndVerify(source, target, REPLACE_EXISTING);
delete(source);
delete(target);
/** Test: copy directory, target exists */
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
createFile(target);
copyAndVerify(source, target, REPLACE_EXISTING);
delete(source);
delete(target);
/** Test: copy directory, target exists and is empty directory */
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
createDirectory(target);
copyAndVerify(source, target, REPLACE_EXISTING);
delete(source);
delete(target);
/** Test: copy directory, target exists and is non-empty directory */
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
createDirectory(target);
entry = target.resolve("foo");
createFile(entry);
try {
copyAndVerify(source, target, REPLACE_EXISTING);
throw new RuntimeException("DirectoryNotEmptyException expected");
} catch (DirectoryNotEmptyException x) {
}
delete(entry);
delete(source);
delete(target);
/*
* Test: copy directory + attributes
*/
source = createSourceDirectory(dir1);
target = getTargetFile(dir2);
copyAndVerify(source, target, COPY_ATTRIBUTES);
delete(source);
delete(target);
// -- symbolic links --
/** Test: Follow link */
if (supportsLinks) {
source = createSourceFile(dir1);
link = dir1.resolve("link");
createSymbolicLink(link, source);
target = getTargetFile(dir2);
copyAndVerify(link, target);
delete(link);
delete(source);
}
/** Test: Copy link (to file) */
if (supportsLinks) {
source = createSourceFile(dir1);
link = dir1.resolve("link");
createSymbolicLink(link, source);
target = getTargetFile(dir2);
copyAndVerify(link, target, NOFOLLOW_LINKS);
delete(link);
delete(source);
}
/** Test: Copy link (to directory) */
if (supportsLinks) {
source = dir1.resolve("mydir");
createDirectory(source);
link = dir1.resolve("link");
createSymbolicLink(link, source);
target = getTargetFile(dir2);
copyAndVerify(link, target, NOFOLLOW_LINKS);
delete(link);
delete(source);
}
/** Test: Copy broken link */
if (supportsLinks) {
assertTrue(notExists(source));
link = dir1.resolve("link");
createSymbolicLink(link, source);
target = getTargetFile(dir2);
copyAndVerify(link, target, NOFOLLOW_LINKS);
delete(link);
}
/** Test: Copy link to UNC (Windows only) */
if (supportsLinks && System.getProperty("os.name").startsWith("Windows")) {
Path unc = Paths.get("\\\\rialto\\share\\file");
link = dir1.resolve("link");
createSymbolicLink(link, unc);
target = getTargetFile(dir2);
copyAndVerify(link, target, NOFOLLOW_LINKS);
delete(link);
}
// -- misc. tests --
/** Test nulls */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
try {
copy(source, null);
throw new RuntimeException("NullPointerException expected");
} catch (NullPointerException x) {
}
try {
copy(source, target, (CopyOption[]) null);
throw new RuntimeException("NullPointerException expected");
} catch (NullPointerException x) {
}
try {
CopyOption[] opts = {REPLACE_EXISTING, null};
copy(source, target, opts);
throw new RuntimeException("NullPointerException expected");
} catch (NullPointerException x) {
}
delete(source);
/** Test UOE */
source = createSourceFile(dir1);
target = getTargetFile(dir2);
try {
copy(source, target, new CopyOption() {});
} catch (UnsupportedOperationException x) {
}
try {
copy(source, target, REPLACE_EXISTING, new CopyOption() {});
} catch (UnsupportedOperationException x) {
}
delete(source);
}
