private static Set<ShardId> findAllShardsForIndex(Path indexPath) throws IOException { Set<ShardId> shardIds = new HashSet<>(); if (Files.isDirectory(indexPath)) { try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexPath)) { String currentIndex = indexPath.getFileName().toString(); for (Path shardPath : stream) { if (Files.isDirectory(shardPath)) { Integer shardId = Ints.tryParse(shardPath.getFileName().toString()); if (shardId != null) { ShardId id = new ShardId(currentIndex, shardId); shardIds.add(id); } } } } } return shardIds; }
public Set<String> findAllIndices() throws IOException { if (nodePaths == null || locks == null) { throw new IllegalStateException("node is not configured to store local location"); } assert assertEnvIsLocked(); Set<String> indices = Sets.newHashSet(); for (NodePath nodePath : nodePaths) { Path indicesLocation = nodePath.indicesPath; if (Files.isDirectory(indicesLocation)) { try (DirectoryStream<Path> stream = Files.newDirectoryStream(indicesLocation)) { for (Path index : stream) { if (Files.isDirectory(index)) { indices.add(index.getFileName().toString()); } } } } } return indices; }
/** * we check whether we need to remove the top-level folder while extracting sometimes (e.g. * github) the downloaded archive contains a top-level folder which needs to be removed */ private Path findPluginRoot(Path dir) throws IOException { if (Files.exists(dir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES))) { return dir; } else { final Path[] topLevelFiles = FileSystemUtils.files(dir); if (topLevelFiles.length == 1 && Files.isDirectory(topLevelFiles[0])) { Path subdir = topLevelFiles[0]; if (Files.exists(subdir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES))) { return subdir; } } } throw new RuntimeException( "Could not find plugin descriptor '" + PluginInfo.ES_PLUGIN_PROPERTIES + "' in plugin zip"); }
private int getNumberOfItems(Path quellOrdner) { int retValue = 0; try { DirectoryStream<Path> qstream = Files.newDirectoryStream(quellOrdner); for (Path qfile : qstream) { if (Files.isDirectory(qfile)) { getNumberOfItems(Paths.get(quellOrdner.toString() + "/" + qfile.getFileName())); } i++; } qstream.close(); } catch (IOException e) { e.printStackTrace(); } retValue = i; return retValue; }
/** * This method tries to write an empty file and moves it using an atomic move operation. This * method throws an {@link IllegalStateException} if this operation is not supported by the * filesystem. This test is executed on each of the data directories. This method cleans up all * files even in the case of an error. */ public void ensureAtomicMoveSupported() throws IOException { final NodePath[] nodePaths = nodePaths(); for (NodePath nodePath : nodePaths) { assert Files.isDirectory(nodePath.path) : nodePath.path + " is not a directory"; final Path src = nodePath.path.resolve("__es__.tmp"); Files.createFile(src); final Path target = nodePath.path.resolve("__es__.final"); try { Files.move(src, target, StandardCopyOption.ATOMIC_MOVE); } catch (AtomicMoveNotSupportedException ex) { throw new IllegalStateException( "atomic_move is not supported by the filesystem on path [" + nodePath.path + "] atomic_move is required for elasticsearch to work correctly.", ex); } finally { Files.deleteIfExists(src); Files.deleteIfExists(target); } } }
/** Process all events for keys queued to the watcher */ private static void forwardToItopic(WatchEvent.Kind<Path> kind, Path dir) { boolean isDir = Files.isDirectory(dir); if (kind == ENTRY_CREATE) { if (!isDir) { Action act = new Action("add_file", Folder.getInternalPath(dir)); // Folder.getFileFromDiskToWinSafe(act.getPath()); Folder.loadFileFromFSToInternal(dir); topic.publish(act); } else { if (Folder.isEmptyFSFolder(dir)) { Folder.createEmptyFolderInInternal(dir); topic.publish(new Action("create_empty_folder", Folder.getInternalPath(dir))); } else { Folder.loadFolderFromFSToInternal(dir); topic.publish(new Action("create_folder", Folder.getInternalPath(dir))); } } } else if (kind == ENTRY_DELETE) { // todo Folder.deleteFromInternal(dir); topic.publish(new Action("delete_entry", Folder.getInternalPath(dir))); } else if (kind == ENTRY_MODIFY) { // todo if (!isDir) { Folder.loadFileFromFSToInternal(dir); topic.publish(new Action("edit_file", Folder.getInternalPath(dir))); } else { if (Folder.isEmptyFSFolder(dir)) { Folder.createEmptyFolderInInternal(dir); topic.publish(new Action("create_empty_folder", Folder.getInternalPath(dir))); } else { Folder.loadFolderFromFSToInternal(dir); topic.publish(new Action("edit_folder", Folder.getInternalPath(dir))); } } } else { // TODO System.out.println("[forwardToItopic] Unexpected Event - kind=" + kind + "dir=" + dir); } }
@Override public void parse() throws IOException { if (conservedRegionPath == null || !Files.exists(conservedRegionPath) || !Files.isDirectory(conservedRegionPath)) { throw new IOException( "Conservation directory whether does not exist, is not a directory or cannot be read"); } Map<String, Path> files = new HashMap<>(); String chromosome; Set<String> chromosomes = new HashSet<>(); // Reading all files in phastCons folder DirectoryStream<Path> directoryStream = Files.newDirectoryStream(conservedRegionPath.resolve("phastCons")); for (Path path : directoryStream) { chromosome = path.getFileName().toString().split("\\.")[0].replace("chr", ""); chromosomes.add(chromosome); files.put(chromosome + "phastCons", path); } // Reading all files in phylop folder directoryStream = Files.newDirectoryStream(conservedRegionPath.resolve("phylop")); for (Path path : directoryStream) { chromosome = path.getFileName().toString().split("\\.")[0].replace("chr", ""); chromosomes.add(chromosome); files.put(chromosome + "phylop", path); } /** Now we can iterate over all the chromosomes found and process the files */ logger.debug("Chromosomes found {}", chromosomes.toString()); for (String chr : chromosomes) { logger.debug("Processing chromosome {}, file {}", chr, files.get(chr + "phastCons")); processFile(files.get(chr + "phastCons"), "phastCons"); logger.debug("Processing chromosome {}, file {}", chr, files.get(chr + "phylop")); processFile(files.get(chr + "phylop"), "phylop"); } }
/** * Tries to find all allocated shards for the given index on the current node. NOTE: This methods * is prone to race-conditions on the filesystem layer since it might not see directories created * concurrently or while it's traversing. * * @param index the index to filter shards * @return a set of shard IDs * @throws IOException if an IOException occurs */ public Set<ShardId> findAllShardIds(final Index index) throws IOException { assert index != null; if (nodePaths == null || locks == null) { throw new IllegalStateException("node is not configured to store local location"); } assert assertEnvIsLocked(); final Set<ShardId> shardIds = Sets.newHashSet(); String indexName = index.name(); for (final NodePath nodePath : nodePaths) { Path location = nodePath.indicesPath; if (Files.isDirectory(location)) { try (DirectoryStream<Path> indexStream = Files.newDirectoryStream(location)) { for (Path indexPath : indexStream) { if (indexName.equals(indexPath.getFileName().toString())) { shardIds.addAll(findAllShardsForIndex(indexPath)); } } } } } return shardIds; }
@Override public void run() { logger.debug("Register root " + startDir.toString()); try { registerAll(startDir); } catch (IOException ex) { logger.error(ex.getMessage()); return; } if (isInterrupted()) return; VOSync.debug("Sync local db with drive"); DbPool.goSql( "Synching the local db with drive", "select NAME from FILES", new SqlWorker<Boolean>() { @Override public Boolean go(Connection conn, PreparedStatement stmt) throws SQLException { ResultSet resSet = stmt.executeQuery(); while (resSet.next()) { try { String fileName = resSet.getString(1); Path filePath = FileSystems.getDefault().getPath(startDir.toString(), fileName.substring(1)); if (!filePath.toFile().exists()) { logger.debug( "Deleting file " + fileName + " existing in DB and not present on disk"); api.delete(fileName); MetaHandler.delete(fileName); } } catch (DropboxException ex) { } } resSet.close(); return true; } }); if (isInterrupted()) return; VOSync.debug("Sync storage"); syncStorage(); logger.debug("Start watching"); while (!isInterrupted()) { WatchKey key; try { key = watcher.take(); } catch (InterruptedException x) { return; } Path dir = keys.get(key); if (dir == null) { System.err.println("WatchKey " + key.toString() + " not recognized!"); continue; } for (WatchEvent<?> event : key.pollEvents()) { Kind<?> kind = event.kind(); // TBD - provide example of how OVERFLOW event is handled if (kind == OVERFLOW) { continue; } // Context for directory entry event is the file name of entry WatchEvent<Path> ev = cast(event); Path name = ev.context(); Path child = dir.resolve(name); Path relativeDir = startDir.relativize(child); String fileRelPath = "/" + fixPath(relativeDir.toString()); // print out event logger.debug(event.kind().name() + ":" + child + " " + name + " " + key); try { if (Files.exists(child, new LinkOption[] {}) && Files.isHidden(child)) { logger.error( "Skipping hidden file " + child.getFileName()); // skip OS generated catalog files } else { if (event.kind() == ENTRY_CREATE) { if (Files.isRegularFile(child, NOFOLLOW_LINKS)) { // file modified uploadFile(fileRelPath, child); } else if (Files.isDirectory(child, NOFOLLOW_LINKS)) { // directory contents changed registerAll(child); } } else if (event.kind() == ENTRY_DELETE) { logger.debug("Deleting " + fileRelPath); api.delete(fileRelPath); MetaHandler.delete(fileRelPath); logger.debug("Deleted!"); } else if (event.kind() == ENTRY_MODIFY) { if (Files.isRegularFile(child, NOFOLLOW_LINKS)) { // file modified uploadFile(fileRelPath, child); } else if (Files.isDirectory(child, NOFOLLOW_LINKS)) { // directory contents changed // logger.debug("Renewing dir: "+relativeDir.toString()); // TODO update folder date // MetaHandler.setFile(fileRelPath, child, rev); } } } } catch (IOException ex) { ex.printStackTrace(); logger.error(ex.getMessage()); } catch (DropboxException ex) { ex.printStackTrace(); logger.error(ex.getMessage()); } } boolean valid = key.reset(); if (!valid) keys.remove(key); } }
private void copyBinDirectory( Path sourcePluginBinDirectory, Path destPluginBinDirectory, String pluginName, Terminal terminal) throws IOException { boolean canCopyFromSource = Files.exists(sourcePluginBinDirectory) && Files.isReadable(sourcePluginBinDirectory) && Files.isDirectory(sourcePluginBinDirectory); if (canCopyFromSource) { terminal.println(VERBOSE, "Found bin, moving to %s", destPluginBinDirectory.toAbsolutePath()); if (Files.exists(destPluginBinDirectory)) { IOUtils.rm(destPluginBinDirectory); } try { Files.createDirectories(destPluginBinDirectory.getParent()); FileSystemUtils.move(sourcePluginBinDirectory, destPluginBinDirectory); } catch (IOException e) { throw new IOException( "Could not move [" + sourcePluginBinDirectory + "] to [" + destPluginBinDirectory + "]", e); } if (Environment.getFileStore(destPluginBinDirectory) .supportsFileAttributeView(PosixFileAttributeView.class)) { final PosixFileAttributes parentDirAttributes = Files.getFileAttributeView( destPluginBinDirectory.getParent(), PosixFileAttributeView.class) .readAttributes(); // copy permissions from parent bin directory final Set<PosixFilePermission> filePermissions = new HashSet<>(); for (PosixFilePermission posixFilePermission : parentDirAttributes.permissions()) { switch (posixFilePermission) { case OWNER_EXECUTE: case GROUP_EXECUTE: case OTHERS_EXECUTE: break; default: filePermissions.add(posixFilePermission); } } // add file execute permissions to existing perms, so execution will work. filePermissions.add(PosixFilePermission.OWNER_EXECUTE); filePermissions.add(PosixFilePermission.GROUP_EXECUTE); filePermissions.add(PosixFilePermission.OTHERS_EXECUTE); Files.walkFileTree( destPluginBinDirectory, new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (attrs.isRegularFile()) { setPosixFileAttributes( file, parentDirAttributes.owner(), parentDirAttributes.group(), filePermissions); } return FileVisitResult.CONTINUE; } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { setPosixFileAttributes( dir, parentDirAttributes.owner(), parentDirAttributes.group(), parentDirAttributes.permissions()); return FileVisitResult.CONTINUE; } }); } else { terminal.println( VERBOSE, "Skipping posix permissions - filestore doesn't support posix permission"); } terminal.println( VERBOSE, "Installed %s into %s", pluginName, destPluginBinDirectory.toAbsolutePath()); } }
private void extract(PluginHandle pluginHandle, Terminal terminal, Path pluginFile) throws IOException { // unzip plugin to a staging temp dir, named for the plugin Path tmp = Files.createTempDirectory(environment.tmpFile(), null); Path root = tmp.resolve(pluginHandle.name); unzipPlugin(pluginFile, root); // find the actual root (in case its unzipped with extra directory wrapping) root = findPluginRoot(root); // read and validate the plugin descriptor PluginInfo info = PluginInfo.readFromProperties(root); terminal.println(VERBOSE, "%s", info); // update name in handle based on 'name' property found in descriptor file pluginHandle = new PluginHandle(info.getName(), pluginHandle.version, pluginHandle.user); final Path extractLocation = pluginHandle.extractedDir(environment); if (Files.exists(extractLocation)) { throw new IOException( "plugin directory " + extractLocation.toAbsolutePath() + " already exists. To update the plugin, uninstall it first using 'remove " + pluginHandle.name + "' command"); } // check for jar hell before any copying if (info.isJvm()) { jarHellCheck(root, info.isIsolated()); } // install plugin FileSystemUtils.copyDirectoryRecursively(root, extractLocation); terminal.println("Installed %s into %s", pluginHandle.name, extractLocation.toAbsolutePath()); // cleanup tryToDeletePath(terminal, tmp, pluginFile); // take care of bin/ by moving and applying permissions if needed Path sourcePluginBinDirectory = extractLocation.resolve("bin"); Path destPluginBinDirectory = pluginHandle.binDir(environment); boolean needToCopyBinDirectory = Files.exists(sourcePluginBinDirectory); if (needToCopyBinDirectory) { if (Files.exists(destPluginBinDirectory) && !Files.isDirectory(destPluginBinDirectory)) { tryToDeletePath(terminal, extractLocation); throw new IOException( "plugin bin directory " + destPluginBinDirectory + " is not a directory"); } try { copyBinDirectory( sourcePluginBinDirectory, destPluginBinDirectory, pluginHandle.name, terminal); } catch (IOException e) { // rollback and remove potentially before installed leftovers terminal.printError( "Error copying bin directory [%s] to [%s], cleaning up, reason: %s", sourcePluginBinDirectory, destPluginBinDirectory, ExceptionsHelper.detailedMessage(e)); tryToDeletePath(terminal, extractLocation, pluginHandle.binDir(environment)); throw e; } } Path sourceConfigDirectory = extractLocation.resolve("config"); Path destConfigDirectory = pluginHandle.configDir(environment); boolean needToCopyConfigDirectory = Files.exists(sourceConfigDirectory); if (needToCopyConfigDirectory) { if (Files.exists(destConfigDirectory) && !Files.isDirectory(destConfigDirectory)) { tryToDeletePath(terminal, extractLocation, destPluginBinDirectory); throw new IOException( "plugin config directory " + destConfigDirectory + " is not a directory"); } try { terminal.println( VERBOSE, "Found config, moving to %s", destConfigDirectory.toAbsolutePath()); moveFilesWithoutOverwriting(sourceConfigDirectory, destConfigDirectory, ".new"); if (Environment.getFileStore(destConfigDirectory) .supportsFileAttributeView(PosixFileAttributeView.class)) { // We copy owner, group and permissions from the parent ES_CONFIG directory, assuming they // were properly set depending // on how es was installed in the first place: can be root:elasticsearch (750) if es was // installed from rpm/deb packages // or most likely elasticsearch:elasticsearch if installed from tar/zip. As for // permissions we don't rely on umask. final PosixFileAttributes parentDirAttributes = Files.getFileAttributeView( destConfigDirectory.getParent(), PosixFileAttributeView.class) .readAttributes(); // for files though, we make sure not to copy execute permissions from the parent dir and // leave them untouched final Set<PosixFilePermission> baseFilePermissions = new HashSet<>(); for (PosixFilePermission posixFilePermission : parentDirAttributes.permissions()) { switch (posixFilePermission) { case OWNER_EXECUTE: case GROUP_EXECUTE: case OTHERS_EXECUTE: break; default: baseFilePermissions.add(posixFilePermission); } } Files.walkFileTree( destConfigDirectory, new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (attrs.isRegularFile()) { Set<PosixFilePermission> newFilePermissions = new HashSet<>(baseFilePermissions); Set<PosixFilePermission> currentFilePermissions = Files.getPosixFilePermissions(file); for (PosixFilePermission posixFilePermission : currentFilePermissions) { switch (posixFilePermission) { case OWNER_EXECUTE: case GROUP_EXECUTE: case OTHERS_EXECUTE: newFilePermissions.add(posixFilePermission); } } setPosixFileAttributes( file, parentDirAttributes.owner(), parentDirAttributes.group(), newFilePermissions); } return FileVisitResult.CONTINUE; } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { setPosixFileAttributes( dir, parentDirAttributes.owner(), parentDirAttributes.group(), parentDirAttributes.permissions()); return FileVisitResult.CONTINUE; } }); } else { terminal.println( VERBOSE, "Skipping posix permissions - filestore doesn't support posix permission"); } terminal.println( VERBOSE, "Installed %s into %s", pluginHandle.name, destConfigDirectory.toAbsolutePath()); } catch (IOException e) { terminal.printError( "Error copying config directory [%s] to [%s], cleaning up, reason: %s", sourceConfigDirectory, destConfigDirectory, ExceptionsHelper.detailedMessage(e)); tryToDeletePath(terminal, extractLocation, destPluginBinDirectory, destConfigDirectory); throw e; } } }
// void processEvents() { public void run() { System.out.println("WatchDir Thread INFO: priority=" + Thread.currentThread().getPriority()); for (; ; ) { // wait for key to be signalled System.out.println("WatchDir INFO: restarting loop...acquiring new key"); WatchKey key; try { key = watcher.take(); } catch (InterruptedException x) { return; } Path dir = keys.get(key); if (dir == null) { System.err.println("WatchKey not recognized!!"); continue; } for (WatchEvent<?> event : key.pollEvents()) { WatchEvent.Kind kind = event.kind(); // TBD - provide example of how OVERFLOW event is handled if (kind == OVERFLOW) { System.out.println("Encountered OVERFLOW Event - " + event); continue; } // Context for directory entry event is the file name of entry WatchEvent<Path> ev = cast(event); Path name = ev.context(); Path child = dir.resolve(name); // print out event System.out.format("[WatchDir] %s: %s\n", event.kind().name(), child); // if directory is created, and watching recursively, then // register it and its sub-directories if (recursive && (kind == ENTRY_CREATE)) { try { if (Files.isDirectory(child, NOFOLLOW_LINKS)) { registerAll(child); } } catch (IOException x) { // ignore to keep sample readbale } } long t = System.currentTimeMillis(); if (!Folder.dontWatch.contains(Folder.getInternalPath(child))) { Thread.currentThread().setPriority(Thread.NORM_PRIORITY); System.out.println( "WatchDir#" + key + " INFO: path=" + child + ", internal=" + Folder.getInternalPath(child) + " is NOT in don't watch list. Forwarding it to other peers. @" + Main.timeToString(t)); // DEBUG forwardToItopic(kind, child); } else { Thread.currentThread().setPriority(Thread.MIN_PRIORITY); System.out.println( "WatchDir#" + key + " INFO: path=" + child + ", internal=" + Folder.getInternalPath(child) + " IS in the don't watch list. NOT forwarding. @" + Main.timeToString(t)); // DEBUG // try{ // Thread.sleep(minDelayBtwnWatchEvents); // } catch(InterruptedException ex) { // System.err.println("Exception:"+ex+" while trying to sleep WatchDir thread"); // ex.printStackTrace(); // } } } // reset key and remove from set if directory no longer accessible boolean valid = key.reset(); if (!valid) { keys.remove(key); // all directories are inaccessible if (keys.isEmpty()) { break; } } } }
/** Process all events for keys queued to the watcher */ void processEvents() { for (; ; ) { // wait for key to be signalled WatchKey key; try { key = watcher.take(); } catch (InterruptedException x) { return; } Path dir = keys.get(key); if (dir == null) { System.err.println("WatchKey not recognized!!"); continue; } for (WatchEvent<?> event : key.pollEvents()) { WatchEvent.Kind kind = event.kind(); // TBD - provide example of how OVERFLOW event is handled if (kind == OVERFLOW) { continue; } // Context for directory entry event is the file name of entry WatchEvent<Path> ev = cast(event); Path name = ev.context(); Path child = dir.resolve(name); // print out event System.out.format("%s: %s\n", event.kind().name(), child); // Printing manager if (event.kind().name().equals("ENTRY_MODIFY") && (child.endsWith("baseFerremundoPointer.csv") || child.endsWith("baseFerremundoPointer2.csv"))) { File file = new File(child.toString()); file.renameTo(new File("/home/dios/FERREMUNDO/BD/baseFerremundoPointer_.csv")); InvoicePrintingManager manager = new InvoicePrintingManager(); manager.manage(); } // if directory is created, and watching recursively, then // register it and its sub-directories if (recursive && (kind == ENTRY_CREATE)) { try { if (Files.isDirectory(child, NOFOLLOW_LINKS)) { registerAll(child); } } catch (IOException x) { // ignore to keep sample readbale } } } // reset key and remove from set if directory no longer accessible boolean valid = key.reset(); if (!valid) { keys.remove(key); // all directories are inaccessible if (keys.isEmpty()) { break; } } } }
public void cnpStart(Path quellOrdner, Path zielOrdner) { try { DirectoryStream<Path> qstream = Files.newDirectoryStream(quellOrdner); for (Path qfile : qstream) { Path target = Paths.get(zielOrdner.toString() + "/" + qfile.getFileName()); if (abbruch) break; if (Files.isDirectory(qfile) && !Files.exists(target)) { Files.createDirectory(target); textArea.append("Verzeichnis: " + qfile + " wurde erstellt" + System.lineSeparator()); cnpStart( Paths.get(quellOrdner.toString() + "/" + qfile.getFileName()), Paths.get(zielOrdner.toString() + "/" + qfile.getFileName())); } else if (Files.isDirectory(qfile) && Files.exists(target)) { textArea.append("Wechsle in Verzeichnis: " + qfile + System.lineSeparator()); cnpStart( Paths.get(quellOrdner.toString() + "/" + qfile.getFileName()), Paths.get(zielOrdner.toString() + "/" + qfile.getFileName())); } // Wenn die Datei noch nicht existiert else if (!Files.exists(target)) { textArea.append( "Datei " + target.toString() + " wurde erstellt" + System.lineSeparator()); Files.copy(qfile, target, StandardCopyOption.REPLACE_EXISTING); } // Wenn Datei im Zielverzeichnis schon existiert else if (Files.exists(target)) { if (cAUeSchr) { textArea.append( "Datei " + target.toString() + " wird absolut überschrieben" + System.lineSeparator()); Files.copy(qfile, target, StandardCopyOption.REPLACE_EXISTING); } else if (cUeSchr) { if (checkAlter( Paths.get(quellOrdner.toString() + "/" + qfile.getFileName()), Paths.get(zielOrdner.toString() + "/" + qfile.getFileName()))) { textArea.append( target.toString() + " wird mit neuer Datei überschrieben" + System.lineSeparator()); Files.copy(qfile, target, StandardCopyOption.REPLACE_EXISTING); } else { textArea.append( target.toString() + " alte Datei bleibt bestehen" + System.lineSeparator()); } } else textArea.append( target.toString() + " alte Datei bleibt bestehen" + System.lineSeparator()); } pbCounter++; progressBar.setValue(pbCounter); } qstream.close(); } catch (IOException e) { e.printStackTrace(); } }