Example #1
0
 public static void load(Collection<FileDesc> files, Path root, int blocSize, Pattern pattern)
     throws IOException {
   root = root.toAbsolutePath().normalize();
   Visitor visitor = new Visitor(root, blocSize, pattern);
   Files.walkFileTree(root, visitor);
   for (Future<FileDesc> future : visitor.futures()) {
     try {
       files.add(future.get());
     } catch (Exception e) {
       log.error("", e);
     }
   }
 }
Example #2
0
  /**
   * Compile takes an absolute path of the app and returns an absolute path of the resulting
   * Vertabrae.
   *
   * @param abs_path_from the absolute path where the resources of an application lives.
   * @param abs_path_to the absolute path to where the application should be built to and served
   *     from.
   * @return the absolute path to serve the application from
   */
  public static String compile(Path abs_path_from, Path abs_path_to) {
    System.out.println("[SPINE] Template path: " + abs_path_from);
    velocity_engine.setProperty(
        RuntimeConstants.FILE_RESOURCE_LOADER_PATH, abs_path_from.resolve("templates").toString());
    velocity_engine.init();

    System.out.println("[SPINE] From path: " + abs_path_from);
    System.out.println("[SPINE] To path: " + abs_path_to);

    try {
      Path dir = abs_path_from.resolve("content");

      Map<String, Vertabrae> file_names = extract_files(dir);

      generate_markup(abs_path_from, abs_path_to, file_names);
      static_files_copy(abs_path_from, abs_path_to);

    } catch (IOException e) {
      e.printStackTrace();
    }

    return abs_path_to.toAbsolutePath().toString();
  }
Example #3
0
  private void copyBinDirectory(
      Path sourcePluginBinDirectory,
      Path destPluginBinDirectory,
      String pluginName,
      Terminal terminal)
      throws IOException {
    boolean canCopyFromSource =
        Files.exists(sourcePluginBinDirectory)
            && Files.isReadable(sourcePluginBinDirectory)
            && Files.isDirectory(sourcePluginBinDirectory);
    if (canCopyFromSource) {
      terminal.println(VERBOSE, "Found bin, moving to %s", destPluginBinDirectory.toAbsolutePath());
      if (Files.exists(destPluginBinDirectory)) {
        IOUtils.rm(destPluginBinDirectory);
      }
      try {
        Files.createDirectories(destPluginBinDirectory.getParent());
        FileSystemUtils.move(sourcePluginBinDirectory, destPluginBinDirectory);
      } catch (IOException e) {
        throw new IOException(
            "Could not move [" + sourcePluginBinDirectory + "] to [" + destPluginBinDirectory + "]",
            e);
      }
      if (Environment.getFileStore(destPluginBinDirectory)
          .supportsFileAttributeView(PosixFileAttributeView.class)) {
        final PosixFileAttributes parentDirAttributes =
            Files.getFileAttributeView(
                    destPluginBinDirectory.getParent(), PosixFileAttributeView.class)
                .readAttributes();
        // copy permissions from parent bin directory
        final Set<PosixFilePermission> filePermissions = new HashSet<>();
        for (PosixFilePermission posixFilePermission : parentDirAttributes.permissions()) {
          switch (posixFilePermission) {
            case OWNER_EXECUTE:
            case GROUP_EXECUTE:
            case OTHERS_EXECUTE:
              break;
            default:
              filePermissions.add(posixFilePermission);
          }
        }
        // add file execute permissions to existing perms, so execution will work.
        filePermissions.add(PosixFilePermission.OWNER_EXECUTE);
        filePermissions.add(PosixFilePermission.GROUP_EXECUTE);
        filePermissions.add(PosixFilePermission.OTHERS_EXECUTE);
        Files.walkFileTree(
            destPluginBinDirectory,
            new SimpleFileVisitor<Path>() {
              @Override
              public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
                  throws IOException {
                if (attrs.isRegularFile()) {
                  setPosixFileAttributes(
                      file,
                      parentDirAttributes.owner(),
                      parentDirAttributes.group(),
                      filePermissions);
                }
                return FileVisitResult.CONTINUE;
              }

              @Override
              public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
                  throws IOException {
                setPosixFileAttributes(
                    dir,
                    parentDirAttributes.owner(),
                    parentDirAttributes.group(),
                    parentDirAttributes.permissions());
                return FileVisitResult.CONTINUE;
              }
            });
      } else {
        terminal.println(
            VERBOSE, "Skipping posix permissions - filestore doesn't support posix permission");
      }
      terminal.println(
          VERBOSE, "Installed %s into %s", pluginName, destPluginBinDirectory.toAbsolutePath());
    }
  }
Example #4
0
  private void extract(PluginHandle pluginHandle, Terminal terminal, Path pluginFile)
      throws IOException {
    // unzip plugin to a staging temp dir, named for the plugin
    Path tmp = Files.createTempDirectory(environment.tmpFile(), null);
    Path root = tmp.resolve(pluginHandle.name);
    unzipPlugin(pluginFile, root);

    // find the actual root (in case its unzipped with extra directory wrapping)
    root = findPluginRoot(root);

    // read and validate the plugin descriptor
    PluginInfo info = PluginInfo.readFromProperties(root);
    terminal.println(VERBOSE, "%s", info);

    // update name in handle based on 'name' property found in descriptor file
    pluginHandle = new PluginHandle(info.getName(), pluginHandle.version, pluginHandle.user);
    final Path extractLocation = pluginHandle.extractedDir(environment);
    if (Files.exists(extractLocation)) {
      throw new IOException(
          "plugin directory "
              + extractLocation.toAbsolutePath()
              + " already exists. To update the plugin, uninstall it first using 'remove "
              + pluginHandle.name
              + "' command");
    }

    // check for jar hell before any copying
    if (info.isJvm()) {
      jarHellCheck(root, info.isIsolated());
    }

    // install plugin
    FileSystemUtils.copyDirectoryRecursively(root, extractLocation);
    terminal.println("Installed %s into %s", pluginHandle.name, extractLocation.toAbsolutePath());

    // cleanup
    tryToDeletePath(terminal, tmp, pluginFile);

    // take care of bin/ by moving and applying permissions if needed
    Path sourcePluginBinDirectory = extractLocation.resolve("bin");
    Path destPluginBinDirectory = pluginHandle.binDir(environment);
    boolean needToCopyBinDirectory = Files.exists(sourcePluginBinDirectory);
    if (needToCopyBinDirectory) {
      if (Files.exists(destPluginBinDirectory) && !Files.isDirectory(destPluginBinDirectory)) {
        tryToDeletePath(terminal, extractLocation);
        throw new IOException(
            "plugin bin directory " + destPluginBinDirectory + " is not a directory");
      }

      try {
        copyBinDirectory(
            sourcePluginBinDirectory, destPluginBinDirectory, pluginHandle.name, terminal);
      } catch (IOException e) {
        // rollback and remove potentially before installed leftovers
        terminal.printError(
            "Error copying bin directory [%s] to [%s], cleaning up, reason: %s",
            sourcePluginBinDirectory, destPluginBinDirectory, ExceptionsHelper.detailedMessage(e));
        tryToDeletePath(terminal, extractLocation, pluginHandle.binDir(environment));
        throw e;
      }
    }

    Path sourceConfigDirectory = extractLocation.resolve("config");
    Path destConfigDirectory = pluginHandle.configDir(environment);
    boolean needToCopyConfigDirectory = Files.exists(sourceConfigDirectory);
    if (needToCopyConfigDirectory) {
      if (Files.exists(destConfigDirectory) && !Files.isDirectory(destConfigDirectory)) {
        tryToDeletePath(terminal, extractLocation, destPluginBinDirectory);
        throw new IOException(
            "plugin config directory " + destConfigDirectory + " is not a directory");
      }

      try {
        terminal.println(
            VERBOSE, "Found config, moving to %s", destConfigDirectory.toAbsolutePath());
        moveFilesWithoutOverwriting(sourceConfigDirectory, destConfigDirectory, ".new");

        if (Environment.getFileStore(destConfigDirectory)
            .supportsFileAttributeView(PosixFileAttributeView.class)) {
          // We copy owner, group and permissions from the parent ES_CONFIG directory, assuming they
          // were properly set depending
          // on how es was installed in the first place: can be root:elasticsearch (750) if es was
          // installed from rpm/deb packages
          // or most likely elasticsearch:elasticsearch if installed from tar/zip. As for
          // permissions we don't rely on umask.
          final PosixFileAttributes parentDirAttributes =
              Files.getFileAttributeView(
                      destConfigDirectory.getParent(), PosixFileAttributeView.class)
                  .readAttributes();
          // for files though, we make sure not to copy execute permissions from the parent dir and
          // leave them untouched
          final Set<PosixFilePermission> baseFilePermissions = new HashSet<>();
          for (PosixFilePermission posixFilePermission : parentDirAttributes.permissions()) {
            switch (posixFilePermission) {
              case OWNER_EXECUTE:
              case GROUP_EXECUTE:
              case OTHERS_EXECUTE:
                break;
              default:
                baseFilePermissions.add(posixFilePermission);
            }
          }
          Files.walkFileTree(
              destConfigDirectory,
              new SimpleFileVisitor<Path>() {
                @Override
                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
                    throws IOException {
                  if (attrs.isRegularFile()) {
                    Set<PosixFilePermission> newFilePermissions =
                        new HashSet<>(baseFilePermissions);
                    Set<PosixFilePermission> currentFilePermissions =
                        Files.getPosixFilePermissions(file);
                    for (PosixFilePermission posixFilePermission : currentFilePermissions) {
                      switch (posixFilePermission) {
                        case OWNER_EXECUTE:
                        case GROUP_EXECUTE:
                        case OTHERS_EXECUTE:
                          newFilePermissions.add(posixFilePermission);
                      }
                    }
                    setPosixFileAttributes(
                        file,
                        parentDirAttributes.owner(),
                        parentDirAttributes.group(),
                        newFilePermissions);
                  }
                  return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
                    throws IOException {
                  setPosixFileAttributes(
                      dir,
                      parentDirAttributes.owner(),
                      parentDirAttributes.group(),
                      parentDirAttributes.permissions());
                  return FileVisitResult.CONTINUE;
                }
              });
        } else {
          terminal.println(
              VERBOSE, "Skipping posix permissions - filestore doesn't support posix permission");
        }

        terminal.println(
            VERBOSE,
            "Installed %s into %s",
            pluginHandle.name,
            destConfigDirectory.toAbsolutePath());
      } catch (IOException e) {
        terminal.printError(
            "Error copying config directory [%s] to [%s], cleaning up, reason: %s",
            sourceConfigDirectory, destConfigDirectory, ExceptionsHelper.detailedMessage(e));
        tryToDeletePath(terminal, extractLocation, destPluginBinDirectory, destConfigDirectory);
        throw e;
      }
    }
  }
  @Inject
  @SuppressForbidden(reason = "System.out.*")
  public NodeEnvironment(Settings settings, Environment environment) throws IOException {
    super(settings);

    this.addNodeId = settings.getAsBoolean(ADD_NODE_ID_TO_CUSTOM_PATH, true);
    this.customPathsEnabled = settings.getAsBoolean(SETTING_CUSTOM_DATA_PATH_ENABLED, false);

    if (!DiscoveryNode.nodeRequiresLocalStorage(settings)) {
      nodePaths = null;
      locks = null;
      localNodeId = -1;
      return;
    }

    final NodePath[] nodePaths = new NodePath[environment.dataWithClusterFiles().length];
    final Lock[] locks = new Lock[nodePaths.length];

    int localNodeId = -1;
    IOException lastException = null;
    int maxLocalStorageNodes = settings.getAsInt("node.max_local_storage_nodes", 50);
    for (int possibleLockId = 0; possibleLockId < maxLocalStorageNodes; possibleLockId++) {
      for (int dirIndex = 0; dirIndex < environment.dataWithClusterFiles().length; dirIndex++) {
        Path dir =
            environment
                .dataWithClusterFiles()[dirIndex]
                .resolve(NODES_FOLDER)
                .resolve(Integer.toString(possibleLockId));
        Files.createDirectories(dir);

        try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
          logger.trace("obtaining node lock on {} ...", dir.toAbsolutePath());
          try {
            locks[dirIndex] = Lucene.acquireLock(luceneDir, NODE_LOCK_FILENAME, 0);
            nodePaths[dirIndex] = new NodePath(dir, environment);
            localNodeId = possibleLockId;
          } catch (LockObtainFailedException ex) {
            logger.trace("failed to obtain node lock on {}", dir.toAbsolutePath());
            // release all the ones that were obtained up until now
            releaseAndNullLocks(locks);
            break;
          }

        } catch (IOException e) {
          logger.trace("failed to obtain node lock on {}", e, dir.toAbsolutePath());
          lastException = new IOException("failed to obtain lock on " + dir.toAbsolutePath(), e);
          // release all the ones that were obtained up until now
          releaseAndNullLocks(locks);
          break;
        }
      }
      if (locks[0] != null) {
        // we found a lock, break
        break;
      }
    }

    if (locks[0] == null) {
      throw new IllegalStateException(
          "Failed to obtain node lock, is the following location writable?: "
              + Arrays.toString(environment.dataWithClusterFiles()),
          lastException);
    }

    this.localNodeId = localNodeId;
    this.locks = locks;
    this.nodePaths = nodePaths;

    if (logger.isDebugEnabled()) {
      logger.debug("using node location [{}], local_node_id [{}]", nodePaths, localNodeId);
    }

    maybeLogPathDetails();

    if (settings.getAsBoolean(SETTING_ENABLE_LUCENE_SEGMENT_INFOS_TRACE, false)) {
      SegmentInfos.setInfoStream(System.out);
    }
  }