Beispiel #1
0
  private void loadAllProjectFlows(Project project) {
    try {
      List<Flow> flows = projectLoader.fetchAllProjectFlows(project);
      Map<String, Flow> flowMap = new HashMap<String, Flow>();
      for (Flow flow : flows) {
        flowMap.put(flow.getId(), flow);
      }

      project.setFlows(flowMap);
    } catch (ProjectManagerException e) {
      throw new RuntimeException("Could not load projects flows from store.", e);
    }
  }
  private void prepareProject(Project project, File directory)
      throws ProjectManagerException, IOException {
    DirectoryFlowLoader loader = new DirectoryFlowLoader(new Props(), logger);
    loader.loadProjectFlow(project, directory);
    if (!loader.getErrors().isEmpty()) {
      for (String error : loader.getErrors()) {
        System.out.println(error);
      }

      throw new RuntimeException("Errors found in setup");
    }

    flowMap = loader.getFlowMap();
    project.setFlows(flowMap);
    FileUtils.copyDirectory(directory, workingDir);
  }
Beispiel #3
0
  public void uploadProject(Project project, File archive, String fileType, User uploader)
      throws ProjectManagerException {
    logger.info("Uploading files to " + project.getName());

    // Unzip.
    File file = null;
    try {
      if (fileType == null) {
        throw new ProjectManagerException("Unknown file type for " + archive.getName());
      } else if ("zip".equals(fileType)) {
        file = unzipFile(archive);
      } else {
        throw new ProjectManagerException("Unsupported archive type for file " + archive.getName());
      }
    } catch (IOException e) {
      throw new ProjectManagerException("Error unzipping file.", e);
    }

    logger.info("Validating Flow for upload " + archive.getName());
    DirectoryFlowLoader loader = new DirectoryFlowLoader(logger);
    loader.loadProjectFlow(file);
    if (!loader.getErrors().isEmpty()) {
      logger.error("Error found in upload to " + project.getName() + ". Cleaning up.");

      try {
        FileUtils.deleteDirectory(file);
      } catch (IOException e) {
        file.deleteOnExit();
        e.printStackTrace();
      }

      StringBuffer errorMessage = new StringBuffer();
      errorMessage.append("Error found in upload. Cannot upload.\n");
      for (String error : loader.getErrors()) {
        errorMessage.append(error);
        errorMessage.append('\n');
      }

      throw new ProjectManagerException(errorMessage.toString());
    }

    Map<String, Props> jobProps = loader.getJobProps();
    List<Props> propProps = loader.getProps();

    synchronized (project) {
      int newVersion = projectLoader.getLatestProjectVersion(project) + 1;
      Map<String, Flow> flows = loader.getFlowMap();
      for (Flow flow : flows.values()) {
        flow.setProjectId(project.getId());
        flow.setVersion(newVersion);
      }

      logger.info("Uploading file to db " + archive.getName());
      projectLoader.uploadProjectFile(
          project, newVersion, fileType, archive.getName(), archive, uploader.getUserId());
      logger.info("Uploading flow to db " + archive.getName());
      projectLoader.uploadFlows(project, newVersion, flows.values());
      logger.info("Changing project versions " + archive.getName());
      projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId());
      project.setFlows(flows);
      logger.info("Uploading Job properties");
      projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values()));
      logger.info("Uploading Props properties");
      projectLoader.uploadProjectProperties(project, propProps);
    }

    logger.info("Uploaded project files. Cleaning up temp files.");
    projectLoader.postEvent(
        project,
        EventType.UPLOADED,
        uploader.getUserId(),
        "Uploaded project files zip " + archive.getName());
    try {
      FileUtils.deleteDirectory(file);
    } catch (IOException e) {
      file.deleteOnExit();
      e.printStackTrace();
    }

    logger.info(
        "Cleaning up old install files older than "
            + (project.getVersion() - projectVersionRetention));
    projectLoader.cleanOlderProjectVersion(
        project.getId(), project.getVersion() - projectVersionRetention);
  }
Beispiel #4
0
  public Map<String, ValidationReport> uploadProject(
      Project project, File archive, String fileType, User uploader, Props additionalProps)
      throws ProjectManagerException {
    logger.info("Uploading files to " + project.getName());

    // Unzip.
    File file = null;
    try {
      if (fileType == null) {
        throw new ProjectManagerException("Unknown file type for " + archive.getName());
      } else if ("zip".equals(fileType)) {
        file = unzipFile(archive);
      } else {
        throw new ProjectManagerException("Unsupported archive type for file " + archive.getName());
      }
    } catch (IOException e) {
      throw new ProjectManagerException("Error unzipping file.", e);
    }

    // Since props is an instance variable of ProjectManager, and each
    // invocation to the uploadProject manager needs to pass a different
    // value for the PROJECT_ARCHIVE_FILE_PATH key, it is necessary to
    // create a new instance of Props to make sure these different values
    // are isolated from each other.
    Props prop = new Props(props);
    prop.putAll(additionalProps);
    prop.put(ValidatorConfigs.PROJECT_ARCHIVE_FILE_PATH, archive.getAbsolutePath());
    // Basically, we want to make sure that for different invocations to the
    // uploadProject method,
    // the validators are using different values for the
    // PROJECT_ARCHIVE_FILE_PATH configuration key.
    // In addition, we want to reload the validator objects for each upload, so
    // that we can change the validator configuration files without having to
    // restart Azkaban web server. If the XmlValidatorManager is an instance
    // variable, 2 consecutive invocations to the uploadProject
    // method might cause the second one to overwrite the
    // PROJECT_ARCHIVE_FILE_PATH configuration parameter
    // of the first, thus causing a wrong archive file path to be passed to the
    // validators. Creating a separate XmlValidatorManager object for each
    // upload will prevent this issue without having to add
    // synchronization between uploads. Since we're already reloading the XML
    // config file and creating validator objects for each upload, this does
    // not add too much additional overhead.
    ValidatorManager validatorManager = new XmlValidatorManager(prop);
    logger.info(
        "Validating project "
            + archive.getName()
            + " using the registered validators "
            + validatorManager.getValidatorsInfo().toString());
    Map<String, ValidationReport> reports = validatorManager.validate(project, file);
    ValidationStatus status = ValidationStatus.PASS;
    for (Entry<String, ValidationReport> report : reports.entrySet()) {
      if (report.getValue().getStatus().compareTo(status) > 0) {
        status = report.getValue().getStatus();
      }
    }
    if (status == ValidationStatus.ERROR) {
      logger.error("Error found in upload to " + project.getName() + ". Cleaning up.");

      try {
        FileUtils.deleteDirectory(file);
      } catch (IOException e) {
        file.deleteOnExit();
        e.printStackTrace();
      }

      return reports;
    }

    DirectoryFlowLoader loader = (DirectoryFlowLoader) validatorManager.getDefaultValidator();
    Map<String, Props> jobProps = loader.getJobProps();
    List<Props> propProps = loader.getProps();

    synchronized (project) {
      int newVersion = projectLoader.getLatestProjectVersion(project) + 1;
      Map<String, Flow> flows = loader.getFlowMap();
      for (Flow flow : flows.values()) {
        flow.setProjectId(project.getId());
        flow.setVersion(newVersion);
      }

      logger.info("Uploading file to db " + archive.getName());
      projectLoader.uploadProjectFile(
          project, newVersion, fileType, archive.getName(), archive, uploader.getUserId());
      logger.info("Uploading flow to db " + archive.getName());
      projectLoader.uploadFlows(project, newVersion, flows.values());
      logger.info("Changing project versions " + archive.getName());
      projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId());
      project.setFlows(flows);
      logger.info("Uploading Job properties");
      projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values()));
      logger.info("Uploading Props properties");
      projectLoader.uploadProjectProperties(project, propProps);
    }

    logger.info("Uploaded project files. Cleaning up temp files.");
    projectLoader.postEvent(
        project,
        EventType.UPLOADED,
        uploader.getUserId(),
        "Uploaded project files zip " + archive.getName());
    try {
      FileUtils.deleteDirectory(file);
    } catch (IOException e) {
      file.deleteOnExit();
      e.printStackTrace();
    }

    logger.info(
        "Cleaning up old install files older than "
            + (project.getVersion() - projectVersionRetention));
    projectLoader.cleanOlderProjectVersion(
        project.getId(), project.getVersion() - projectVersionRetention);

    return reports;
  }