private void loadAllProjectFlows(Project project) { try { List<Flow> flows = projectLoader.fetchAllProjectFlows(project); Map<String, Flow> flowMap = new HashMap<String, Flow>(); for (Flow flow : flows) { flowMap.put(flow.getId(), flow); } project.setFlows(flowMap); } catch (ProjectManagerException e) { throw new RuntimeException("Could not load projects flows from store.", e); } }
private void prepareProject(Project project, File directory) throws ProjectManagerException, IOException { DirectoryFlowLoader loader = new DirectoryFlowLoader(new Props(), logger); loader.loadProjectFlow(project, directory); if (!loader.getErrors().isEmpty()) { for (String error : loader.getErrors()) { System.out.println(error); } throw new RuntimeException("Errors found in setup"); } flowMap = loader.getFlowMap(); project.setFlows(flowMap); FileUtils.copyDirectory(directory, workingDir); }
public void uploadProject(Project project, File archive, String fileType, User uploader) throws ProjectManagerException { logger.info("Uploading files to " + project.getName()); // Unzip. File file = null; try { if (fileType == null) { throw new ProjectManagerException("Unknown file type for " + archive.getName()); } else if ("zip".equals(fileType)) { file = unzipFile(archive); } else { throw new ProjectManagerException("Unsupported archive type for file " + archive.getName()); } } catch (IOException e) { throw new ProjectManagerException("Error unzipping file.", e); } logger.info("Validating Flow for upload " + archive.getName()); DirectoryFlowLoader loader = new DirectoryFlowLoader(logger); loader.loadProjectFlow(file); if (!loader.getErrors().isEmpty()) { logger.error("Error found in upload to " + project.getName() + ". Cleaning up."); try { FileUtils.deleteDirectory(file); } catch (IOException e) { file.deleteOnExit(); e.printStackTrace(); } StringBuffer errorMessage = new StringBuffer(); errorMessage.append("Error found in upload. Cannot upload.\n"); for (String error : loader.getErrors()) { errorMessage.append(error); errorMessage.append('\n'); } throw new ProjectManagerException(errorMessage.toString()); } Map<String, Props> jobProps = loader.getJobProps(); List<Props> propProps = loader.getProps(); synchronized (project) { int newVersion = projectLoader.getLatestProjectVersion(project) + 1; Map<String, Flow> flows = loader.getFlowMap(); for (Flow flow : flows.values()) { flow.setProjectId(project.getId()); flow.setVersion(newVersion); } logger.info("Uploading file to db " + archive.getName()); projectLoader.uploadProjectFile( project, newVersion, fileType, archive.getName(), archive, uploader.getUserId()); logger.info("Uploading flow to db " + archive.getName()); projectLoader.uploadFlows(project, newVersion, flows.values()); logger.info("Changing project versions " + archive.getName()); projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId()); project.setFlows(flows); logger.info("Uploading Job properties"); projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values())); logger.info("Uploading Props properties"); projectLoader.uploadProjectProperties(project, propProps); } logger.info("Uploaded project files. Cleaning up temp files."); projectLoader.postEvent( project, EventType.UPLOADED, uploader.getUserId(), "Uploaded project files zip " + archive.getName()); try { FileUtils.deleteDirectory(file); } catch (IOException e) { file.deleteOnExit(); e.printStackTrace(); } logger.info( "Cleaning up old install files older than " + (project.getVersion() - projectVersionRetention)); projectLoader.cleanOlderProjectVersion( project.getId(), project.getVersion() - projectVersionRetention); }
public Map<String, ValidationReport> uploadProject( Project project, File archive, String fileType, User uploader, Props additionalProps) throws ProjectManagerException { logger.info("Uploading files to " + project.getName()); // Unzip. File file = null; try { if (fileType == null) { throw new ProjectManagerException("Unknown file type for " + archive.getName()); } else if ("zip".equals(fileType)) { file = unzipFile(archive); } else { throw new ProjectManagerException("Unsupported archive type for file " + archive.getName()); } } catch (IOException e) { throw new ProjectManagerException("Error unzipping file.", e); } // Since props is an instance variable of ProjectManager, and each // invocation to the uploadProject manager needs to pass a different // value for the PROJECT_ARCHIVE_FILE_PATH key, it is necessary to // create a new instance of Props to make sure these different values // are isolated from each other. Props prop = new Props(props); prop.putAll(additionalProps); prop.put(ValidatorConfigs.PROJECT_ARCHIVE_FILE_PATH, archive.getAbsolutePath()); // Basically, we want to make sure that for different invocations to the // uploadProject method, // the validators are using different values for the // PROJECT_ARCHIVE_FILE_PATH configuration key. // In addition, we want to reload the validator objects for each upload, so // that we can change the validator configuration files without having to // restart Azkaban web server. If the XmlValidatorManager is an instance // variable, 2 consecutive invocations to the uploadProject // method might cause the second one to overwrite the // PROJECT_ARCHIVE_FILE_PATH configuration parameter // of the first, thus causing a wrong archive file path to be passed to the // validators. Creating a separate XmlValidatorManager object for each // upload will prevent this issue without having to add // synchronization between uploads. Since we're already reloading the XML // config file and creating validator objects for each upload, this does // not add too much additional overhead. ValidatorManager validatorManager = new XmlValidatorManager(prop); logger.info( "Validating project " + archive.getName() + " using the registered validators " + validatorManager.getValidatorsInfo().toString()); Map<String, ValidationReport> reports = validatorManager.validate(project, file); ValidationStatus status = ValidationStatus.PASS; for (Entry<String, ValidationReport> report : reports.entrySet()) { if (report.getValue().getStatus().compareTo(status) > 0) { status = report.getValue().getStatus(); } } if (status == ValidationStatus.ERROR) { logger.error("Error found in upload to " + project.getName() + ". Cleaning up."); try { FileUtils.deleteDirectory(file); } catch (IOException e) { file.deleteOnExit(); e.printStackTrace(); } return reports; } DirectoryFlowLoader loader = (DirectoryFlowLoader) validatorManager.getDefaultValidator(); Map<String, Props> jobProps = loader.getJobProps(); List<Props> propProps = loader.getProps(); synchronized (project) { int newVersion = projectLoader.getLatestProjectVersion(project) + 1; Map<String, Flow> flows = loader.getFlowMap(); for (Flow flow : flows.values()) { flow.setProjectId(project.getId()); flow.setVersion(newVersion); } logger.info("Uploading file to db " + archive.getName()); projectLoader.uploadProjectFile( project, newVersion, fileType, archive.getName(), archive, uploader.getUserId()); logger.info("Uploading flow to db " + archive.getName()); projectLoader.uploadFlows(project, newVersion, flows.values()); logger.info("Changing project versions " + archive.getName()); projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId()); project.setFlows(flows); logger.info("Uploading Job properties"); projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values())); logger.info("Uploading Props properties"); projectLoader.uploadProjectProperties(project, propProps); } logger.info("Uploaded project files. Cleaning up temp files."); projectLoader.postEvent( project, EventType.UPLOADED, uploader.getUserId(), "Uploaded project files zip " + archive.getName()); try { FileUtils.deleteDirectory(file); } catch (IOException e) { file.deleteOnExit(); e.printStackTrace(); } logger.info( "Cleaning up old install files older than " + (project.getVersion() - projectVersionRetention)); projectLoader.cleanOlderProjectVersion( project.getId(), project.getVersion() - projectVersionRetention); return reports; }