private void loadAllProjectFlows(Project project) {
try {
List<Flow> flows = projectLoader.fetchAllProjectFlows(project);
Map<String, Flow> flowMap = new HashMap<String, Flow>();
for (Flow flow : flows) {
flowMap.put(flow.getId(), flow);
}
project.setFlows(flowMap);
} catch (ProjectManagerException e) {
throw new RuntimeException("Could not load projects flows from store.", e);
}
}
private void prepareProject(Project project, File directory)
throws ProjectManagerException, IOException {
DirectoryFlowLoader loader = new DirectoryFlowLoader(new Props(), logger);
loader.loadProjectFlow(project, directory);
if (!loader.getErrors().isEmpty()) {
for (String error : loader.getErrors()) {
System.out.println(error);
}
throw new RuntimeException("Errors found in setup");
}
flowMap = loader.getFlowMap();
project.setFlows(flowMap);
FileUtils.copyDirectory(directory, workingDir);
}
public void uploadProject(Project project, File archive, String fileType, User uploader)
throws ProjectManagerException {
logger.info("Uploading files to " + project.getName());
// Unzip.
File file = null;
try {
if (fileType == null) {
throw new ProjectManagerException("Unknown file type for " + archive.getName());
} else if ("zip".equals(fileType)) {
file = unzipFile(archive);
} else {
throw new ProjectManagerException("Unsupported archive type for file " + archive.getName());
}
} catch (IOException e) {
throw new ProjectManagerException("Error unzipping file.", e);
}
logger.info("Validating Flow for upload " + archive.getName());
DirectoryFlowLoader loader = new DirectoryFlowLoader(logger);
loader.loadProjectFlow(file);
if (!loader.getErrors().isEmpty()) {
logger.error("Error found in upload to " + project.getName() + ". Cleaning up.");
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
StringBuffer errorMessage = new StringBuffer();
errorMessage.append("Error found in upload. Cannot upload.\n");
for (String error : loader.getErrors()) {
errorMessage.append(error);
errorMessage.append('\n');
}
throw new ProjectManagerException(errorMessage.toString());
}
Map<String, Props> jobProps = loader.getJobProps();
List<Props> propProps = loader.getProps();
synchronized (project) {
int newVersion = projectLoader.getLatestProjectVersion(project) + 1;
Map<String, Flow> flows = loader.getFlowMap();
for (Flow flow : flows.values()) {
flow.setProjectId(project.getId());
flow.setVersion(newVersion);
}
logger.info("Uploading file to db " + archive.getName());
projectLoader.uploadProjectFile(
project, newVersion, fileType, archive.getName(), archive, uploader.getUserId());
logger.info("Uploading flow to db " + archive.getName());
projectLoader.uploadFlows(project, newVersion, flows.values());
logger.info("Changing project versions " + archive.getName());
projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId());
project.setFlows(flows);
logger.info("Uploading Job properties");
projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values()));
logger.info("Uploading Props properties");
projectLoader.uploadProjectProperties(project, propProps);
}
logger.info("Uploaded project files. Cleaning up temp files.");
projectLoader.postEvent(
project,
EventType.UPLOADED,
uploader.getUserId(),
"Uploaded project files zip " + archive.getName());
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
logger.info(
"Cleaning up old install files older than "
+ (project.getVersion() - projectVersionRetention));
projectLoader.cleanOlderProjectVersion(
project.getId(), project.getVersion() - projectVersionRetention);
}
public Map<String, ValidationReport> uploadProject(
Project project, File archive, String fileType, User uploader, Props additionalProps)
throws ProjectManagerException {
logger.info("Uploading files to " + project.getName());
// Unzip.
File file = null;
try {
if (fileType == null) {
throw new ProjectManagerException("Unknown file type for " + archive.getName());
} else if ("zip".equals(fileType)) {
file = unzipFile(archive);
} else {
throw new ProjectManagerException("Unsupported archive type for file " + archive.getName());
}
} catch (IOException e) {
throw new ProjectManagerException("Error unzipping file.", e);
}
// Since props is an instance variable of ProjectManager, and each
// invocation to the uploadProject manager needs to pass a different
// value for the PROJECT_ARCHIVE_FILE_PATH key, it is necessary to
// create a new instance of Props to make sure these different values
// are isolated from each other.
Props prop = new Props(props);
prop.putAll(additionalProps);
prop.put(ValidatorConfigs.PROJECT_ARCHIVE_FILE_PATH, archive.getAbsolutePath());
// Basically, we want to make sure that for different invocations to the
// uploadProject method,
// the validators are using different values for the
// PROJECT_ARCHIVE_FILE_PATH configuration key.
// In addition, we want to reload the validator objects for each upload, so
// that we can change the validator configuration files without having to
// restart Azkaban web server. If the XmlValidatorManager is an instance
// variable, 2 consecutive invocations to the uploadProject
// method might cause the second one to overwrite the
// PROJECT_ARCHIVE_FILE_PATH configuration parameter
// of the first, thus causing a wrong archive file path to be passed to the
// validators. Creating a separate XmlValidatorManager object for each
// upload will prevent this issue without having to add
// synchronization between uploads. Since we're already reloading the XML
// config file and creating validator objects for each upload, this does
// not add too much additional overhead.
ValidatorManager validatorManager = new XmlValidatorManager(prop);
logger.info(
"Validating project "
+ archive.getName()
+ " using the registered validators "
+ validatorManager.getValidatorsInfo().toString());
Map<String, ValidationReport> reports = validatorManager.validate(project, file);
ValidationStatus status = ValidationStatus.PASS;
for (Entry<String, ValidationReport> report : reports.entrySet()) {
if (report.getValue().getStatus().compareTo(status) > 0) {
status = report.getValue().getStatus();
}
}
if (status == ValidationStatus.ERROR) {
logger.error("Error found in upload to " + project.getName() + ". Cleaning up.");
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
return reports;
}
DirectoryFlowLoader loader = (DirectoryFlowLoader) validatorManager.getDefaultValidator();
Map<String, Props> jobProps = loader.getJobProps();
List<Props> propProps = loader.getProps();
synchronized (project) {
int newVersion = projectLoader.getLatestProjectVersion(project) + 1;
Map<String, Flow> flows = loader.getFlowMap();
for (Flow flow : flows.values()) {
flow.setProjectId(project.getId());
flow.setVersion(newVersion);
}
logger.info("Uploading file to db " + archive.getName());
projectLoader.uploadProjectFile(
project, newVersion, fileType, archive.getName(), archive, uploader.getUserId());
logger.info("Uploading flow to db " + archive.getName());
projectLoader.uploadFlows(project, newVersion, flows.values());
logger.info("Changing project versions " + archive.getName());
projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId());
project.setFlows(flows);
logger.info("Uploading Job properties");
projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values()));
logger.info("Uploading Props properties");
projectLoader.uploadProjectProperties(project, propProps);
}
logger.info("Uploaded project files. Cleaning up temp files.");
projectLoader.postEvent(
project,
EventType.UPLOADED,
uploader.getUserId(),
"Uploaded project files zip " + archive.getName());
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
logger.info(
"Cleaning up old install files older than "
+ (project.getVersion() - projectVersionRetention));
projectLoader.cleanOlderProjectVersion(
project.getId(), project.getVersion() - projectVersionRetention);
return reports;
}
