public void updateProjectPermission(
Project project, String name, Permission perm, boolean group, User modifier)
throws ProjectManagerException {
logger.info(
"User "
+ modifier.getUserId()
+ " updating permissions for project "
+ project.getName()
+ " for "
+ name
+ " "
+ perm.toString());
projectLoader.updatePermission(project, name, perm, group);
if (group) {
projectLoader.postEvent(
project,
EventType.GROUP_PERMISSION,
modifier.getUserId(),
"Permission for group " + name + " set to " + perm.toString());
} else {
projectLoader.postEvent(
project,
EventType.USER_PERMISSION,
modifier.getUserId(),
"Permission for user " + name + " set to " + perm.toString());
}
}
private void updateProjectSettings(Connection connection, Project project, EncodingType encType)
throws ProjectManagerException {
QueryRunner runner = new QueryRunner();
final String UPDATE_PROJECT_SETTINGS =
"UPDATE projects SET enc_type=?, settings_blob=? WHERE id=?";
String json = JSONUtils.toJSON(project.toObject());
byte[] data = null;
try {
byte[] stringData = json.getBytes("UTF-8");
data = stringData;
if (encType == EncodingType.GZIP) {
data = GZIPUtils.gzipBytes(stringData);
}
logger.debug(
"NumChars: " + json.length() + " UTF-8:" + stringData.length + " Gzip:" + data.length);
} catch (IOException e) {
throw new ProjectManagerException("Failed to encode. ", e);
}
try {
runner.update(
connection, UPDATE_PROJECT_SETTINGS, encType.getNumVal(), data, project.getId());
connection.commit();
} catch (SQLException e) {
throw new ProjectManagerException(
"Error updating project " + project.getName() + " version " + project.getVersion(), e);
}
}
public synchronized Project removeProject(Project project, User deleter)
throws ProjectManagerException {
projectLoader.removeProject(project, deleter.getUserId());
projectLoader.postEvent(project, EventType.DELETED, deleter.getUserId(), null);
projectsByName.remove(project.getName());
projectsById.remove(project.getId());
return project;
}
@Override
public void updatePermission(Project project, String name, Permission perm, boolean isGroup)
throws ProjectManagerException {
QueryRunner runner = createQueryRunner();
if (this.allowsOnDuplicateKey()) {
long updateTime = System.currentTimeMillis();
final String INSERT_PROJECT_PERMISSION =
"INSERT INTO project_permissions (project_id, modified_time, name, permissions, isGroup) values (?,?,?,?,?)"
+ "ON DUPLICATE KEY UPDATE modified_time = VALUES(modified_time), permissions = VALUES(permissions)";
try {
runner.update(
INSERT_PROJECT_PERMISSION, project.getId(), updateTime, name, perm.toFlags(), isGroup);
} catch (SQLException e) {
logger.error(e);
throw new ProjectManagerException(
"Error updating project " + project.getName() + " permissions for " + name, e);
}
} else {
long updateTime = System.currentTimeMillis();
final String MERGE_PROJECT_PERMISSION =
"MERGE INTO project_permissions (project_id, modified_time, name, permissions, isGroup) KEY (project_id, name) values (?,?,?,?,?)";
try {
runner.update(
MERGE_PROJECT_PERMISSION, project.getId(), updateTime, name, perm.toFlags(), isGroup);
} catch (SQLException e) {
logger.error(e);
throw new ProjectManagerException(
"Error updating project " + project.getName() + " permissions for " + name, e);
}
}
if (isGroup) {
project.setGroupPermission(name, perm);
} else {
project.setUserPermission(name, perm);
}
}
@Override
public int getLatestProjectVersion(Project project) throws ProjectManagerException {
QueryRunner runner = createQueryRunner();
IntHander handler = new IntHander();
try {
return runner.query(IntHander.SELECT_LATEST_VERSION, handler, project.getId());
} catch (SQLException e) {
logger.error(e);
throw new ProjectManagerException(
"Error marking project " + project.getName() + " as inactive", e);
}
}
@Override
public ProjectFileHandler getUploadedFile(Project project, int version)
throws ProjectManagerException {
logger.info("Retrieving to " + project.getName() + " version:" + version);
Connection connection = getConnection();
ProjectFileHandler handler = null;
try {
handler = getUploadedFile(connection, project.getId(), version);
} finally {
DbUtils.closeQuietly(connection);
}
return handler;
}
@Override
public void removeProject(Project project, String user) throws ProjectManagerException {
QueryRunner runner = createQueryRunner();
long updateTime = System.currentTimeMillis();
final String UPDATE_INACTIVE_PROJECT =
"UPDATE projects SET active=false,modified_time=?,last_modified_by=? WHERE id=?";
try {
runner.update(UPDATE_INACTIVE_PROJECT, updateTime, user, project.getId());
} catch (SQLException e) {
logger.error(e);
throw new ProjectManagerException(
"Error marking project " + project.getName() + " as inactive", e);
}
}
public List<Project> getProjectsByRegex(String regexPattern) {
List<Project> allProjects = new ArrayList<Project>();
Pattern pattern;
try {
pattern = Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE);
} catch (PatternSyntaxException e) {
logger.error("Bad regex pattern " + regexPattern);
return allProjects;
}
for (Project project : getProjects()) {
if (pattern.matcher(project.getName()).find()) {
allProjects.add(project);
}
}
return allProjects;
}
private void loadAllProjects() {
List<Project> projects;
try {
projects = projectLoader.fetchAllActiveProjects();
} catch (ProjectManagerException e) {
throw new RuntimeException("Could not load projects from store.", e);
}
for (Project proj : projects) {
projectsByName.put(proj.getName(), proj);
projectsById.put(proj.getId(), proj);
}
for (Project proj : projects) {
loadAllProjectFlows(proj);
}
}
public void removeProjectProxyUser(Project project, String proxyName, User modifier)
throws ProjectManagerException {
logger.info(
"User "
+ modifier.getUserId()
+ " removing proxy user "
+ proxyName
+ " from project "
+ project.getName());
project.removeProxyUser(proxyName);
projectLoader.postEvent(
project,
EventType.PROXY_USER,
modifier.getUserId(),
"Proxy user " + proxyName + " has been removed form the project.");
updateProjectSetting(project);
}
public void addProjectProxyUser(Project project, String proxyName, User modifier)
throws ProjectManagerException {
logger.info(
"User "
+ modifier.getUserId()
+ " adding proxy user "
+ proxyName
+ " to project "
+ project.getName());
project.addProxyUser(proxyName);
projectLoader.postEvent(
project,
EventType.PROXY_USER,
modifier.getUserId(),
"Proxy user " + proxyName + " is added to project.");
updateProjectSetting(project);
}
@Override
public void updateDescription(Project project, String description, String user)
throws ProjectManagerException {
QueryRunner runner = createQueryRunner();
final String UPDATE_PROJECT_DESCRIPTION =
"UPDATE projects SET description=?,modified_time=?,last_modified_by=? WHERE id=?";
long updateTime = System.currentTimeMillis();
try {
runner.update(UPDATE_PROJECT_DESCRIPTION, description, updateTime, user, project.getId());
project.setDescription(description);
project.setLastModifiedTimestamp(updateTime);
project.setLastModifiedUser(user);
} catch (SQLException e) {
logger.error(e);
throw new ProjectManagerException(
"Error marking project " + project.getName() + " as inactive", e);
}
}
@Override
public void changeProjectVersion(Project project, int version, String user)
throws ProjectManagerException {
long timestamp = System.currentTimeMillis();
QueryRunner runner = createQueryRunner();
try {
final String UPDATE_PROJECT_VERSION =
"UPDATE projects SET version=?,modified_time=?,last_modified_by=? WHERE id=?";
runner.update(UPDATE_PROJECT_VERSION, version, timestamp, user, project.getId());
project.setVersion(version);
project.setLastModifiedTimestamp(timestamp);
project.setLastModifiedUser(user);
} catch (SQLException e) {
logger.error(e);
throw new ProjectManagerException(
"Error updating switching project version " + project.getName(), e);
}
}
private List<Triple<String, Boolean, Permission>> fetchPermissionsForProject(
Connection connection, Project project) throws ProjectManagerException {
ProjectPermissionsResultHandler permHander = new ProjectPermissionsResultHandler();
QueryRunner runner = new QueryRunner();
List<Triple<String, Boolean, Permission>> permissions = null;
try {
permissions =
runner.query(
connection,
ProjectPermissionsResultHandler.SELECT_PROJECT_PERMISSION,
permHander,
project.getId());
} catch (SQLException e) {
throw new ProjectManagerException(
"Query for permissions for " + project.getName() + " failed.", e);
}
return permissions;
}
public Project createProject(String projectName, String description, User creator)
throws ProjectManagerException {
if (projectName == null || projectName.trim().isEmpty()) {
throw new ProjectManagerException("Project name cannot be empty.");
} else if (description == null || description.trim().isEmpty()) {
throw new ProjectManagerException("Description cannot be empty.");
} else if (creator == null) {
throw new ProjectManagerException("Valid creator user must be set.");
} else if (!projectName.matches("[a-zA-Z][a-zA-Z_0-9|-]*")) {
throw new ProjectManagerException(
"Project names must start with a letter, followed by any number of letters, digits, '-' or '_'.");
}
if (projectsByName.contains(projectName)) {
throw new ProjectManagerException("Project already exists.");
}
logger.info("Trying to create " + projectName + " by user " + creator.getUserId());
Project newProject = projectLoader.createNewProject(projectName, description, creator);
projectsByName.put(newProject.getName(), newProject);
projectsById.put(newProject.getId(), newProject);
if (creatorDefaultPermissions) {
// Add permission to project
projectLoader.updatePermission(
newProject, creator.getUserId(), new Permission(Permission.Type.ADMIN), false);
// Add proxy user
newProject.getProxyUsers().add(creator.getUserId());
try {
updateProjectSetting(newProject);
} catch (ProjectManagerException e) {
e.printStackTrace();
throw e;
}
}
projectLoader.postEvent(newProject, EventType.CREATED, creator.getUserId(), null);
return newProject;
}
@Override
public void removePermission(Project project, String name, boolean isGroup)
throws ProjectManagerException {
QueryRunner runner = createQueryRunner();
final String DELETE_PROJECT_PERMISSION =
"DELETE FROM project_permissions WHERE project_id=? AND name=? AND isGroup=?";
try {
runner.update(DELETE_PROJECT_PERMISSION, project.getId(), name, isGroup);
} catch (SQLException e) {
logger.error(e);
throw new ProjectManagerException(
"Error deleting project " + project.getName() + " permissions for " + name, e);
}
if (isGroup) {
project.removeGroupPermission(name);
} else {
project.removeUserPermission(name);
}
}
public List<Project> getUserProjectsByRegex(User user, String regexPattern) {
List<Project> array = new ArrayList<Project>();
Pattern pattern;
try {
pattern = Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE);
} catch (PatternSyntaxException e) {
logger.error("Bad regex pattern " + regexPattern);
return array;
}
for (Project project : projectsById.values()) {
Permission perm = project.getUserPermission(user);
if (perm != null && (perm.isPermissionSet(Type.ADMIN) || perm.isPermissionSet(Type.READ))) {
if (pattern.matcher(project.getName()).find()) {
array.add(project);
}
}
}
return array;
}
private void updateProjectProperty(
Connection connection, Project project, String name, Props props)
throws ProjectManagerException, IOException {
QueryRunner runner = new QueryRunner();
final String UPDATE_PROPERTIES =
"UPDATE project_properties SET property=? WHERE project_id=? AND version=? AND name=?";
String propertyJSON = PropsUtils.toJSONString(props, true);
byte[] data = propertyJSON.getBytes("UTF-8");
logger.info("UTF-8 size:" + data.length);
if (defaultEncodingType == EncodingType.GZIP) {
data = GZIPUtils.gzipBytes(data);
}
try {
runner.update(
connection, UPDATE_PROPERTIES, data, project.getId(), project.getVersion(), name);
connection.commit();
} catch (SQLException e) {
throw new ProjectManagerException(
"Error updating property " + project.getName() + " version " + project.getVersion(), e);
}
}
@Override
public void uploadProjectFile(
Project project,
int version,
String filetype,
String filename,
File localFile,
String uploader)
throws ProjectManagerException {
logger.info("Uploading to " + project.getName() + " version:" + version + " file:" + filename);
Connection connection = getConnection();
try {
uploadProjectFile(connection, project, version, filetype, filename, localFile, uploader);
connection.commit();
logger.info("Commiting upload " + localFile.getName());
} catch (SQLException e) {
logger.error(e);
throw new ProjectManagerException("Error getting DB connection.", e);
} finally {
DbUtils.closeQuietly(connection);
}
}
public void removeProjectPermission(Project project, String name, boolean group, User modifier)
throws ProjectManagerException {
logger.info(
"User "
+ modifier.getUserId()
+ " removing permissions for project "
+ project.getName()
+ " for "
+ name);
projectLoader.removePermission(project, name, group);
if (group) {
projectLoader.postEvent(
project,
EventType.GROUP_PERMISSION,
modifier.getUserId(),
"Permission for group " + name + " removed.");
} else {
projectLoader.postEvent(
project,
EventType.USER_PERMISSION,
modifier.getUserId(),
"Permission for user " + name + " removed.");
}
}
@Override
public List<Flow> fetchAllProjectFlows(Project project) throws ProjectManagerException {
QueryRunner runner = createQueryRunner();
ProjectFlowsResultHandler handler = new ProjectFlowsResultHandler();
List<Flow> flows = null;
try {
flows =
runner.query(
ProjectFlowsResultHandler.SELECT_ALL_PROJECT_FLOWS,
handler,
project.getId(),
project.getVersion());
} catch (SQLException e) {
throw new ProjectManagerException(
"Error fetching flows from project "
+ project.getName()
+ " version "
+ project.getVersion(),
e);
}
return flows;
}
private void uploadProjectProperty(
Connection connection, Project project, String name, Props props)
throws ProjectManagerException, IOException {
QueryRunner runner = new QueryRunner();
final String INSERT_PROPERTIES =
"INSERT INTO project_properties (project_id, version, name, modified_time, encoding_type, property) values (?,?,?,?,?,?)";
String propertyJSON = PropsUtils.toJSONString(props, true);
byte[] data = propertyJSON.getBytes("UTF-8");
logger.info("UTF-8 size:" + data.length);
if (defaultEncodingType == EncodingType.GZIP) {
data = GZIPUtils.gzipBytes(data);
}
try {
runner.update(
connection,
INSERT_PROPERTIES,
project.getId(),
project.getVersion(),
name,
System.currentTimeMillis(),
defaultEncodingType.getNumVal(),
data);
connection.commit();
} catch (SQLException e) {
throw new ProjectManagerException(
"Error uploading project properties "
+ name
+ " into "
+ project.getName()
+ " version "
+ project.getVersion(),
e);
}
}
public void uploadProject(Project project, File archive, String fileType, User uploader)
throws ProjectManagerException {
logger.info("Uploading files to " + project.getName());
// Unzip.
File file = null;
try {
if (fileType == null) {
throw new ProjectManagerException("Unknown file type for " + archive.getName());
} else if ("zip".equals(fileType)) {
file = unzipFile(archive);
} else {
throw new ProjectManagerException("Unsupported archive type for file " + archive.getName());
}
} catch (IOException e) {
throw new ProjectManagerException("Error unzipping file.", e);
}
logger.info("Validating Flow for upload " + archive.getName());
DirectoryFlowLoader loader = new DirectoryFlowLoader(logger);
loader.loadProjectFlow(file);
if (!loader.getErrors().isEmpty()) {
logger.error("Error found in upload to " + project.getName() + ". Cleaning up.");
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
StringBuffer errorMessage = new StringBuffer();
errorMessage.append("Error found in upload. Cannot upload.\n");
for (String error : loader.getErrors()) {
errorMessage.append(error);
errorMessage.append('\n');
}
throw new ProjectManagerException(errorMessage.toString());
}
Map<String, Props> jobProps = loader.getJobProps();
List<Props> propProps = loader.getProps();
synchronized (project) {
int newVersion = projectLoader.getLatestProjectVersion(project) + 1;
Map<String, Flow> flows = loader.getFlowMap();
for (Flow flow : flows.values()) {
flow.setProjectId(project.getId());
flow.setVersion(newVersion);
}
logger.info("Uploading file to db " + archive.getName());
projectLoader.uploadProjectFile(
project, newVersion, fileType, archive.getName(), archive, uploader.getUserId());
logger.info("Uploading flow to db " + archive.getName());
projectLoader.uploadFlows(project, newVersion, flows.values());
logger.info("Changing project versions " + archive.getName());
projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId());
project.setFlows(flows);
logger.info("Uploading Job properties");
projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values()));
logger.info("Uploading Props properties");
projectLoader.uploadProjectProperties(project, propProps);
}
logger.info("Uploaded project files. Cleaning up temp files.");
projectLoader.postEvent(
project,
EventType.UPLOADED,
uploader.getUserId(),
"Uploaded project files zip " + archive.getName());
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
logger.info(
"Cleaning up old install files older than "
+ (project.getVersion() - projectVersionRetention));
projectLoader.cleanOlderProjectVersion(
project.getId(), project.getVersion() - projectVersionRetention);
}
private void uploadProjectFile(
Connection connection,
Project project,
int version,
String filetype,
String filename,
File localFile,
String uploader)
throws ProjectManagerException {
QueryRunner runner = new QueryRunner();
long updateTime = System.currentTimeMillis();
logger.info("Creating message digest for upload " + localFile.getName());
byte[] md5 = null;
try {
md5 = Md5Hasher.md5Hash(localFile);
} catch (IOException e) {
throw new ProjectManagerException("Error getting md5 hash.", e);
}
logger.info("Md5 hash created");
// Really... I doubt we'll get a > 2gig file. So int casting it is!
byte[] buffer = new byte[CHUCK_SIZE];
final String INSERT_PROJECT_FILES =
"INSERT INTO project_files (project_id, version, chunk, size, file) values (?,?,?,?,?)";
// int numChunks = (localFileSize / buffer.length) + 1;
BufferedInputStream bufferedStream = null;
int chunk = 0;
try {
bufferedStream = new BufferedInputStream(new FileInputStream(localFile));
int size = bufferedStream.read(buffer);
while (size >= 0) {
logger.info("Read bytes for " + filename + " size:" + size);
byte[] buf = buffer;
if (size < buffer.length) {
buf = Arrays.copyOfRange(buffer, 0, size);
}
try {
logger.info("Running update for " + filename + " chunk " + chunk);
runner.update(
connection, INSERT_PROJECT_FILES, project.getId(), version, chunk, size, buf);
logger.info("Finished update for " + filename + " chunk " + chunk);
} catch (SQLException e) {
throw new ProjectManagerException("Error chunking", e);
}
++chunk;
size = bufferedStream.read(buffer);
}
} catch (IOException e) {
throw new ProjectManagerException("Error chunking file " + filename);
} finally {
IOUtils.closeQuietly(bufferedStream);
}
final String INSERT_PROJECT_VERSION =
"INSERT INTO project_versions (project_id, version, upload_time, uploader, file_type, file_name, md5, num_chunks) values (?,?,?,?,?,?,?,?)";
try {
runner.update(
connection,
INSERT_PROJECT_VERSION,
project.getId(),
version,
updateTime,
uploader,
filetype,
filename,
md5,
chunk);
} catch (SQLException e) {
logger.error(e);
throw new ProjectManagerException("Error updating project version " + project.getName(), e);
}
}
public Map<String, ValidationReport> uploadProject(
Project project, File archive, String fileType, User uploader, Props additionalProps)
throws ProjectManagerException {
logger.info("Uploading files to " + project.getName());
// Unzip.
File file = null;
try {
if (fileType == null) {
throw new ProjectManagerException("Unknown file type for " + archive.getName());
} else if ("zip".equals(fileType)) {
file = unzipFile(archive);
} else {
throw new ProjectManagerException("Unsupported archive type for file " + archive.getName());
}
} catch (IOException e) {
throw new ProjectManagerException("Error unzipping file.", e);
}
// Since props is an instance variable of ProjectManager, and each
// invocation to the uploadProject manager needs to pass a different
// value for the PROJECT_ARCHIVE_FILE_PATH key, it is necessary to
// create a new instance of Props to make sure these different values
// are isolated from each other.
Props prop = new Props(props);
prop.putAll(additionalProps);
prop.put(ValidatorConfigs.PROJECT_ARCHIVE_FILE_PATH, archive.getAbsolutePath());
// Basically, we want to make sure that for different invocations to the
// uploadProject method,
// the validators are using different values for the
// PROJECT_ARCHIVE_FILE_PATH configuration key.
// In addition, we want to reload the validator objects for each upload, so
// that we can change the validator configuration files without having to
// restart Azkaban web server. If the XmlValidatorManager is an instance
// variable, 2 consecutive invocations to the uploadProject
// method might cause the second one to overwrite the
// PROJECT_ARCHIVE_FILE_PATH configuration parameter
// of the first, thus causing a wrong archive file path to be passed to the
// validators. Creating a separate XmlValidatorManager object for each
// upload will prevent this issue without having to add
// synchronization between uploads. Since we're already reloading the XML
// config file and creating validator objects for each upload, this does
// not add too much additional overhead.
ValidatorManager validatorManager = new XmlValidatorManager(prop);
logger.info(
"Validating project "
+ archive.getName()
+ " using the registered validators "
+ validatorManager.getValidatorsInfo().toString());
Map<String, ValidationReport> reports = validatorManager.validate(project, file);
ValidationStatus status = ValidationStatus.PASS;
for (Entry<String, ValidationReport> report : reports.entrySet()) {
if (report.getValue().getStatus().compareTo(status) > 0) {
status = report.getValue().getStatus();
}
}
if (status == ValidationStatus.ERROR) {
logger.error("Error found in upload to " + project.getName() + ". Cleaning up.");
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
return reports;
}
DirectoryFlowLoader loader = (DirectoryFlowLoader) validatorManager.getDefaultValidator();
Map<String, Props> jobProps = loader.getJobProps();
List<Props> propProps = loader.getProps();
synchronized (project) {
int newVersion = projectLoader.getLatestProjectVersion(project) + 1;
Map<String, Flow> flows = loader.getFlowMap();
for (Flow flow : flows.values()) {
flow.setProjectId(project.getId());
flow.setVersion(newVersion);
}
logger.info("Uploading file to db " + archive.getName());
projectLoader.uploadProjectFile(
project, newVersion, fileType, archive.getName(), archive, uploader.getUserId());
logger.info("Uploading flow to db " + archive.getName());
projectLoader.uploadFlows(project, newVersion, flows.values());
logger.info("Changing project versions " + archive.getName());
projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId());
project.setFlows(flows);
logger.info("Uploading Job properties");
projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values()));
logger.info("Uploading Props properties");
projectLoader.uploadProjectProperties(project, propProps);
}
logger.info("Uploaded project files. Cleaning up temp files.");
projectLoader.postEvent(
project,
EventType.UPLOADED,
uploader.getUserId(),
"Uploaded project files zip " + archive.getName());
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
logger.info(
"Cleaning up old install files older than "
+ (project.getVersion() - projectVersionRetention));
projectLoader.cleanOlderProjectVersion(
project.getId(), project.getVersion() - projectVersionRetention);
return reports;
}