private void uploadFlow( Connection connection, Project project, int version, Flow flow, EncodingType encType) throws ProjectManagerException, IOException { QueryRunner runner = new QueryRunner(); String json = JSONUtils.toJSON(flow.toObject()); byte[] stringData = json.getBytes("UTF-8"); byte[] data = stringData; logger.info("UTF-8 size:" + data.length); if (encType == EncodingType.GZIP) { data = GZIPUtils.gzipBytes(stringData); } logger.info("Flow upload " + flow.getId() + " is byte size " + data.length); final String INSERT_FLOW = "INSERT INTO project_flows (project_id, version, flow_id, modified_time, encoding_type, json) values (?,?,?,?,?,?)"; try { runner.update( connection, INSERT_FLOW, project.getId(), version, flow.getId(), System.currentTimeMillis(), encType.getNumVal(), data); } catch (SQLException e) { throw new ProjectManagerException("Error inserting flow " + flow.getId(), e); } }
private void updateProjectSettings(Connection connection, Project project, EncodingType encType) throws ProjectManagerException { QueryRunner runner = new QueryRunner(); final String UPDATE_PROJECT_SETTINGS = "UPDATE projects SET enc_type=?, settings_blob=? WHERE id=?"; String json = JSONUtils.toJSON(project.toObject()); byte[] data = null; try { byte[] stringData = json.getBytes("UTF-8"); data = stringData; if (encType == EncodingType.GZIP) { data = GZIPUtils.gzipBytes(stringData); } logger.debug( "NumChars: " + json.length() + " UTF-8:" + stringData.length + " Gzip:" + data.length); } catch (IOException e) { throw new ProjectManagerException("Failed to encode. ", e); } try { runner.update( connection, UPDATE_PROJECT_SETTINGS, encType.getNumVal(), data, project.getId()); connection.commit(); } catch (SQLException e) { throw new ProjectManagerException( "Error updating project " + project.getName() + " version " + project.getVersion(), e); } }
/** * Permanently delete all project files and properties data for all versions of a project and log * event in project_events table * * @param project * @param deleter * @return * @throws ProjectManagerException */ public synchronized Project purgeProject(Project project, User deleter) throws ProjectManagerException { projectLoader.cleanOlderProjectVersion(project.getId(), project.getVersion() + 1); projectLoader.postEvent( project, EventType.PURGE, deleter.getUserId(), String.format("Purged versions before %d", project.getVersion() + 1)); return project; }
public synchronized Project removeProject(Project project, User deleter) throws ProjectManagerException { projectLoader.removeProject(project, deleter.getUserId()); projectLoader.postEvent(project, EventType.DELETED, deleter.getUserId(), null); projectsByName.remove(project.getName()); projectsById.remove(project.getId()); return project; }
@Override public void updatePermission(Project project, String name, Permission perm, boolean isGroup) throws ProjectManagerException { QueryRunner runner = createQueryRunner(); if (this.allowsOnDuplicateKey()) { long updateTime = System.currentTimeMillis(); final String INSERT_PROJECT_PERMISSION = "INSERT INTO project_permissions (project_id, modified_time, name, permissions, isGroup) values (?,?,?,?,?)" + "ON DUPLICATE KEY UPDATE modified_time = VALUES(modified_time), permissions = VALUES(permissions)"; try { runner.update( INSERT_PROJECT_PERMISSION, project.getId(), updateTime, name, perm.toFlags(), isGroup); } catch (SQLException e) { logger.error(e); throw new ProjectManagerException( "Error updating project " + project.getName() + " permissions for " + name, e); } } else { long updateTime = System.currentTimeMillis(); final String MERGE_PROJECT_PERMISSION = "MERGE INTO project_permissions (project_id, modified_time, name, permissions, isGroup) KEY (project_id, name) values (?,?,?,?,?)"; try { runner.update( MERGE_PROJECT_PERMISSION, project.getId(), updateTime, name, perm.toFlags(), isGroup); } catch (SQLException e) { logger.error(e); throw new ProjectManagerException( "Error updating project " + project.getName() + " permissions for " + name, e); } } if (isGroup) { project.setGroupPermission(name, perm); } else { project.setUserPermission(name, perm); } }
@Override public int getLatestProjectVersion(Project project) throws ProjectManagerException { QueryRunner runner = createQueryRunner(); IntHander handler = new IntHander(); try { return runner.query(IntHander.SELECT_LATEST_VERSION, handler, project.getId()); } catch (SQLException e) { logger.error(e); throw new ProjectManagerException( "Error marking project " + project.getName() + " as inactive", e); } }
@Override public ProjectFileHandler getUploadedFile(Project project, int version) throws ProjectManagerException { logger.info("Retrieving to " + project.getName() + " version:" + version); Connection connection = getConnection(); ProjectFileHandler handler = null; try { handler = getUploadedFile(connection, project.getId(), version); } finally { DbUtils.closeQuietly(connection); } return handler; }
@Override public void removeProject(Project project, String user) throws ProjectManagerException { QueryRunner runner = createQueryRunner(); long updateTime = System.currentTimeMillis(); final String UPDATE_INACTIVE_PROJECT = "UPDATE projects SET active=false,modified_time=?,last_modified_by=? WHERE id=?"; try { runner.update(UPDATE_INACTIVE_PROJECT, updateTime, user, project.getId()); } catch (SQLException e) { logger.error(e); throw new ProjectManagerException( "Error marking project " + project.getName() + " as inactive", e); } }
private void loadAllProjects() { List<Project> projects; try { projects = projectLoader.fetchAllActiveProjects(); } catch (ProjectManagerException e) { throw new RuntimeException("Could not load projects from store.", e); } for (Project proj : projects) { projectsByName.put(proj.getName(), proj); projectsById.put(proj.getId(), proj); } for (Project proj : projects) { loadAllProjectFlows(proj); } }
@Override public boolean postEvent(Project project, EventType type, String user, String message) { QueryRunner runner = createQueryRunner(); final String INSERT_PROJECT_EVENTS = "INSERT INTO project_events (project_id, event_type, event_time, username, message) values (?,?,?,?,?)"; long updateTime = System.currentTimeMillis(); try { runner.update( INSERT_PROJECT_EVENTS, project.getId(), type.getNumVal(), updateTime, user, message); } catch (SQLException e) { e.printStackTrace(); return false; } return true; }
@Override public void changeProjectVersion(Project project, int version, String user) throws ProjectManagerException { long timestamp = System.currentTimeMillis(); QueryRunner runner = createQueryRunner(); try { final String UPDATE_PROJECT_VERSION = "UPDATE projects SET version=?,modified_time=?,last_modified_by=? WHERE id=?"; runner.update(UPDATE_PROJECT_VERSION, version, timestamp, user, project.getId()); project.setVersion(version); project.setLastModifiedTimestamp(timestamp); project.setLastModifiedUser(user); } catch (SQLException e) { logger.error(e); throw new ProjectManagerException( "Error updating switching project version " + project.getName(), e); } }
@Override public void updateDescription(Project project, String description, String user) throws ProjectManagerException { QueryRunner runner = createQueryRunner(); final String UPDATE_PROJECT_DESCRIPTION = "UPDATE projects SET description=?,modified_time=?,last_modified_by=? WHERE id=?"; long updateTime = System.currentTimeMillis(); try { runner.update(UPDATE_PROJECT_DESCRIPTION, description, updateTime, user, project.getId()); project.setDescription(description); project.setLastModifiedTimestamp(updateTime); project.setLastModifiedUser(user); } catch (SQLException e) { logger.error(e); throw new ProjectManagerException( "Error marking project " + project.getName() + " as inactive", e); } }
private List<Triple<String, Boolean, Permission>> fetchPermissionsForProject( Connection connection, Project project) throws ProjectManagerException { ProjectPermissionsResultHandler permHander = new ProjectPermissionsResultHandler(); QueryRunner runner = new QueryRunner(); List<Triple<String, Boolean, Permission>> permissions = null; try { permissions = runner.query( connection, ProjectPermissionsResultHandler.SELECT_PROJECT_PERMISSION, permHander, project.getId()); } catch (SQLException e) { throw new ProjectManagerException( "Query for permissions for " + project.getName() + " failed.", e); } return permissions; }
/** * Get all the logs for a given project * * @param project * @return * @throws ProjectManagerException */ public List<ProjectLogEvent> getProjectEvents(Project project, int num, int skip) throws ProjectManagerException { QueryRunner runner = createQueryRunner(); ProjectLogsResultHandler logHandler = new ProjectLogsResultHandler(); List<ProjectLogEvent> events = null; try { events = runner.query( ProjectLogsResultHandler.SELECT_PROJECT_EVENTS_ORDER, logHandler, project.getId(), num, skip); } catch (SQLException e) { logger.error(e); } return events; }
public Project createProject(String projectName, String description, User creator) throws ProjectManagerException { if (projectName == null || projectName.trim().isEmpty()) { throw new ProjectManagerException("Project name cannot be empty."); } else if (description == null || description.trim().isEmpty()) { throw new ProjectManagerException("Description cannot be empty."); } else if (creator == null) { throw new ProjectManagerException("Valid creator user must be set."); } else if (!projectName.matches("[a-zA-Z][a-zA-Z_0-9|-]*")) { throw new ProjectManagerException( "Project names must start with a letter, followed by any number of letters, digits, '-' or '_'."); } if (projectsByName.contains(projectName)) { throw new ProjectManagerException("Project already exists."); } logger.info("Trying to create " + projectName + " by user " + creator.getUserId()); Project newProject = projectLoader.createNewProject(projectName, description, creator); projectsByName.put(newProject.getName(), newProject); projectsById.put(newProject.getId(), newProject); if (creatorDefaultPermissions) { // Add permission to project projectLoader.updatePermission( newProject, creator.getUserId(), new Permission(Permission.Type.ADMIN), false); // Add proxy user newProject.getProxyUsers().add(creator.getUserId()); try { updateProjectSetting(newProject); } catch (ProjectManagerException e) { e.printStackTrace(); throw e; } } projectLoader.postEvent(newProject, EventType.CREATED, creator.getUserId(), null); return newProject; }
@Override public void updateFlow(Project project, int version, Flow flow) throws ProjectManagerException { logger.info("Uploading flows"); Connection connection = getConnection(); try { QueryRunner runner = new QueryRunner(); String json = JSONUtils.toJSON(flow.toObject()); byte[] stringData = json.getBytes("UTF-8"); byte[] data = stringData; logger.info("UTF-8 size:" + data.length); if (defaultEncodingType == EncodingType.GZIP) { data = GZIPUtils.gzipBytes(stringData); } logger.info("Flow upload " + flow.getId() + " is byte size " + data.length); final String UPDATE_FLOW = "UPDATE project_flows SET encoding_type=?,json=? WHERE project_id=? AND version=? AND flow_id=?"; try { runner.update( connection, UPDATE_FLOW, defaultEncodingType.getNumVal(), data, project.getId(), version, flow.getId()); } catch (SQLException e) { e.printStackTrace(); throw new ProjectManagerException("Error inserting flow " + flow.getId(), e); } connection.commit(); } catch (IOException e) { throw new ProjectManagerException("Flow Upload failed.", e); } catch (SQLException e) { throw new ProjectManagerException("Flow Upload failed commit.", e); } finally { DbUtils.closeQuietly(connection); } }
@Override public void removePermission(Project project, String name, boolean isGroup) throws ProjectManagerException { QueryRunner runner = createQueryRunner(); final String DELETE_PROJECT_PERMISSION = "DELETE FROM project_permissions WHERE project_id=? AND name=? AND isGroup=?"; try { runner.update(DELETE_PROJECT_PERMISSION, project.getId(), name, isGroup); } catch (SQLException e) { logger.error(e); throw new ProjectManagerException( "Error deleting project " + project.getName() + " permissions for " + name, e); } if (isGroup) { project.removeGroupPermission(name); } else { project.removeUserPermission(name); } }
@Override public Flow fetchFlow(Project project, String flowId) throws ProjectManagerException { QueryRunner runner = createQueryRunner(); ProjectFlowsResultHandler handler = new ProjectFlowsResultHandler(); try { List<Flow> flows = runner.query( ProjectFlowsResultHandler.SELECT_PROJECT_FLOW, handler, project.getId(), project.getVersion(), flowId); if (flows.isEmpty()) { return null; } else { return flows.get(0); } } catch (SQLException e) { throw new ProjectManagerException("Error fetching flow " + flowId, e); } }
private void updateProjectProperty( Connection connection, Project project, String name, Props props) throws ProjectManagerException, IOException { QueryRunner runner = new QueryRunner(); final String UPDATE_PROPERTIES = "UPDATE project_properties SET property=? WHERE project_id=? AND version=? AND name=?"; String propertyJSON = PropsUtils.toJSONString(props, true); byte[] data = propertyJSON.getBytes("UTF-8"); logger.info("UTF-8 size:" + data.length); if (defaultEncodingType == EncodingType.GZIP) { data = GZIPUtils.gzipBytes(data); } try { runner.update( connection, UPDATE_PROPERTIES, data, project.getId(), project.getVersion(), name); connection.commit(); } catch (SQLException e) { throw new ProjectManagerException( "Error updating property " + project.getName() + " version " + project.getVersion(), e); } }
@Override public Props fetchProjectProperty(Project project, String propsName) throws ProjectManagerException { QueryRunner runner = createQueryRunner(); ProjectPropertiesResultsHandler handler = new ProjectPropertiesResultsHandler(); try { List<Pair<String, Props>> properties = runner.query( ProjectPropertiesResultsHandler.SELECT_PROJECT_PROPERTY, handler, project.getId(), project.getVersion(), propsName); if (properties == null || properties.isEmpty()) { return null; } return properties.get(0).getSecond(); } catch (SQLException e) { throw new ProjectManagerException("Error fetching property " + propsName, e); } }
private void uploadProjectProperty( Connection connection, Project project, String name, Props props) throws ProjectManagerException, IOException { QueryRunner runner = new QueryRunner(); final String INSERT_PROPERTIES = "INSERT INTO project_properties (project_id, version, name, modified_time, encoding_type, property) values (?,?,?,?,?,?)"; String propertyJSON = PropsUtils.toJSONString(props, true); byte[] data = propertyJSON.getBytes("UTF-8"); logger.info("UTF-8 size:" + data.length); if (defaultEncodingType == EncodingType.GZIP) { data = GZIPUtils.gzipBytes(data); } try { runner.update( connection, INSERT_PROPERTIES, project.getId(), project.getVersion(), name, System.currentTimeMillis(), defaultEncodingType.getNumVal(), data); connection.commit(); } catch (SQLException e) { throw new ProjectManagerException( "Error uploading project properties " + name + " into " + project.getName() + " version " + project.getVersion(), e); } }
@Override public List<Flow> fetchAllProjectFlows(Project project) throws ProjectManagerException { QueryRunner runner = createQueryRunner(); ProjectFlowsResultHandler handler = new ProjectFlowsResultHandler(); List<Flow> flows = null; try { flows = runner.query( ProjectFlowsResultHandler.SELECT_ALL_PROJECT_FLOWS, handler, project.getId(), project.getVersion()); } catch (SQLException e) { throw new ProjectManagerException( "Error fetching flows from project " + project.getName() + " version " + project.getVersion(), e); } return flows; }
public void uploadProject(Project project, File archive, String fileType, User uploader) throws ProjectManagerException { logger.info("Uploading files to " + project.getName()); // Unzip. File file = null; try { if (fileType == null) { throw new ProjectManagerException("Unknown file type for " + archive.getName()); } else if ("zip".equals(fileType)) { file = unzipFile(archive); } else { throw new ProjectManagerException("Unsupported archive type for file " + archive.getName()); } } catch (IOException e) { throw new ProjectManagerException("Error unzipping file.", e); } logger.info("Validating Flow for upload " + archive.getName()); DirectoryFlowLoader loader = new DirectoryFlowLoader(logger); loader.loadProjectFlow(file); if (!loader.getErrors().isEmpty()) { logger.error("Error found in upload to " + project.getName() + ". Cleaning up."); try { FileUtils.deleteDirectory(file); } catch (IOException e) { file.deleteOnExit(); e.printStackTrace(); } StringBuffer errorMessage = new StringBuffer(); errorMessage.append("Error found in upload. Cannot upload.\n"); for (String error : loader.getErrors()) { errorMessage.append(error); errorMessage.append('\n'); } throw new ProjectManagerException(errorMessage.toString()); } Map<String, Props> jobProps = loader.getJobProps(); List<Props> propProps = loader.getProps(); synchronized (project) { int newVersion = projectLoader.getLatestProjectVersion(project) + 1; Map<String, Flow> flows = loader.getFlowMap(); for (Flow flow : flows.values()) { flow.setProjectId(project.getId()); flow.setVersion(newVersion); } logger.info("Uploading file to db " + archive.getName()); projectLoader.uploadProjectFile( project, newVersion, fileType, archive.getName(), archive, uploader.getUserId()); logger.info("Uploading flow to db " + archive.getName()); projectLoader.uploadFlows(project, newVersion, flows.values()); logger.info("Changing project versions " + archive.getName()); projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId()); project.setFlows(flows); logger.info("Uploading Job properties"); projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values())); logger.info("Uploading Props properties"); projectLoader.uploadProjectProperties(project, propProps); } logger.info("Uploaded project files. Cleaning up temp files."); projectLoader.postEvent( project, EventType.UPLOADED, uploader.getUserId(), "Uploaded project files zip " + archive.getName()); try { FileUtils.deleteDirectory(file); } catch (IOException e) { file.deleteOnExit(); e.printStackTrace(); } logger.info( "Cleaning up old install files older than " + (project.getVersion() - projectVersionRetention)); projectLoader.cleanOlderProjectVersion( project.getId(), project.getVersion() - projectVersionRetention); }
private void uploadProjectFile( Connection connection, Project project, int version, String filetype, String filename, File localFile, String uploader) throws ProjectManagerException { QueryRunner runner = new QueryRunner(); long updateTime = System.currentTimeMillis(); logger.info("Creating message digest for upload " + localFile.getName()); byte[] md5 = null; try { md5 = Md5Hasher.md5Hash(localFile); } catch (IOException e) { throw new ProjectManagerException("Error getting md5 hash.", e); } logger.info("Md5 hash created"); // Really... I doubt we'll get a > 2gig file. So int casting it is! byte[] buffer = new byte[CHUCK_SIZE]; final String INSERT_PROJECT_FILES = "INSERT INTO project_files (project_id, version, chunk, size, file) values (?,?,?,?,?)"; // int numChunks = (localFileSize / buffer.length) + 1; BufferedInputStream bufferedStream = null; int chunk = 0; try { bufferedStream = new BufferedInputStream(new FileInputStream(localFile)); int size = bufferedStream.read(buffer); while (size >= 0) { logger.info("Read bytes for " + filename + " size:" + size); byte[] buf = buffer; if (size < buffer.length) { buf = Arrays.copyOfRange(buffer, 0, size); } try { logger.info("Running update for " + filename + " chunk " + chunk); runner.update( connection, INSERT_PROJECT_FILES, project.getId(), version, chunk, size, buf); logger.info("Finished update for " + filename + " chunk " + chunk); } catch (SQLException e) { throw new ProjectManagerException("Error chunking", e); } ++chunk; size = bufferedStream.read(buffer); } } catch (IOException e) { throw new ProjectManagerException("Error chunking file " + filename); } finally { IOUtils.closeQuietly(bufferedStream); } final String INSERT_PROJECT_VERSION = "INSERT INTO project_versions (project_id, version, upload_time, uploader, file_type, file_name, md5, num_chunks) values (?,?,?,?,?,?,?,?)"; try { runner.update( connection, INSERT_PROJECT_VERSION, project.getId(), version, updateTime, uploader, filetype, filename, md5, chunk); } catch (SQLException e) { logger.error(e); throw new ProjectManagerException("Error updating project version " + project.getName(), e); } }
public Map<String, ValidationReport> uploadProject( Project project, File archive, String fileType, User uploader, Props additionalProps) throws ProjectManagerException { logger.info("Uploading files to " + project.getName()); // Unzip. File file = null; try { if (fileType == null) { throw new ProjectManagerException("Unknown file type for " + archive.getName()); } else if ("zip".equals(fileType)) { file = unzipFile(archive); } else { throw new ProjectManagerException("Unsupported archive type for file " + archive.getName()); } } catch (IOException e) { throw new ProjectManagerException("Error unzipping file.", e); } // Since props is an instance variable of ProjectManager, and each // invocation to the uploadProject manager needs to pass a different // value for the PROJECT_ARCHIVE_FILE_PATH key, it is necessary to // create a new instance of Props to make sure these different values // are isolated from each other. Props prop = new Props(props); prop.putAll(additionalProps); prop.put(ValidatorConfigs.PROJECT_ARCHIVE_FILE_PATH, archive.getAbsolutePath()); // Basically, we want to make sure that for different invocations to the // uploadProject method, // the validators are using different values for the // PROJECT_ARCHIVE_FILE_PATH configuration key. // In addition, we want to reload the validator objects for each upload, so // that we can change the validator configuration files without having to // restart Azkaban web server. If the XmlValidatorManager is an instance // variable, 2 consecutive invocations to the uploadProject // method might cause the second one to overwrite the // PROJECT_ARCHIVE_FILE_PATH configuration parameter // of the first, thus causing a wrong archive file path to be passed to the // validators. Creating a separate XmlValidatorManager object for each // upload will prevent this issue without having to add // synchronization between uploads. Since we're already reloading the XML // config file and creating validator objects for each upload, this does // not add too much additional overhead. ValidatorManager validatorManager = new XmlValidatorManager(prop); logger.info( "Validating project " + archive.getName() + " using the registered validators " + validatorManager.getValidatorsInfo().toString()); Map<String, ValidationReport> reports = validatorManager.validate(project, file); ValidationStatus status = ValidationStatus.PASS; for (Entry<String, ValidationReport> report : reports.entrySet()) { if (report.getValue().getStatus().compareTo(status) > 0) { status = report.getValue().getStatus(); } } if (status == ValidationStatus.ERROR) { logger.error("Error found in upload to " + project.getName() + ". Cleaning up."); try { FileUtils.deleteDirectory(file); } catch (IOException e) { file.deleteOnExit(); e.printStackTrace(); } return reports; } DirectoryFlowLoader loader = (DirectoryFlowLoader) validatorManager.getDefaultValidator(); Map<String, Props> jobProps = loader.getJobProps(); List<Props> propProps = loader.getProps(); synchronized (project) { int newVersion = projectLoader.getLatestProjectVersion(project) + 1; Map<String, Flow> flows = loader.getFlowMap(); for (Flow flow : flows.values()) { flow.setProjectId(project.getId()); flow.setVersion(newVersion); } logger.info("Uploading file to db " + archive.getName()); projectLoader.uploadProjectFile( project, newVersion, fileType, archive.getName(), archive, uploader.getUserId()); logger.info("Uploading flow to db " + archive.getName()); projectLoader.uploadFlows(project, newVersion, flows.values()); logger.info("Changing project versions " + archive.getName()); projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId()); project.setFlows(flows); logger.info("Uploading Job properties"); projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values())); logger.info("Uploading Props properties"); projectLoader.uploadProjectProperties(project, propProps); } logger.info("Uploaded project files. Cleaning up temp files."); projectLoader.postEvent( project, EventType.UPLOADED, uploader.getUserId(), "Uploaded project files zip " + archive.getName()); try { FileUtils.deleteDirectory(file); } catch (IOException e) { file.deleteOnExit(); e.printStackTrace(); } logger.info( "Cleaning up old install files older than " + (project.getVersion() - projectVersionRetention)); projectLoader.cleanOlderProjectVersion( project.getId(), project.getVersion() - projectVersionRetention); return reports; }