private void uploadFlow(
Connection connection, Project project, int version, Flow flow, EncodingType encType)
throws ProjectManagerException, IOException {
QueryRunner runner = new QueryRunner();
String json = JSONUtils.toJSON(flow.toObject());
byte[] stringData = json.getBytes("UTF-8");
byte[] data = stringData;
logger.info("UTF-8 size:" + data.length);
if (encType == EncodingType.GZIP) {
data = GZIPUtils.gzipBytes(stringData);
}
logger.info("Flow upload " + flow.getId() + " is byte size " + data.length);
final String INSERT_FLOW =
"INSERT INTO project_flows (project_id, version, flow_id, modified_time, encoding_type, json) values (?,?,?,?,?,?)";
try {
runner.update(
connection,
INSERT_FLOW,
project.getId(),
version,
flow.getId(),
System.currentTimeMillis(),
encType.getNumVal(),
data);
} catch (SQLException e) {
throw new ProjectManagerException("Error inserting flow " + flow.getId(), e);
}
}
public ExecutableFlow(Flow flow) {
this.projectId = flow.getProjectId();
this.flowId = flow.getId();
this.version = flow.getVersion();
this.setFlow(flow);
}
public ExecutableFlow(int executionId, Flow flow) {
this.projectId = flow.getProjectId();
this.flowId = flow.getId();
this.version = flow.getVersion();
this.executionId = executionId;
this.setFlow(flow);
}
private void loadAllProjectFlows(Project project) {
try {
List<Flow> flows = projectLoader.fetchAllProjectFlows(project);
Map<String, Flow> flowMap = new HashMap<String, Flow>();
for (Flow flow : flows) {
flowMap.put(flow.getId(), flow);
}
project.setFlows(flowMap);
} catch (ProjectManagerException e) {
throw new RuntimeException("Could not load projects flows from store.", e);
}
}
@Override
public void updateFlow(Project project, int version, Flow flow) throws ProjectManagerException {
logger.info("Uploading flows");
Connection connection = getConnection();
try {
QueryRunner runner = new QueryRunner();
String json = JSONUtils.toJSON(flow.toObject());
byte[] stringData = json.getBytes("UTF-8");
byte[] data = stringData;
logger.info("UTF-8 size:" + data.length);
if (defaultEncodingType == EncodingType.GZIP) {
data = GZIPUtils.gzipBytes(stringData);
}
logger.info("Flow upload " + flow.getId() + " is byte size " + data.length);
final String UPDATE_FLOW =
"UPDATE project_flows SET encoding_type=?,json=? WHERE project_id=? AND version=? AND flow_id=?";
try {
runner.update(
connection,
UPDATE_FLOW,
defaultEncodingType.getNumVal(),
data,
project.getId(),
version,
flow.getId());
} catch (SQLException e) {
e.printStackTrace();
throw new ProjectManagerException("Error inserting flow " + flow.getId(), e);
}
connection.commit();
} catch (IOException e) {
throw new ProjectManagerException("Flow Upload failed.", e);
} catch (SQLException e) {
throw new ProjectManagerException("Flow Upload failed commit.", e);
} finally {
DbUtils.closeQuietly(connection);
}
}
@Override
public List<Flow> handle(ResultSet rs) throws SQLException {
if (!rs.next()) {
return Collections.<Flow>emptyList();
}
ArrayList<Flow> flows = new ArrayList<Flow>();
do {
// int projectId = rs.getInt(1);
// int version = rs.getInt(2);
String flowId = rs.getString(3);
// long modifiedTime = rs.getLong(4);
int encodingType = rs.getInt(5);
byte[] dataBytes = rs.getBytes(6);
if (dataBytes == null) {
continue;
}
EncodingType encType = EncodingType.fromInteger(encodingType);
Object flowObj = null;
try {
// Convoluted way to inflate strings. Should find common package or
// helper function.
if (encType == EncodingType.GZIP) {
// Decompress the sucker.
String jsonString = GZIPUtils.unGzipString(dataBytes, "UTF-8");
flowObj = JSONUtils.parseJSONFromString(jsonString);
} else {
String jsonString = new String(dataBytes, "UTF-8");
flowObj = JSONUtils.parseJSONFromString(jsonString);
}
Flow flow = Flow.flowFromObject(flowObj);
flows.add(flow);
} catch (IOException e) {
throw new SQLException("Error retrieving flow data " + flowId, e);
}
} while (rs.next());
return flows;
}
private void setFlow(Flow flow) {
for (Node node : flow.getNodes()) {
String id = node.getId();
ExecutableNode exNode = new ExecutableNode(node, this);
executableNodes.put(id, exNode);
}
for (Edge edge : flow.getEdges()) {
ExecutableNode sourceNode = executableNodes.get(edge.getSourceId());
ExecutableNode targetNode = executableNodes.get(edge.getTargetId());
sourceNode.addOutNode(edge.getTargetId());
targetNode.addInNode(edge.getSourceId());
}
if (flow.getSuccessEmails() != null) {
successEmails = new ArrayList<String>(flow.getSuccessEmails());
}
if (flow.getFailureEmails() != null) {
failureEmails = new ArrayList<String>(flow.getFailureEmails());
}
flowProps.putAll(flow.getAllFlowProps());
}
public void uploadProject(Project project, File archive, String fileType, User uploader)
throws ProjectManagerException {
logger.info("Uploading files to " + project.getName());
// Unzip.
File file = null;
try {
if (fileType == null) {
throw new ProjectManagerException("Unknown file type for " + archive.getName());
} else if ("zip".equals(fileType)) {
file = unzipFile(archive);
} else {
throw new ProjectManagerException("Unsupported archive type for file " + archive.getName());
}
} catch (IOException e) {
throw new ProjectManagerException("Error unzipping file.", e);
}
logger.info("Validating Flow for upload " + archive.getName());
DirectoryFlowLoader loader = new DirectoryFlowLoader(logger);
loader.loadProjectFlow(file);
if (!loader.getErrors().isEmpty()) {
logger.error("Error found in upload to " + project.getName() + ". Cleaning up.");
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
StringBuffer errorMessage = new StringBuffer();
errorMessage.append("Error found in upload. Cannot upload.\n");
for (String error : loader.getErrors()) {
errorMessage.append(error);
errorMessage.append('\n');
}
throw new ProjectManagerException(errorMessage.toString());
}
Map<String, Props> jobProps = loader.getJobProps();
List<Props> propProps = loader.getProps();
synchronized (project) {
int newVersion = projectLoader.getLatestProjectVersion(project) + 1;
Map<String, Flow> flows = loader.getFlowMap();
for (Flow flow : flows.values()) {
flow.setProjectId(project.getId());
flow.setVersion(newVersion);
}
logger.info("Uploading file to db " + archive.getName());
projectLoader.uploadProjectFile(
project, newVersion, fileType, archive.getName(), archive, uploader.getUserId());
logger.info("Uploading flow to db " + archive.getName());
projectLoader.uploadFlows(project, newVersion, flows.values());
logger.info("Changing project versions " + archive.getName());
projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId());
project.setFlows(flows);
logger.info("Uploading Job properties");
projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values()));
logger.info("Uploading Props properties");
projectLoader.uploadProjectProperties(project, propProps);
}
logger.info("Uploaded project files. Cleaning up temp files.");
projectLoader.postEvent(
project,
EventType.UPLOADED,
uploader.getUserId(),
"Uploaded project files zip " + archive.getName());
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
logger.info(
"Cleaning up old install files older than "
+ (project.getVersion() - projectVersionRetention));
projectLoader.cleanOlderProjectVersion(
project.getId(), project.getVersion() - projectVersionRetention);
}
public void updateFlow(Project project, Flow flow) throws ProjectManagerException {
projectLoader.updateFlow(project, flow.getVersion(), flow);
}
public Map<String, ValidationReport> uploadProject(
Project project, File archive, String fileType, User uploader, Props additionalProps)
throws ProjectManagerException {
logger.info("Uploading files to " + project.getName());
// Unzip.
File file = null;
try {
if (fileType == null) {
throw new ProjectManagerException("Unknown file type for " + archive.getName());
} else if ("zip".equals(fileType)) {
file = unzipFile(archive);
} else {
throw new ProjectManagerException("Unsupported archive type for file " + archive.getName());
}
} catch (IOException e) {
throw new ProjectManagerException("Error unzipping file.", e);
}
// Since props is an instance variable of ProjectManager, and each
// invocation to the uploadProject manager needs to pass a different
// value for the PROJECT_ARCHIVE_FILE_PATH key, it is necessary to
// create a new instance of Props to make sure these different values
// are isolated from each other.
Props prop = new Props(props);
prop.putAll(additionalProps);
prop.put(ValidatorConfigs.PROJECT_ARCHIVE_FILE_PATH, archive.getAbsolutePath());
// Basically, we want to make sure that for different invocations to the
// uploadProject method,
// the validators are using different values for the
// PROJECT_ARCHIVE_FILE_PATH configuration key.
// In addition, we want to reload the validator objects for each upload, so
// that we can change the validator configuration files without having to
// restart Azkaban web server. If the XmlValidatorManager is an instance
// variable, 2 consecutive invocations to the uploadProject
// method might cause the second one to overwrite the
// PROJECT_ARCHIVE_FILE_PATH configuration parameter
// of the first, thus causing a wrong archive file path to be passed to the
// validators. Creating a separate XmlValidatorManager object for each
// upload will prevent this issue without having to add
// synchronization between uploads. Since we're already reloading the XML
// config file and creating validator objects for each upload, this does
// not add too much additional overhead.
ValidatorManager validatorManager = new XmlValidatorManager(prop);
logger.info(
"Validating project "
+ archive.getName()
+ " using the registered validators "
+ validatorManager.getValidatorsInfo().toString());
Map<String, ValidationReport> reports = validatorManager.validate(project, file);
ValidationStatus status = ValidationStatus.PASS;
for (Entry<String, ValidationReport> report : reports.entrySet()) {
if (report.getValue().getStatus().compareTo(status) > 0) {
status = report.getValue().getStatus();
}
}
if (status == ValidationStatus.ERROR) {
logger.error("Error found in upload to " + project.getName() + ". Cleaning up.");
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
return reports;
}
DirectoryFlowLoader loader = (DirectoryFlowLoader) validatorManager.getDefaultValidator();
Map<String, Props> jobProps = loader.getJobProps();
List<Props> propProps = loader.getProps();
synchronized (project) {
int newVersion = projectLoader.getLatestProjectVersion(project) + 1;
Map<String, Flow> flows = loader.getFlowMap();
for (Flow flow : flows.values()) {
flow.setProjectId(project.getId());
flow.setVersion(newVersion);
}
logger.info("Uploading file to db " + archive.getName());
projectLoader.uploadProjectFile(
project, newVersion, fileType, archive.getName(), archive, uploader.getUserId());
logger.info("Uploading flow to db " + archive.getName());
projectLoader.uploadFlows(project, newVersion, flows.values());
logger.info("Changing project versions " + archive.getName());
projectLoader.changeProjectVersion(project, newVersion, uploader.getUserId());
project.setFlows(flows);
logger.info("Uploading Job properties");
projectLoader.uploadProjectProperties(project, new ArrayList<Props>(jobProps.values()));
logger.info("Uploading Props properties");
projectLoader.uploadProjectProperties(project, propProps);
}
logger.info("Uploaded project files. Cleaning up temp files.");
projectLoader.postEvent(
project,
EventType.UPLOADED,
uploader.getUserId(),
"Uploaded project files zip " + archive.getName());
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
file.deleteOnExit();
e.printStackTrace();
}
logger.info(
"Cleaning up old install files older than "
+ (project.getVersion() - projectVersionRetention));
projectLoader.cleanOlderProjectVersion(
project.getId(), project.getVersion() - projectVersionRetention);
return reports;
}