/**
* The constructor creates an instance of NavigatorPlugin to manage all of the requests/responses
* to/from Cloudera Navigator
*/
public LineageCreator(
String clientApplicationUrl,
String navigatorUrl,
String metadataUri,
String username,
String password,
String jobName,
String projectName,
Boolean autoCommit,
Boolean disableValidationSSL,
int apiVersion) {
Map<String, Object> configurationMap = new HashMap<String, Object>();
configurationMap.put(LineageCreator.APP_URL, clientApplicationUrl);
configurationMap.put(LineageCreator.METADATA_URI, metadataUri);
configurationMap.put(LineageCreator.USERNAME, username);
configurationMap.put(LineageCreator.PASSWORD, password);
// File Format allow only JSON for the moment
configurationMap.put(LineageCreator.FILE_FORMAT, "JSON"); // $NON-NLS-1$
configurationMap.put(LineageCreator.NAMESPACE, "Talend"); // $NON-NLS-1$
// TODO the next two options are not fully integrated on the PluginConfigurationFactory API.
// They will be soon.
if (disableValidationSSL) {
configurationMap.put(LineageCreator.DISABLE_SSL_VALIDATION, "true"); // $NON-NLS-1$
}
if (autoCommit) {
configurationMap.put(LineageCreator.AUTOCOMMIT, "true"); // $NON-NLS-1$
}
configurationMap.put(LineageCreator.API_VERSION, apiVersion); // $NON-NLS-1$
// Extract Navigator URL
configurationMap.put(LineageCreator.NAV_URL, ClouderaAPIUtil.extractNavigatorURL(navigatorUrl));
this.plugin = NavigatorPlugin.fromConfigMap(configurationMap);
// We call the Cloudera Navigator API using the client
// This call will fail if Cloudera Navigator is not available
// TODO: Wrap this call with the same die on error that we use when writing the entities
this.fileSystem = this.plugin.getClient().getOnlySource(SourceType.HDFS);
this.jobName = jobName;
this.projectName = projectName;
this.creationInstant = new Instant();
}
/**
* Creates a TalendDataSet with the data retrieved from an input/output tFileInputOutputXXX Talend
* Graphical Node.
*
* @param schema The Hashmap of column name and column type of the component.
* @param componentName The component name
* @param fileSystemPath The HDFS path to the file where the component read/write data
* @param fileFormat The file format as a FileFormat
*/
public void addDataset(
Map<String, String> schema,
String componentName,
String fileSystemPath,
FileFormat fileFormat) {
HdfsEntity container =
new HdfsEntity(fileSystemPath, EntityType.DIRECTORY, this.fileSystem.getIdentity());
TalendDataset dataset =
new TalendDataset(
ClouderaAPIUtil.getDatasetName(fileSystemPath),
componentName,
this.jobName + this.projectName);
dataset.setDataContainer(container);
dataset.setFileFormat(fileFormat);
dataset.setFields(ClouderaFieldConvertor.convertToTalendField(schema));
dataset.addTags(ImmutableList.of("Talend", this.jobName)); // $NON-NLS-1$
dataset.setCreated(this.creationInstant);
this.datasets.add(dataset);
}
