Example #1
0
  /**
   * The constructor creates an instance of NavigatorPlugin to manage all of the requests/responses
   * to/from Cloudera Navigator
   */
  public LineageCreator(
      String clientApplicationUrl,
      String navigatorUrl,
      String metadataUri,
      String username,
      String password,
      String jobName,
      String projectName,
      Boolean autoCommit,
      Boolean disableValidationSSL,
      int apiVersion) {
    Map<String, Object> configurationMap = new HashMap<String, Object>();
    configurationMap.put(LineageCreator.APP_URL, clientApplicationUrl);
    configurationMap.put(LineageCreator.METADATA_URI, metadataUri);
    configurationMap.put(LineageCreator.USERNAME, username);
    configurationMap.put(LineageCreator.PASSWORD, password);
    // File Format allow only JSON for the moment
    configurationMap.put(LineageCreator.FILE_FORMAT, "JSON"); // $NON-NLS-1$
    configurationMap.put(LineageCreator.NAMESPACE, "Talend"); // $NON-NLS-1$

    // TODO the next two options are not fully integrated on the PluginConfigurationFactory API.
    // They will be soon.
    if (disableValidationSSL) {
      configurationMap.put(LineageCreator.DISABLE_SSL_VALIDATION, "true"); // $NON-NLS-1$
    }
    if (autoCommit) {
      configurationMap.put(LineageCreator.AUTOCOMMIT, "true"); // $NON-NLS-1$
    }

    configurationMap.put(LineageCreator.API_VERSION, apiVersion); // $NON-NLS-1$

    // Extract Navigator URL
    configurationMap.put(LineageCreator.NAV_URL, ClouderaAPIUtil.extractNavigatorURL(navigatorUrl));

    this.plugin = NavigatorPlugin.fromConfigMap(configurationMap);
    // We call the Cloudera Navigator API using the client
    // This call will fail if Cloudera Navigator is not available
    // TODO: Wrap this call with the same die on error that we use when writing the entities
    this.fileSystem = this.plugin.getClient().getOnlySource(SourceType.HDFS);
    this.jobName = jobName;
    this.projectName = projectName;
    this.creationInstant = new Instant();
  }
Example #2
0
  /**
   * Creates a TalendDataSet with the data retrieved from an input/output tFileInputOutputXXX Talend
   * Graphical Node.
   *
   * @param schema The Hashmap of column name and column type of the component.
   * @param componentName The component name
   * @param fileSystemPath The HDFS path to the file where the component read/write data
   * @param fileFormat The file format as a FileFormat
   */
  public void addDataset(
      Map<String, String> schema,
      String componentName,
      String fileSystemPath,
      FileFormat fileFormat) {
    HdfsEntity container =
        new HdfsEntity(fileSystemPath, EntityType.DIRECTORY, this.fileSystem.getIdentity());

    TalendDataset dataset =
        new TalendDataset(
            ClouderaAPIUtil.getDatasetName(fileSystemPath),
            componentName,
            this.jobName + this.projectName);
    dataset.setDataContainer(container);
    dataset.setFileFormat(fileFormat);
    dataset.setFields(ClouderaFieldConvertor.convertToTalendField(schema));
    dataset.addTags(ImmutableList.of("Talend", this.jobName)); // $NON-NLS-1$
    dataset.setCreated(this.creationInstant);

    this.datasets.add(dataset);
  }