/**
   * Extract fields from a file into a numeric array for machine learning.
   *
   * @param analyst The analyst to use.
   * @param headers The headers for the input data.
   * @param csv The CSV that holds the input data.
   * @param outputLength The length of the returned array.
   * @param skipOutput True if the output should be skipped.
   * @return The encoded data.
   */
  public static final double[] extractFields(
      final EncogAnalyst analyst,
      final CSVHeaders headers,
      final ReadCSV csv,
      final int outputLength,
      final boolean skipOutput) {
    final double[] output = new double[outputLength];
    int outputIndex = 0;
    for (final AnalystField stat : analyst.getScript().getNormalize().getNormalizedFields()) {

      stat.init();

      if (stat.getAction() == NormalizationAction.Ignore) {
        continue;
      }

      if (stat.isOutput() && skipOutput) {
        continue;
      }

      int index = headers.find(stat.getName());
      final String str = csv.get(index).trim();

      // is this an unknown value?
      if (str.equals("?") || str.length() == 0) {
        HandleMissingValues handler = analyst.getScript().getNormalize().getMissingValues();
        double[] d = handler.handleMissing(analyst, stat);

        // should we skip the entire row
        if (d == null) {
          return null;
        }

        // copy the returned values in place of the missing values
        for (int i = 0; i < d.length; i++) {
          output[outputIndex++] = d[i];
        }
      } else {
        // known value
        if (stat.getAction() == NormalizationAction.Normalize) {
          double d = csv.getFormat().parse(str);
          d = stat.normalize(d);
          output[outputIndex++] = d;
        } else if (stat.getAction() == NormalizationAction.PassThrough) {
          double d = csv.getFormat().parse(str);
          output[outputIndex++] = d;
        } else {
          final double[] d = stat.encode(str);
          for (final double element : d) {
            output[outputIndex++] = element;
          }
        }
      }
    }

    return output;
  }
  /**
   * Analyze the file.
   *
   * @param inputFilename The input file.
   * @param expectInputHeaders True, if input headers are present.
   * @param inputFormat The format.
   * @param theAnalyst The analyst to use.
   */
  public void analyze(
      final File inputFilename,
      final boolean expectInputHeaders,
      final CSVFormat inputFormat,
      final EncogAnalyst theAnalyst) {
    this.setInputFilename(inputFilename);
    this.setInputFormat(inputFormat);
    this.setExpectInputHeaders(expectInputHeaders);
    this.analyst = theAnalyst;
    this.setAnalyzed(true);

    this.analystHeaders = new CSVHeaders(inputFilename, expectInputHeaders, inputFormat);

    for (final AnalystField field : analyst.getScript().getNormalize().getNormalizedFields()) {
      field.init();
    }

    this.series = new TimeSeriesUtil(analyst, true, this.analystHeaders.getHeaders());
  }