/**
   * Determine the ideal fields.
   *
   * @param headerList The headers.
   * @return The indexes of the ideal fields.
   */
  private int[] determineIdealFields(final CSVHeaders headerList) {

    int[] result;
    final String type = getProp().getPropertyString(ScriptProperties.ML_CONFIG_TYPE);

    // is it non-supervised?
    if (type.equals(MLMethodFactory.TYPE_SOM)) {
      result = new int[0];
      return result;
    }

    final List<Integer> fields = new ArrayList<Integer>();

    for (int currentIndex = 0; currentIndex < headerList.size(); currentIndex++) {
      final String baseName = headerList.getBaseHeader(currentIndex);
      final int slice = headerList.getSlice(currentIndex);
      final AnalystField field = getAnalyst().getScript().findNormalizedField(baseName, slice);

      if (field != null && field.isOutput()) {
        fields.add(currentIndex);
      }
    }

    // allocate result array
    result = new int[fields.size()];
    for (int i = 0; i < result.length; i++) {
      result[i] = fields.get(i);
    }

    return result;
  }
  /**
   * Extract fields from a file into a numeric array for machine learning.
   *
   * @param analyst The analyst to use.
   * @param headers The headers for the input data.
   * @param csv The CSV that holds the input data.
   * @param outputLength The length of the returned array.
   * @param skipOutput True if the output should be skipped.
   * @return The encoded data.
   */
  public static final double[] extractFields(
      final EncogAnalyst analyst,
      final CSVHeaders headers,
      final ReadCSV csv,
      final int outputLength,
      final boolean skipOutput) {
    final double[] output = new double[outputLength];
    int outputIndex = 0;
    for (final AnalystField stat : analyst.getScript().getNormalize().getNormalizedFields()) {

      stat.init();

      if (stat.getAction() == NormalizationAction.Ignore) {
        continue;
      }

      if (stat.isOutput() && skipOutput) {
        continue;
      }

      int index = headers.find(stat.getName());
      final String str = csv.get(index).trim();

      // is this an unknown value?
      if (str.equals("?") || str.length() == 0) {
        HandleMissingValues handler = analyst.getScript().getNormalize().getMissingValues();
        double[] d = handler.handleMissing(analyst, stat);

        // should we skip the entire row
        if (d == null) {
          return null;
        }

        // copy the returned values in place of the missing values
        for (int i = 0; i < d.length; i++) {
          output[outputIndex++] = d[i];
        }
      } else {
        // known value
        if (stat.getAction() == NormalizationAction.Normalize) {
          double d = csv.getFormat().parse(str);
          d = stat.normalize(d);
          output[outputIndex++] = d;
        } else if (stat.getAction() == NormalizationAction.PassThrough) {
          double d = csv.getFormat().parse(str);
          output[outputIndex++] = d;
        } else {
          final double[] d = stat.encode(str);
          for (final double element : d) {
            output[outputIndex++] = element;
          }
        }
      }
    }

    return output;
  }