/**
* Extract fields from a file into a numeric array for machine learning.
*
* @param analyst The analyst to use.
* @param headers The headers for the input data.
* @param csv The CSV that holds the input data.
* @param outputLength The length of the returned array.
* @param skipOutput True if the output should be skipped.
* @return The encoded data.
*/
public static final double[] extractFields(
final EncogAnalyst analyst,
final CSVHeaders headers,
final ReadCSV csv,
final int outputLength,
final boolean skipOutput) {
final double[] output = new double[outputLength];
int outputIndex = 0;
for (final AnalystField stat : analyst.getScript().getNormalize().getNormalizedFields()) {
stat.init();
if (stat.getAction() == NormalizationAction.Ignore) {
continue;
}
if (stat.isOutput() && skipOutput) {
continue;
}
int index = headers.find(stat.getName());
final String str = csv.get(index).trim();
// is this an unknown value?
if (str.equals("?") || str.length() == 0) {
HandleMissingValues handler = analyst.getScript().getNormalize().getMissingValues();
double[] d = handler.handleMissing(analyst, stat);
// should we skip the entire row
if (d == null) {
return null;
}
// copy the returned values in place of the missing values
for (int i = 0; i < d.length; i++) {
output[outputIndex++] = d[i];
}
} else {
// known value
if (stat.getAction() == NormalizationAction.Normalize) {
double d = csv.getFormat().parse(str);
d = stat.normalize(d);
output[outputIndex++] = d;
} else if (stat.getAction() == NormalizationAction.PassThrough) {
double d = csv.getFormat().parse(str);
output[outputIndex++] = d;
} else {
final double[] d = stat.encode(str);
for (final double element : d) {
output[outputIndex++] = element;
}
}
}
}
return output;
}
/**
* Analyze the file.
*
* @param inputFilename The input file.
* @param expectInputHeaders True, if input headers are present.
* @param inputFormat The format.
* @param theAnalyst The analyst to use.
*/
public void analyze(
final File inputFilename,
final boolean expectInputHeaders,
final CSVFormat inputFormat,
final EncogAnalyst theAnalyst) {
this.setInputFilename(inputFilename);
this.setInputFormat(inputFormat);
this.setExpectInputHeaders(expectInputHeaders);
this.analyst = theAnalyst;
this.setAnalyzed(true);
this.analystHeaders = new CSVHeaders(inputFilename, expectInputHeaders, inputFormat);
for (final AnalystField field : analyst.getScript().getNormalize().getNormalizedFields()) {
field.init();
}
this.series = new TimeSeriesUtil(analyst, true, this.analystHeaders.getHeaders());
}