public static void copy(File source, File target) {
FileOutputStream fos = null;
InputStream is = null;
try {
fos = new FileOutputStream(target);
is = new FileInputStream(source);
copy(is, fos);
} catch (final IOException e) {
throw new EncogError(e);
} finally {
if (fos != null) {
try {
fos.close();
} catch (IOException e) {
EncogLogging.log(e);
}
}
if (is != null) {
try {
is.close();
} catch (IOException e) {
EncogLogging.log(e);
}
}
}
}
/** {@inheritDoc} */
@Override
public boolean executeCommand(final String args) {
// get filenames
final String sourceID =
getProp().getPropertyString(ScriptProperties.GENERATE_CONFIG_SOURCE_FILE);
final String targetID =
getProp().getPropertyString(ScriptProperties.GENERATE_CONFIG_TARGET_FILE);
final CSVFormat format = getAnalyst().getScript().determineFormat();
EncogLogging.log(EncogLogging.LEVEL_DEBUG, "Beginning generate");
EncogLogging.log(EncogLogging.LEVEL_DEBUG, "source file:" + sourceID);
EncogLogging.log(EncogLogging.LEVEL_DEBUG, "target file:" + targetID);
final File sourceFile = getScript().resolveFilename(sourceID);
final File targetFile = getScript().resolveFilename(targetID);
// mark generated
getScript().markGenerated(targetID);
// read file
final boolean headers = getScript().expectInputHeaders(sourceID);
final CSVHeaders headerList = new CSVHeaders(sourceFile, headers, format);
final int[] input = determineInputFields(headerList);
final int[] ideal = determineIdealFields(headerList);
EncogUtility.convertCSV2Binary(sourceFile, format, targetFile, input, ideal, headers);
return false;
}
/**
* Normalize the input file. Write to the specified file.
*
* @param file The file to write to.
*/
public void normalize(final File file) {
if (this.analyst == null) {
throw new EncogError("Can't normalize yet, file has not been analyzed.");
}
ReadCSV csv = null;
PrintWriter tw = null;
try {
csv = new ReadCSV(getInputFilename().toString(), isExpectInputHeaders(), getFormat());
tw = new PrintWriter(new FileWriter(file));
// write headers, if needed
if (isProduceOutputHeaders()) {
writeHeaders(tw);
}
resetStatus();
final int outputLength = this.analyst.determineTotalColumns();
// write file contents
while (csv.next() && !shouldStop()) {
updateStatus(false);
double[] output =
AnalystNormalizeCSV.extractFields(
this.analyst, this.analystHeaders, csv, outputLength, false);
if (this.series.getTotalDepth() > 1) {
output = this.series.process(output);
}
if (output != null) {
final StringBuilder line = new StringBuilder();
NumberList.toList(getFormat(), line, output);
tw.println(line);
}
}
} catch (final IOException e) {
throw new QuantError(e);
} finally {
reportDone(false);
if (csv != null) {
try {
csv.close();
} catch (final Exception ex) {
EncogLogging.log(ex);
}
}
if (tw != null) {
try {
tw.close();
} catch (final Exception ex) {
EncogLogging.log(ex);
}
}
}
}
/** {@inheritDoc} */
@Override
public final boolean executeCommand(final String args) {
// get filenames
final String evalID = getProp().getPropertyString(ScriptProperties.ML_CONFIG_EVAL_FILE);
final String resourceID =
getProp().getPropertyString(ScriptProperties.ML_CONFIG_MACHINE_LEARNING_FILE);
final String outputID = getProp().getPropertyString(ScriptProperties.ML_CONFIG_OUTPUT_FILE);
EncogLogging.log(EncogLogging.LEVEL_DEBUG, "Beginning evaluate raw");
EncogLogging.log(EncogLogging.LEVEL_DEBUG, "evaluate file:" + evalID);
EncogLogging.log(EncogLogging.LEVEL_DEBUG, "resource file:" + resourceID);
final File evalFile = getScript().resolveFilename(evalID);
final File resourceFile = getScript().resolveFilename(resourceID);
final File outputFile = getAnalyst().getScript().resolveFilename(outputID);
final MLRegression method = (MLRegression) EncogDirectoryPersistence.loadObject(resourceFile);
final boolean headers = getScript().expectInputHeaders(evalID);
final AnalystEvaluateRawCSV eval = new AnalystEvaluateRawCSV();
eval.setScript(getScript());
getAnalyst().setCurrentQuantTask(eval);
eval.setReport(new AnalystReportBridge(getAnalyst()));
eval.analyze(
getAnalyst(),
evalFile,
headers,
getProp().getPropertyCSVFormat(ScriptProperties.SETUP_CONFIG_CSV_FORMAT));
eval.process(outputFile, method);
getAnalyst().setCurrentQuantTask(null);
return eval.shouldStop();
}
/**
* Navigate to a page based on a URL object. This will be an HTTP GET operation.
*
* @param url The URL to navigate to.
*/
public final void navigate(final URL url) {
try {
EncogLogging.log(EncogLogging.LEVEL_INFO, "Navigating to page:" + url);
final URLConnection connection = url.openConnection();
final InputStream is = connection.getInputStream();
navigate(url, is);
is.close();
} catch (final IOException e) {
EncogLogging.log(EncogLogging.LEVEL_ERROR, e);
throw new BrowseError(e);
}
}
/**
* Navigate based on a string URL.
*
* @param url The URL to navigate to.
*/
public final void navigate(final String url) {
try {
navigate(new URL(url));
} catch (final MalformedURLException e) {
EncogLogging.log(EncogLogging.LEVEL_ERROR, e);
throw new BrowseError(e);
}
}
/**
* Navigate based on a form. Complete and post the form.
*
* @param form The form to be posted.
* @param submit The submit button on the form to simulate clicking.
*/
public final void navigate(final Form form, final Input submit) {
try {
EncogLogging.log(EncogLogging.LEVEL_INFO, "Posting a form");
InputStream is;
URLConnection connection = null;
OutputStream os;
URL targetURL;
if (form.getMethod() == Form.Method.GET) {
os = new ByteArrayOutputStream();
} else {
connection = form.getAction().getUrl().openConnection();
os = connection.getOutputStream();
}
// add the parameters if present
final FormUtility formData = new FormUtility(os, null);
for (final DocumentRange dr : form.getElements()) {
if (dr instanceof FormElement) {
final FormElement element = (FormElement) dr;
if ((element == submit) || element.isAutoSend()) {
final String name = element.getName();
String value = element.getValue();
if (name != null) {
if (value == null) {
value = "";
}
formData.add(name, value);
}
}
}
}
// now execute the command
if (form.getMethod() == Form.Method.GET) {
String action = form.getAction().getUrl().toString();
os.close();
action += "?";
action += os.toString();
targetURL = new URL(action);
connection = targetURL.openConnection();
is = connection.getInputStream();
} else {
targetURL = form.getAction().getUrl();
os.close();
is = connection.getInputStream();
}
navigate(targetURL, is);
is.close();
} catch (final IOException e) {
throw new BrowseError(e);
}
}
/**
* Initialize this strategy.
*
* @param train The training algorithm.
*/
public final void init(final MLTrain train) {
this.train = train;
this.ready = false;
this.setter = (LearningRate) train;
this.trainingSize = train.getTraining().getRecordCount();
this.currentLearningRate = 1.0 / this.trainingSize;
EncogLogging.log(
EncogLogging.LEVEL_DEBUG, "Starting learning rate: " + this.currentLearningRate);
this.setter.setLearningRate(this.currentLearningRate);
}
/**
* Load the web page from a string that contains HTML.
*
* @param str A string containing HTML.
* @return The loaded WebPage.
*/
public final WebPage load(final String str) {
try {
final ByteArrayInputStream bis = new ByteArrayInputStream(str.getBytes());
final WebPage result = load(bis);
bis.close();
return result;
} catch (final IOException e) {
EncogLogging.log(e);
throw new BrowseError(e);
}
}
/** Called just after a training iteration. */
public final void postIteration() {
if (this.ready) {
if (this.train.getError() > this.lastError) {
this.currentLearningRate *= SmartLearningRate.LEARNING_DECAY;
this.setter.setLearningRate(this.currentLearningRate);
EncogLogging.log(
EncogLogging.LEVEL_DEBUG, "Adjusting learning rate to {}" + this.currentLearningRate);
}
} else {
this.ready = true;
}
}
/** Called just before a training iteration. */
public void preIteration() {
if (train.getError() > this.acceptableThreshold) {
if (!Double.isNaN(lastError)) {
double improve = (lastError - train.getError());
if (improve < this.required) {
this.badCycleCount++;
if (this.badCycleCount > this.cycles) {
EncogLogging.log(EncogLogging.LEVEL_DEBUG, "Failed to improve network, resetting.");
this.method.reset();
this.badCycleCount = 0;
this.lastError = Double.NaN;
}
} else {
this.badCycleCount = 0;
}
} else lastError = train.getError();
}
lastError = Math.min(this.train.getError(), lastError);
}
/**
* Navigate to a page and post the specified data.
*
* @param url The URL to post the data to.
* @param is The data to post to the page.
*/
public final void navigate(final URL url, final InputStream is) {
EncogLogging.log(EncogLogging.LEVEL_INFO, "POSTing to page:" + url);
final LoadWebPage load = new LoadWebPage(url);
this.currentPage = load.load(is);
}
/** {@inheritDoc} */
@Override
public final void prepareWrite(final int recordCount, final int inputSize, final int idealSize) {
this.inputCount = inputSize;
this.idealCount = idealSize;
ZipInputStream zis = null;
try {
this.fos = new FileOutputStream(this.file);
this.zos = new ZipOutputStream(this.fos);
final InputStream is =
ResourceInputStream.openResourceInputStream("org/encog/data/blank.xlsx");
zis = new ZipInputStream(is);
ZipEntry theEntry;
while (zis.available() > 0) {
theEntry = zis.getNextEntry();
if ((entry != null) && !"xl/worksheets/sheet1.xml".equals(entry.getName())) {
final ZipEntry entry2 = new ZipEntry(theEntry);
entry2.setCompressedSize(-1);
this.zos.putNextEntry(entry2);
final byte[] theBuffer = new byte[(int) entry.getSize()];
zis.read(theBuffer);
this.zos.write(theBuffer);
this.zos.closeEntry();
}
}
zis.close();
zis = null;
this.buffer = new ByteArrayOutputStream();
this.xmlOut = new WriteXML(this.buffer);
this.xmlOut.beginDocument();
this.xmlOut.addAttribute(
"xmlns", "http://schemas.openxmlformats.org/spreadsheetml/2006/main");
this.xmlOut.addAttribute(
"xmlns:r", "http://schemas.openxmlformats.org/officeDocument/2006/relationships");
this.xmlOut.beginTag("worksheet");
final StringBuilder d = new StringBuilder();
d.append(toColumn(this.inputCount + this.idealCount));
d.append("" + recordCount);
this.xmlOut.addAttribute("ref", "A1:" + d.toString());
this.xmlOut.beginTag("dimension");
this.xmlOut.endTag();
this.xmlOut.beginTag("sheetViews");
this.xmlOut.addAttribute("tabSelected", "1");
this.xmlOut.addAttribute("workbookViewId", "0");
this.xmlOut.beginTag("sheetView");
this.xmlOut.endTag();
this.xmlOut.endTag();
this.xmlOut.addAttribute("defaultRowHeight", "15");
this.xmlOut.beginTag("sheetFormatPtr");
this.xmlOut.endTag();
this.row = 1;
this.xmlOut.beginTag("sheetData");
} catch (final IOException ex) {
throw new BufferedDataError(ex);
} finally {
if (zis != null) {
try {
zis.close();
} catch (IOException e) {
EncogLogging.log(e);
}
}
}
}
/**
* Construct an exception that holds another exception.
*
* @param msg A message.
* @param t The other exception.
*/
public EncogError(final String msg, final Throwable t) {
super(msg, t);
EncogLogging.log(EncogLogging.LEVEL_ERROR, msg);
EncogLogging.log(EncogLogging.LEVEL_ERROR, t);
}
/**
* Construct an exception that holds another exception.
*
* @param t The other exception.
*/
public EncogError(final Throwable t) {
super(t);
EncogLogging.log(EncogLogging.LEVEL_ERROR, t);
}
/**
* Construct a message exception.
*
* @param msg The exception message.
*/
public EncogError(final String msg) {
super(msg);
EncogLogging.log(EncogLogging.LEVEL_ERROR, msg);
}