@Override
public void updateConfig(XMLConfiguration config) {
nsTableId = config.getInt("storage_table", 511);
dataExpireTime = config.getInt("data_expiretime", 7 * 24 * 3600);
topNum = config.getInt("top_num", 30);
debug = config.getBoolean("debug", false);
}
@Override
public void loadFromXML(Reader in) {
XMLConfiguration xml = ConfigurationUtil.newXMLConfiguration(in);
setSourceReferenceField(xml.getString("sourceReferenceField", sourceReferenceField));
setKeepSourceReferenceField(
xml.getBoolean("sourceReferenceField[@keep]", keepSourceReferenceField));
setTargetReferenceField(xml.getString("targetReferenceField", targetReferenceField));
setSourceContentField(xml.getString("sourceContentField", sourceContentField));
setKeepSourceContentField(xml.getBoolean("sourceContentField[@keep]", keepSourceContentField));
setTargetContentField(xml.getString("targetContentField", targetContentField));
setQueueDir(xml.getString("queueDir", DEFAULT_QUEUE_DIR));
setQueueSize(xml.getInt("queueSize", AbstractBatchCommitter.DEFAULT_QUEUE_SIZE));
setCommitBatchSize(
xml.getInt("commitBatchSize", AbstractBatchCommitter.DEFAULT_COMMIT_BATCH_SIZE));
setMaxRetries(xml.getInt("maxRetries", 0));
setMaxRetryWait(xml.getInt("maxRetryWait", 0));
loadFromXml(xml);
}
public static boolean loadConfig(String configFilePath) {
String tmp = System.getProperty("os.name").toLowerCase();
if (tmp.startsWith("windows")) {
g.os = g.OS_WINDOWS;
} else if (tmp.startsWith("linux")) {
g.os = g.OS_LINUX;
} else {
g.os = g.OS_OTHER;
}
tmp = System.getProperty("sun.arch.data.model");
if (tmp.equals("64")) {
g.arch = g.ARCH_64;
} else {
g.arch = g.ARCH_32;
}
try {
if (configFilePath == null) {
config = new XMLConfiguration(configPath + "config.xml");
} else {
config = new XMLConfiguration(configFilePath);
}
dbConfig = config.getString("db", "").toLowerCase();
siteUrl = config.getString("serverInfo.siteUrl");
siteName = config.getString("serverInfo.siteName");
uploadTemp = rootPath + config.getString("serverInfo.uploadTemp");
uploadSizeMax = config.getLong("serverInfo.uploadSizeMax", 0);
sessionExpiredTime = config.getInt("serverInfo.sessionExpiredTime", 0);
sessionIdName = config.getString("serverInfo.sessionIdName", "ycsid");
String _s = g.getConfig().getString("startup");
if ("".equals(_s)) {
startup = null;
} else {
_s = _s.replaceAll("\\s", "");
startup = _s.split(";");
}
_s = null;
// 加载全局参数
List<HierarchicalConfiguration> fields = config.configurationsAt("globalVars.var");
if (fields != null && fields.size() > 0) {
vars = new String[fields.size()][2];
HierarchicalConfiguration sub;
for (int i = 0; i < fields.size(); i++) {
sub = fields.get(i);
vars[i][0] = sub.getString("[@name]");
vars[i][1] = sub.getString("[@value]");
}
sub = null;
}
return true;
} catch (ConfigurationException e) {
return false;
}
}
/**
* The constructor is {@code private} to have strict control what instances exist at any time.
* Instead of the constructor the {@code public} <i>static factory method</i> {@link
* #getInstance(XMLConfiguration, CoreService, URL, URL, DefaultHttpClient)} returns the instances
* of the class.
*
* @param xmlConfiguration The {@code XMLConfiguration} object (of the {@code StorageService})
* @param coreService The corresponding {@code CoreService}
* @param apiBaseURL The base {@code URL} of the API
* @param serviceURL The service {@code URL}
* @param httpClient The <i>Apache</i> {@code DefaultHttpClient}
*/
private DownloadManager(
XMLConfiguration xmlConfiguration,
CoreService coreService,
URL apiBaseURL,
URL serviceURL,
DefaultHttpClient httpClient) {
this.coreService = coreService;
this.apiBaseURL = apiBaseURL;
this.serviceURL = serviceURL;
this.httpClient = httpClient;
executorService =
Executors.newFixedThreadPool(xmlConfiguration.getInt("download/max-parallel-downloads"));
}
@SuppressWarnings("unchecked")
public BasicDataSource getDataSource(String dataSourceName, String dbName)
throws SecurityException, ClassNotFoundException, IllegalArgumentException,
InstantiationException, IllegalAccessException, InvocationTargetException {
Configuration dbConf = confDbs.configurationAt(dbName);
String dbClass = dbConf.getString("class");
Constructor[] a = Class.forName(dbClass).getConstructors();
BasicDataSource ds = (BasicDataSource) a[0].newInstance();
ds.configDriver(confDbs, dbName);
ds.configDataSource(confDatasources, dataSourceName);
if (confDatasources.getProperty(dataSourceName + ".fillRandom") != null) {
int fillRandomValues = confDatasources.getInt(dataSourceName + ".fillRandom");
if (fillRandomValues == 1) ds.fillRandomValues();
}
ds.setName(dataSourceName + dbName);
return ds;
}
@SuppressWarnings("unchecked")
public void test() {
try {
confMonoTests.setFileName("confMonoTests.xml");
confDbs.setFileName("confDbs.xml");
confDatasources.setFileName("confDatasources.xml");
confMonoTests.load();
confDbs.load();
confDatasources.load();
List<String> runs = confMonoTests.getList("run");
log.info("Start of testing");
// runs = new TestRun[confRuns.getInt("threads")];
int numRuns = confMonoTests.getInt("threads");
String resultsFile = "";
semRuns = new Semaphore(numRuns);
// Iterate through runs
for (String run : runs) {
int dbId = 0;
run = "runs." + run;
// Iterate through dbs
while (confMonoTests.getProperty(run + ".dbs.db(" + dbId + ").name") != null) {
String dbName = confMonoTests.getString(run + ".dbs.db(" + dbId + ").name");
List<String> datasources =
confMonoTests.getList(run + ".dbs.db(" + dbId + ").datasources");
int datasourceId = 0;
// Iterate through datasources
for (String datasourceName : datasources) {
int methodId = 0;
// Load possible normalizers
List<String> numericalNorms =
confMonoTests.getList(run + ".normalizers.numericalNormalizer");
List<String> nominalNormalizers =
confMonoTests.getList(run + ".normalizers.nominalNormalizer");
List<String> listNormalizer =
confMonoTests.getList(run + ".normalizers.listNormalizer");
// Iterate through tests
if (confMonoTests.getProperty(run + ".test.path") != null) {
synchronized (semRuns) {
// Waiting for a free slot;
semRuns.acquire();
System.gc();
BasicDataSource trainDataSource = getDataSource(datasourceName, dbName);
trainDataSource.configDriver(confMonoTests, run + ".dbs.db(" + dbId + ")");
trainDataSource.configDataSource(
confMonoTests, run + ".dbs.db(" + dbId + ").datasources(" + datasourceId + ")");
BasicDataSource testDataSource = getDataSource(datasourceName, dbName);
testDataSource.configDriver(confMonoTests, run + ".dbs.db(" + dbId + ")");
testDataSource.configDataSource(
confMonoTests, run + ".dbs.db(" + dbId + ").datasources(" + datasourceId + ")");
MonotonicityTest mt = new MonotonicityTest();
mt.configTest(confMonoTests, run + ".test");
log.info("Testing datasource " + trainDataSource.getName());
// Making new thread for a new test.
new TestRun(
this,
mt,
trainDataSource,
testDataSource,
getNorms(nominalNormalizers),
getNorms(numericalNorms),
getNorms(listNormalizer))
.start();
// Wait 1 second for avoid of the colision
// in writing of results.
Thread.sleep(1000);
}
}
datasourceId++;
}
dbId++;
}
}
// Waiting for all other threads to finish.
for (int i = 0; i < numRuns; i++) semRuns.acquire();
log.info("End of testing");
Process p = Runtime.getRuntime().exec("cmd ", null, null);
p.getOutputStream().write(("loadResults.bat " + resultsFile + "\n").getBytes());
p.getOutputStream().flush();
p.getOutputStream().write("exit\n".getBytes());
p.getOutputStream().flush();
BufferedReader stdOut = new BufferedReader(new InputStreamReader(p.getInputStream()));
BufferedReader stdErr = new BufferedReader(new InputStreamReader(p.getErrorStream()));
while (true) {
try {
p.exitValue();
break;
} catch (Exception e) {
while (stdOut.ready() || stdErr.ready()) {
if (stdOut.ready()) stdOut.readLine();
else stdErr.readLine();
}
}
}
p.waitFor();
} catch (Exception e) {
e.printStackTrace();
}
}
public static int getInt(String key, int defaultValue) {
return pc.getInt(key, defaultValue);
}
@SuppressWarnings("unchecked")
public void test() {
try {
confRuns.setFileName("confRuns.xml");
confDbs.setFileName("confDbs.xml");
confDatasources.setFileName("confDatasources.xml");
confMethods.setFileName("confMethods.xml");
confRuns.load();
confDbs.load();
confDatasources.load();
confMethods.load();
List<String> runs = confRuns.getList("run");
log.info("Start of testing");
// runs = new TestRun[confRuns.getInt("threads")];
int numRuns = confRuns.getInt("threads");
String resultsFile = "";
semRuns = new Semaphore(numRuns);
// Iterate through runs
for (String run : runs) {
int dbId = 0;
run = "runs." + run;
// Iterate through dbs
while (confRuns.getProperty(run + ".dbs.db(" + dbId + ").name") != null) {
String dbName = confRuns.getString(run + ".dbs.db(" + dbId + ").name");
List<String> datasources = confRuns.getList(run + ".dbs.db(" + dbId + ").datasources");
int datasourceId = 0;
// Iterate through datasources
for (String datasourceName : datasources) {
int methodId = 0;
// Iterate through methods
while (confRuns.getProperty(run + ".methods.method(" + methodId + ").name") != null) {
String methodName =
confRuns.getString(run + ".methods.method(" + methodId + ").name");
int testId = 0;
// Iterate through tests
while (confRuns.getProperty(run + ".tests.test(" + testId + ").class") != null) {
synchronized (semRuns) {
// Waiting for a free slot;
semRuns.acquire();
String testClass = confRuns.getString(run + ".tests.test(" + testId + ").class");
Constructor[] a = Class.forName(testClass).getConstructors();
log.info("Datasource " + datasourceName + ", method " + methodName);
System.gc();
BasicDataSource trainDataSource = getDataSource(datasourceName, dbName);
trainDataSource.configDriver(confRuns, run + ".dbs.db(" + dbId + ")");
trainDataSource.configDataSource(
confRuns, run + ".dbs.db(" + dbId + ").datasources(" + datasourceId + ")");
BasicDataSource testDataSource = getDataSource(datasourceName, dbName);
testDataSource.configDriver(confRuns, run + ".dbs.db(" + dbId + ")");
testDataSource.configDataSource(
confRuns, run + ".dbs.db(" + dbId + ").datasources(" + datasourceId + ")");
// ExportAsTHSource.exportAsTHSource(trainDataSource, "C:\\data\\",
// trainDataSource.getName());
// writeRecords(trainDataSource);
// writeRecords(testDataSource);
// DataSourceStatistics b = new DataSourceStatistics(trainDataSource.getName());
// b.getStatistics(trainDataSource);
// semRuns.release();
InductiveMethod method = getMethod(methodName);
method.configClassifier(confRuns, run + ".methods.method(" + methodId + ")");
Test test = (Test) a[0].newInstance();
test.configTest(confRuns, run + ".tests.test(" + testId + ")");
resultsFile = test.getResultsInterpreter().getFilePrefix() + ".csv";
log.info("Testing datasource " + trainDataSource.getName());
// Making new thread for a new test.
new TestRun(this, trainDataSource, testDataSource, method, test).start();
// Wait 1 second for avoid of the colision
// in writing of results.
Thread.sleep(1000);
}
testId++;
}
methodId++;
}
datasourceId++;
}
dbId++;
}
}
// Waiting for all other threads to finish.
for (int i = 0; i < numRuns; i++) semRuns.acquire();
log.info("End of testing");
Process p = Runtime.getRuntime().exec("cmd ", null, null);
p.getOutputStream().write(("loadResults.bat " + resultsFile + "\n").getBytes());
p.getOutputStream().flush();
p.getOutputStream().write("exit\n".getBytes());
p.getOutputStream().flush();
BufferedReader stdOut = new BufferedReader(new InputStreamReader(p.getInputStream()));
BufferedReader stdErr = new BufferedReader(new InputStreamReader(p.getErrorStream()));
while (true) {
try {
p.exitValue();
break;
} catch (Exception e) {
while (stdOut.ready() || stdErr.ready()) {
if (stdOut.ready()) stdOut.readLine();
else stdErr.readLine();
}
}
}
p.waitFor();
} catch (Exception e) {
e.printStackTrace();
}
}
