@Override
public AnalyticOutPut doAnalysis(AnalyticSource source) throws AnalysisException {
try {
DataSet dataSet = getDataSet((DataBaseAnalyticSource) source, source.getAnalyticConfig());
// dataSet.recalculateAllcolumnStatistics();
DatabaseConnection databaseConnection =
((DBTable) dataSet.getDBTable()).getDatabaseConnection();
PivotTableConfig config = (PivotTableConfig) source.getAnalyticConfig();
setInputSchema(((DataBaseAnalyticSource) source).getTableInfo().getSchema());
setInputTable(((DataBaseAnalyticSource) source).getTableInfo().getTableName());
setOutputType(config.getOutputType());
setOutputSchema(config.getOutputSchema());
setOutputTable(config.getOutputTable());
setDropIfExist(config.getDropIfExist());
columnNames = config.getPivotColumn();
groupColumn = config.getGroupByColumn();
aggColumn = config.getAggregateColumn();
aggrType = config.getAggregateType();
String dbType = databaseConnection.getProperties().getName();
if (config.getUseArray() != null && config.getUseArray().equalsIgnoreCase("true")) {
if (dbType.equals(DataSourceInfoDB2.dBType) || dbType.equals(DataSourceInfoNZ.dBType)) {
useArray = false;
} else {
useArray = true;
}
} else {
useArray = false;
}
generateStoragePrameterString((DataBaseAnalyticSource) source);
performOperation(databaseConnection, dataSet, config.getLocale());
DataBaseInfo dbInfo = ((DataBaseAnalyticSource) source).getDataBaseInfo();
AnalyzerOutPutTableObject outPut = getResultTableSampleRow(databaseConnection, dbInfo);
outPut.setAnalyticNodeMetaInfo(createNodeMetaInfo(config.getLocale()));
outPut.setDbInfo(dbInfo);
outPut.setSchemaName(getOutputSchema());
outPut.setTableName(getOutputTable());
return outPut;
} catch (Exception e) {
logger.error(e);
if (e instanceof WrongUsedException) {
throw new AnalysisError(this, (WrongUsedException) e);
} else if (e instanceof AnalysisError) {
throw (AnalysisError) e;
} else {
throw new AnalysisException(e);
}
}
}
public List<Stop> getStop(DataSet dataSet, boolean loadData) throws OperatorException {
List<Stop> result = new LinkedList<Stop>();
if (loadData) {
result.add(new ClassPureStop());
} else {
result.add(new DBPureStop());
}
result.add(new NoColumnStop());
result.add(new NoDataStop());
long maxDepth = para.getMaxDepth();
if (maxDepth <= 0) {
maxDepth = dataSet.size();
}
result.add(new DepthStop(maxDepth));
return result;
}
protected StringBuffer getNominalUpdate(
DataSet dataSet,
Column predictedLabel,
String newTableName,
int numberOfClasses,
StringBuffer[] classProbabilitiesSql,
StringBuffer caseSql,
StringBuffer[] biggerSql) {
StringBuffer sql = new StringBuffer();
caseSql.append(" (case ");
for (int c = 0; c < numberOfClasses - 1; c++) {
caseSql
.append(" when ")
.append(biggerSql[c])
.append(" then '")
.append(StringHandler.escQ(getLabel().getMapping().mapIndex(c)))
.append("'");
}
caseSql
.append(" else '")
.append(StringHandler.escQ(getLabel().getMapping().mapIndex(numberOfClasses - 1)))
.append("' end)");
sql.append(
" update " + newTableName + " set " + StringHandler.doubleQ(predictedLabel.getName()))
.append("=")
.append(caseSql);
for (int c = 0; c < numberOfClasses; c++) {
sql.append(
", "
+ StringHandler.doubleQ(
dataSet
.getColumns()
.getSpecial(
Column.CONFIDENCE_NAME + "_" + getLabel().getMapping().mapIndex(c))
.getName()))
.append(" = ")
.append(classProbabilitiesSql[c]);
}
sql.append(getWherePredict());
return sql;
}
protected ConstructTree getTB(DataSet dataSet) throws OperatorException {
if (dataSet.getColumns().getLabel().isNominal() || (para.isForWoe() == true)) {
if (para.isUseChiSquare() == true || para.isForWoe() == true) {
return new ConstructTree(
createStandard(false),
createStandard(true),
getStop(dataSet, false),
getStop(dataSet, true),
new DBBuildLeaf(),
new BuildLeaf(),
getPrune(false),
para.isNoPrePruning(),
para.getPrepruningAlternativesNumber(),
para.getSplitMinSize(),
para.getMinLeafSize(),
para.getThresholdLoadData(),
para.isUseChiSquare());
} else {
return new ConstructTree(
createStandard(false),
createStandard(true),
getStop(dataSet, false),
getStop(dataSet, true),
new DBBuildLeaf(),
new BuildLeaf(),
getPrune(false),
para.isNoPrePruning(),
para.getPrepruningAlternativesNumber(),
para.getSplitMinSize(),
para.getMinLeafSize(),
para.getThresholdLoadData());
}
}
return null;
}
@Override
protected Model train(AnalyticSource analyticSource) throws AnalysisException {
ResultSet rs = null;
Statement st = null;
try {
IDataSourceInfo dataSourceInfo =
DataSourceInfoFactory.createConnectionInfo(analyticSource.getDataSourceType());
dbtype = dataSourceInfo.getDBType();
RandomForestModel lastResult = null;
RandomForestIMP randomForestImpl = null;
// if (dbtype.equalsIgnoreCase(DataSourceInfoOracle.dBType)) {
// randomForestTrainer = new AdaboostOracle();
//
// } else
if (dbtype.equalsIgnoreCase(DataSourceInfoGreenplum.dBType)
|| dbtype.equalsIgnoreCase(DataSourceInfoPostgres.dBType)) {
randomForestImpl = new RandomForestGreenplum();
} else if (dbtype.equalsIgnoreCase(DataSourceInfoOracle.dBType)) {
randomForestImpl = new RandomForestOracle();
} else if (dbtype.equalsIgnoreCase(DataSourceInfoDB2.dBType)) {
randomForestImpl = new RandomForestDB2();
((RandomForestDB2) randomForestImpl)
.setConnection(((DataBaseAnalyticSource) analyticSource).getConnection());
} else if (dbtype.equalsIgnoreCase(DataSourceInfoNZ.dBType)) {
randomForestImpl = new RandomForestNZ();
} else {
throw new AnalysisException("Databse type is not supported for Random Forest:" + dbtype);
// return null;
}
try {
dataSet =
getDataSet((DataBaseAnalyticSource) analyticSource, analyticSource.getAnalyticConfig());
} catch (OperatorException e1) {
logger.error(e1);
throw new OperatorException(e1.getLocalizedMessage());
}
setSpecifyColumn(dataSet, analyticSource.getAnalyticConfig());
dataSet.computeAllColumnStatistics();
RandomForestConfig rfConfig = (RandomForestConfig) analyticSource.getAnalyticConfig();
String dbSystem = ((DataBaseAnalyticSource) analyticSource).getDataBaseInfo().getSystem();
String url = ((DataBaseAnalyticSource) analyticSource).getDataBaseInfo().getUrl();
String userName = ((DataBaseAnalyticSource) analyticSource).getDataBaseInfo().getUserName();
String password = ((DataBaseAnalyticSource) analyticSource).getDataBaseInfo().getPassword();
String inputSchema = ((DataBaseAnalyticSource) analyticSource).getTableInfo().getSchema();
String tableName = ((DataBaseAnalyticSource) analyticSource).getTableInfo().getTableName();
String useSSL = ((DataBaseAnalyticSource) analyticSource).getDataBaseInfo().getUseSSL();
String sampleWithReplacement = rfConfig.getSampleWithReplacement();
long timeStamp = System.currentTimeMillis();
pnewTable = "pnew" + timeStamp;
sampleTable = "s" + timeStamp;
String dependentColumn = rfConfig.getDependentColumn();
String columnNames = rfConfig.getColumnNames();
String[] totalColumns = columnNames.split(",");
int subSize = Integer.parseInt(rfConfig.getNodeColumnNumber());
int forestSize = Integer.parseInt(rfConfig.getForestSize());
Connection conncetion = null;
if (dbtype.equalsIgnoreCase(DataSourceInfoGreenplum.dBType)
|| dbtype.equalsIgnoreCase(DataSourceInfoPostgres.dBType)) {
lastResult = new RandomForestModelGreenplum(dataSet);
} else if (dbtype.equalsIgnoreCase(DataSourceInfoOracle.dBType)) {
lastResult = new RandomForestModelOracle(dataSet);
} else if (dbtype.equalsIgnoreCase(DataSourceInfoDB2.dBType)) {
lastResult = new RandomForestModelDB2(dataSet);
} else if (dbtype.equalsIgnoreCase(DataSourceInfoNZ.dBType)) {
lastResult = new RandomForestModelNZ(dataSet);
}
lastResult.setColumnNames(columnNames);
lastResult.setDependColumn(dependentColumn);
lastResult.setTableName(tableName);
conncetion = ((DataBaseAnalyticSource) analyticSource).getConnection();
Model result = null;
try {
st = conncetion.createStatement();
} catch (SQLException e) {
logger.error(e);
throw new AnalysisException(e);
}
// Iterator<String> dependvalueIterator = dataSet.getColumns()
// .getLabel().getMapping().getValues().iterator();
if (dataSet.getColumns().getLabel() instanceof NominalColumn) {
if (dataSet.getColumns().getLabel().getMapping().getValues().size() <= 1) {
String e =
SDKLanguagePack.getMessage(
SDKLanguagePack.ADABOOST_SAMPLE_ERRINFO, rfConfig.getLocale());
logger.error(e);
throw new AnalysisException(e);
}
if (dataSet.getColumns().getLabel().getMapping().getValues().size()
> AlpineMinerConfig.ADABOOST_MAX_DEPENDENT_COUNT) {
String e =
SDKLanguagePack.getMessage(
SDKLanguagePack.ADABOOST_MAX_DEPENDENT_COUNT_ERRINFO, rfConfig.getLocale());
logger.error(e);
throw new AnalysisException(e);
}
}
try {
randomForestImpl.randomForestTrainInit(
inputSchema, tableName, timeStamp, dependentColumn, st, dataSet);
} catch (SQLException e) {
logger.error(e);
throw new AnalysisException(e);
}
CartConfig config = new CartConfig();
config.setDependentColumn(dependentColumn);
config.setConfidence(rfConfig.getConfidence());
config.setMaximal_depth(rfConfig.getMaximal_depth());
config.setMinimal_leaf_size(rfConfig.getMinimal_leaf_size());
config.setMinimal_size_for_split(rfConfig.getMinimal_size_for_split());
config.setNo_pre_pruning("true");
config.setNo_pruning("true");
for (int i = 0; i < forestSize; i++) {
CartTrainer analyzer = new CartTrainer();
if (sampleWithReplacement == Resources.TrueOpt) {
randomForestImpl.randomForestSample(
inputSchema,
timeStamp + "" + i,
dependentColumn,
st,
rs,
pnewTable,
sampleTable + i,
rfConfig.getLocale());
} else {
randomForestImpl.randomForestSampleNoReplace(
inputSchema,
timeStamp + "" + i,
dependentColumn,
st,
rs,
pnewTable,
sampleTable + i,
rfConfig.getLocale(),
dataSet.size());
}
String subColumns = getSubColumns(totalColumns, subSize);
config.setColumnNames(subColumns);
DataBaseAnalyticSource tempsource =
new DataBaseAnalyticSource(
dbSystem, url, userName, password, inputSchema, sampleTable + i, useSSL);
tempsource.setAnalyticConfiguration(config);
tempsource.setConenction(conncetion);
result =
((AnalyzerOutPutTrainModel) analyzer.doAnalysis(tempsource))
.getEngineModel()
.getModel();
String OOBTable = "OOB" + sampleTable + i;
randomForestImpl.generateOOBTable(
inputSchema, OOBTable, pnewTable, sampleTable + i, st, rs);
DataBaseAnalyticSource tempPredictSource =
new DataBaseAnalyticSource(
dbSystem, url, userName, password, inputSchema, OOBTable, useSSL);
String predictOutTable = "OOBPredict" + sampleTable;
EngineModel em = new EngineModel();
em.setModel(result);
PredictorConfig tempconfig = new PredictorConfig(em);
tempconfig.setDropIfExist(dropIfExists);
tempconfig.setOutputSchema(inputSchema);
tempconfig.setOutputTable(predictOutTable);
tempPredictSource.setAnalyticConfiguration(tempconfig);
tempPredictSource.setConenction(conncetion);
AbstractDBModelPredictor predictor = new CartPredictor();
predictor.doAnalysis(tempPredictSource); // use the weak alg , do
double OOBError = 0.0;
if (result instanceof DecisionTreeModel) {
OOBError =
randomForestImpl.getOOBError(
tempPredictSource, dependentColumn, "P(" + dependentColumn + ")");
lastResult.getOobEstimateError().add(OOBError);
} else if (result instanceof RegressionTreeModel) {
OOBError = randomForestImpl.getMSE(tempPredictSource, "P(" + dependentColumn + ")");
lastResult.getOobLoss().add(OOBError);
double OOBMape =
randomForestImpl.getMAPE(
tempPredictSource, dependentColumn, "P(" + dependentColumn + ")");
lastResult.getOobMape().add(OOBMape);
} else {
OOBError = Double.NaN;
lastResult.getOobLoss().add(OOBError);
}
lastResult.addModel((SingleModel) result);
randomForestImpl.clearTrainResult(inputSchema, sampleTable + i);
randomForestImpl.clearTrainResult(inputSchema, predictOutTable);
randomForestImpl.clearTrainResult(inputSchema, OOBTable);
}
return lastResult;
} catch (Exception e) {
logger.error(e);
if (e instanceof WrongUsedException) {
throw new AnalysisError(this, (WrongUsedException) e);
} else if (e instanceof AnalysisError) {
throw (AnalysisError) e;
} else {
throw new AnalysisException(e);
}
} finally {
try {
if (st != null) {
st.close();
}
if (rs != null) {
rs.close();
}
} catch (SQLException e) {
logger.error(e);
throw new AnalysisException(e.getLocalizedMessage());
}
}
}
public Model train(DataSet dataSet, SVMParameter parameter) throws OperatorException {
para = parameter;
setDataSourceInfo(
DataSourceInfoFactory.createConnectionInfo(
((DBTable) dataSet.getDBTable()).getDatabaseConnection().getProperties().getName()));
DatabaseConnection databaseConnection =
((DBTable) dataSet.getDBTable()).getDatabaseConnection();
Column label = dataSet.getColumns().getLabel();
String labelString = StringHandler.doubleQ(label.getName());
DataSet newDataSet = getTransformer().TransformCategoryToNumeric_new(dataSet);
String newTableName = ((DBTable) newDataSet.getDBTable()).getTableName();
Statement st = null;
ResultSet rs = null;
try {
st = databaseConnection.createStatement(false);
} catch (SQLException e) {
e.printStackTrace();
throw new OperatorException(e.getLocalizedMessage());
}
StringBuffer ind = getColumnArray(newDataSet);
StringBuffer where = getColumnWhere(newDataSet);
where.append(" and ").append(labelString).append(" is not null ");
SVMRegressionModel model = new SVMRegressionModel(dataSet, newDataSet);
if (!newDataSet.equals(dataSet)) {
model.setAllTransformMap_valueKey(getTransformer().getAllTransformMap_valueKey());
}
model.setKernelType(para.getKernelType());
model.setDegree(para.getDegree());
model.setGamma(para.getGamma());
String sql =
"select (model).inds, (model).cum_err, (model).epsilon, (model).rho, (model).b, (model).nsvs, (model).ind_dim, (model).weights, (model).individuals from (select alpine_miner_online_sv_reg('"
+ newTableName
+ "','"
+ ind
+ "','"
+ labelString
+ "','"
+ where
+ "',"
+ para.getKernelType()
+ ","
+ para.getDegree()
+ ","
+ para.getGamma()
+ ","
+ para.getEta()
+ ","
+ ((SVMRegressionParameter) para).getSlambda()
+ ","
+ para.getNu()
+ ") as model ";
if (getDataSourceInfo().getDBType().equals(DataSourceInfoOracle.dBType)) {
sql += " from dual ";
}
sql += ") a";
try {
itsLogger.debug("SVMRegression.train():sql=" + sql);
rs = st.executeQuery(sql.toString());
setModel(rs, model);
if (getTransformer().isTransform()) {
dropTable(st, newTableName);
}
rs.close();
st.close();
} catch (SQLException e) {
e.printStackTrace();
throw new OperatorException(e.getLocalizedMessage());
}
return model;
}
protected void adjustPerWholeData(
DataSet dataSet,
List<String[]> hiddenLayers,
int maxCycles,
double maxError,
double learningRate,
double momentum,
boolean decay,
boolean normalize,
AlpineRandom randomGenerator,
int fetchSize,
boolean adjustPerRow,
Column label,
int numberOfClasses,
DatabaseConnection databaseConnection,
String tableName)
throws OperatorException {
double totalSize = dataSet.size();
Statement st = null;
ResultSet rs = null;
try {
st = databaseConnection.createStatement(true);
} catch (SQLException e) {
throw new OperatorException(e.getLocalizedMessage());
}
double error = 0;
// optimization loop
for (int cycle = 0; cycle < maxCycles; cycle++) {
double tempRate = learningRate;
if (decay) {
tempRate /= (cycle + 1);
}
StringBuffer sql = new StringBuffer();
sql.append("select ").append(getArraySum()).append("(");
// sql.append("select ").append("(");
sql.append(getAllWeightChange(getNumberOfClasses(label)));
sql.append(") from ").append(tableName);
try {
if (useFloatArraySumCursor()) {
StringBuffer varcharArray = CommonUtility.splitOracleSqlToVarcharArray(sql);
sql = new StringBuffer();
sql.append("select floatarraysum_cursor(").append(varcharArray).append(") from dual");
}
itsLogger.debug("NNModel.adjustPerWholeData():sql=" + sql);
rs = st.executeQuery(sql.toString());
if (rs.next()) {
Double[] currentChanges = getResult(rs);
// String currentChangesString= new String();
for (int i = 0; i < currentChanges.length; i++) {
if (Double.isNaN(currentChanges[i]) || Double.isInfinite(currentChanges[i])) {
try {
if (rs != null) {
rs.close();
}
if (st != null) {
st.close();
}
databaseConnection.getConnection().setAutoCommit(true);
} catch (SQLException e) {
e.printStackTrace();
// throw new OperatorException(e.getLocalizedMessage ());
copyBestErrorWeightsToWeights();
return;
}
copyBestErrorWeightsToWeights();
return;
}
}
error = currentChanges[currentChanges.length - 1] / numberOfClasses / totalSize;
if (error < bestError) {
bestError = error;
copyWeightsToBestErrorWeights();
}
updateWeight(currentChanges, tempRate, momentum);
}
} catch (SQLException e) {
e.printStackTrace();
copyBestErrorWeightsToWeights();
// throw new OperatorException(e.getLocalizedMessage());
return;
}
// error /= totalSize;
itsLogger.debug("cycle" + cycle + ";error:" + error);
if (error < maxError) {
itsLogger.debug("loop break : " + cycle + " error: " + error);
break;
}
if (Double.isInfinite(error) || Double.isNaN(error)) {
if (Tools.isLessEqual(learningRate, 0.0d)) // should hardly happen
throw new RuntimeException("Cannot reset network to a smaller learning rate.");
learningRate /= 2;
train(
dataSet,
hiddenLayers,
maxCycles,
maxError,
learningRate,
momentum,
decay,
normalize,
randomGenerator,
fetchSize,
adjustPerRow);
}
}
copyBestErrorWeightsToWeights();
try {
if (rs != null) {
rs.close();
}
if (st != null) {
st.close();
}
} catch (SQLException e) {
e.printStackTrace();
// throw new OperatorException(e.getLocalizedMessage ());
// copyBestErrorWeightsToWeights();
return;
}
}
/* (non-Javadoc)
* @see com.alpine.datamining.api.impl.db.AbstractDBModelTrainer#train(com.alpine.datamining.api.AnalyticSource)
*/
@Override
protected Model train(AnalyticSource source) throws AnalysisException {
ResultSet rs = null;
Statement st = null;
EMModel trainModel = null;
try {
IDataSourceInfo dataSourceInfo =
DataSourceInfoFactory.createConnectionInfo(source.getDataSourceType());
dbtype = dataSourceInfo.getDBType();
EMConfig config = (EMConfig) source.getAnalyticConfig();
String anaColumns = config.getColumnNames();
String[] columnsArray = anaColumns.split(",");
List<String> transformColumns = new ArrayList<String>();
for (int i = 0; i < columnsArray.length; i++) {
transformColumns.add(columnsArray[i]);
}
DataSet dataSet = getDataSet((DataBaseAnalyticSource) source, config);
filerColumens(dataSet, transformColumns);
dataSet.computeAllColumnStatistics();
ColumnTypeTransformer transformer = new ColumnTypeTransformer();
DataSet newDataSet = transformer.TransformCategoryToNumeric_new(dataSet);
String tableName = ((DBTable) newDataSet.getDBTable()).getTableName();
Columns columns = newDataSet.getColumns();
List<String> newTransformColumns = new ArrayList<String>();
HashMap<String, String> transformMap = new HashMap<String, String>();
for (String key : transformer.getAllTransformMap_valueKey().keySet()) {
HashMap<String, String> values = (transformer.getAllTransformMap_valueKey()).get(key);
for (String lowKey : values.keySet()) {
transformMap.put(values.get(lowKey), lowKey);
}
}
Iterator<Column> attributeIter = columns.iterator();
while (attributeIter.hasNext()) {
Column column = attributeIter.next();
newTransformColumns.add(column.getName());
}
int maxIterationNumber = Integer.parseInt(config.getMaxIterationNumber());
int clusterNumber = Integer.parseInt(config.getClusterNumber());
double epsilon = Double.parseDouble(config.getEpsilon());
int initClusterSize = 10;
if (config.getInitClusterSize() != null) {
initClusterSize = Integer.parseInt(config.getInitClusterSize());
}
if (newDataSet.size() < initClusterSize * clusterNumber) {
initClusterSize = (int) (newDataSet.size() / clusterNumber + 1);
} // TODO get it from config and make sure it will not be too large
EMClusterImpl emImpl = EMClusterFactory.createEMAnalyzer(dbtype);
trainModel = EMClusterFactory.createEMModel(dbtype, newDataSet);
Connection connection = null;
connection = ((DataBaseAnalyticSource) source).getConnection();
st = connection.createStatement();
ArrayList<Double> tempResult =
emImpl.emTrain(
connection,
st,
tableName,
maxIterationNumber,
epsilon,
clusterNumber,
newTransformColumns,
initClusterSize,
trainModel);
trainModel = generateEMModel(trainModel, newTransformColumns, clusterNumber, tempResult);
if (!newDataSet.equals(this.dataSet)) {
trainModel.setAllTransformMap_valueKey(transformMap);
}
} catch (Exception e) {
logger.error(e);
if (e instanceof WrongUsedException) {
throw new AnalysisError(this, (WrongUsedException) e);
} else if (e instanceof AnalysisError) {
throw (AnalysisError) e;
} else {
throw new AnalysisException(e);
}
} finally {
try {
if (st != null) {
st.close();
}
if (rs != null) {
rs.close();
}
} catch (SQLException e) {
logger.debug(e.toString());
throw new AnalysisException(e.getLocalizedMessage());
}
}
return trainModel;
}
@Override
protected AnalyzerOutPutObject doValidate(DataBaseAnalyticSource source, EvaluatorConfig config)
throws AnalysisException {
DoubleListData result = null;
List<DoubleListData> resultList = new ArrayList<DoubleListData>();
try {
DataSet dataSet = getDataSet(source, source.getAnalyticConfig());
// dataSet.recalculateAllcolumnStatistics();
Operator operator = OperatorUtil.createOperator(LiftDataGeneratorGeneral.class);
EvaluatorParameter parameter = new EvaluatorParameter();
if (config.getUseModel() != null) {
// operator.setParameter(LiftDataGeneratorGeneral.PARAMETER_USE_MODEL,
// config.getUseModel());
parameter.setUseModel(Boolean.parseBoolean(config.getUseModel()));
}
if (config.getColumnValue() != null) {
// operator.setParameter(LiftDataGeneratorGeneral.PARAMETER_TARGET_CLASS,
// config.getColumnValue());
parameter.setColumnValue(config.getColumnValue());
}
operator.setParameter(parameter);
if (config.getUseModel().equals("true")) {
List<EngineModel> models = config.getTrainedModel();
Model model = null;
for (int i = 0; i < models.size(); i++) {
Container container = new Container();
model = models.get(i).getModel();
container = container.append(dataSet);
container = container.append(model);
Container resultContainer = operator.apply(container);
result = resultContainer.get(DoubleListData.class);
result.setSourceName(models.get(i).getName());
resultList.add(result);
}
} else {
Container container = new Container();
String targetClass = config.getColumnValue();
Column confidence =
dataSet.getColumns().get(Column.CONFIDENCE_NAME + "(" + targetClass + ")");
dataSet
.getColumns()
.setSpecialColumn(confidence, Column.CONFIDENCE_NAME + "_" + targetClass);
container = container.append(dataSet);
Container resultContainer = operator.apply(container);
result = resultContainer.get(DoubleListData.class);
result.setSourceName(getName());
resultList.add(result);
}
} catch (Exception e) {
logger.error(e);
if (e instanceof WrongUsedException) {
throw new AnalysisError(this, (WrongUsedException) e);
} else if (e instanceof AnalysisError) {
throw (AnalysisError) e;
} else {
throw new AnalysisException(e);
}
}
AnalyzerOutPutObject out = new AnalyzerOutPutObject(resultList);
out.setAnalyticNodeMetaInfo(createNodeMetaInfo(config.getLocale()));
return out;
}
public Model learn(DataSet dataSet, LinearRegressionParameter para, String columnNames)
throws OperatorException {
this.dataSet = dataSet;
ArrayList<String> columnNamesList = new ArrayList<String>();
if (columnNames != null && !StringUtil.isEmpty(columnNames.trim())) {
String[] columnNamesArray = columnNames.split(",");
for (String s : columnNamesArray) {
columnNamesList.add(s);
}
}
transformer.setColumnNames(columnNamesList);
transformer.setAnalysisInterActionModel(para.getAnalysisInterActionModel());
newDataSet = transformer.TransformCategoryToNumeric_new(dataSet, groupbyColumn);
DatabaseConnection databaseConnection =
((DBTable) newDataSet.getDBTable()).getDatabaseConnection();
Column label = newDataSet.getColumns().getLabel();
String labelName = StringHandler.doubleQ(label.getName());
String tableName = ((DBTable) dataSet.getDBTable()).getTableName();
String newTableName = ((DBTable) newDataSet.getDBTable()).getTableName();
try {
st = databaseConnection.createStatement(false);
} catch (SQLException e) {
e.printStackTrace();
throw new OperatorException(e.getLocalizedMessage());
}
try {
newDataSet.computeAllColumnStatistics();
Columns atts = newDataSet.getColumns();
Iterator<Column> atts_i = atts.iterator();
int count = 0;
String[] columnNamesArray = new String[atts.size()];
while (atts_i.hasNext()) {
Column att = atts_i.next();
columnNamesArray[count] = att.getName();
count++;
}
null_list = calculateNull(dataSet);
null_list_group = calculateNullGroup(newDataSet, atts);
StringBuilder sb_notNull = getWhere(atts);
getCoefficientAndR2Group(
columnNames,
dataSet,
labelName,
tableName,
newTableName,
atts,
columnNamesArray,
sb_notNull);
HashMap<String, Long> degreeOfFreedom = new HashMap<String, Long>();
for (String groupValue : groupCount.keySet()) {
long tempDof = groupCount.get(groupValue) - columnNamesArray.length - 1;
if (tempDof <= 0) {
model.getOneModel(groupValue).setS(Double.NaN);
}
degreeOfFreedom.put(groupValue, tempDof);
}
// if (dof <= 0)
// {
// model.setS(Double.NaN);
// return model;
// }
StringBuffer sSQL =
createSSQLLGroup(
newDataSet, newTableName, label, columnNamesArray, coefficients, sb_notNull);
HashMap<String, Double> sValueMap = new HashMap<String, Double>();
try {
itsLogger.debug(classLogInfo + ".learn():sql=" + sSQL);
rs = st.executeQuery(sSQL.toString());
while (rs.next()) {
String groupValue = rs.getString(2);
if (groupValue == null) {
continue;
}
if (dataErrorList.contains(groupValue)) {
sValueMap.put(groupValue, Double.NaN);
} else {
sValueMap.put(groupValue, rs.getDouble(1));
}
}
rs.close();
} catch (SQLException e) {
e.printStackTrace();
itsLogger.error(e.getMessage(), e);
throw new OperatorException(e.getLocalizedMessage());
}
HashMap<String, Matrix> varianceCovarianceMatrix =
getVarianceCovarianceMatrixGroup(newTableName, columnNamesArray, st);
for (String groupValue : varianceCovarianceMatrix.keySet()) {
if (varianceCovarianceMatrix == null) {
model
.getOneModel(groupValue)
.setErrorString(
AlpineDataAnalysisLanguagePack.getMessage(
AlpineDataAnalysisLanguagePack.MATRIX_IS_SIGULAR,
AlpineThreadLocal.getLocale())
+ Tools.getLineSeparator());
}
}
caculateStatistics(
columnNamesArray,
coefficients,
model,
sValueMap,
varianceCovarianceMatrix,
degreeOfFreedom);
for (String groupValue : null_list_group.keySet()) {
if (null_list_group.get(groupValue).size() != 0) {
StringBuilder sb_null = new StringBuilder();
for (int i = 0; i < null_list_group.get(groupValue).size(); i++) {
sb_null
.append(StringHandler.doubleQ(null_list_group.get(groupValue).get(i)))
.append(",");
}
sb_null = sb_null.deleteCharAt(sb_null.length() - 1);
String table_exist_null =
AlpineDataAnalysisLanguagePack.getMessage(
AlpineDataAnalysisLanguagePack.TABLE_EXIST_NULL, AlpineThreadLocal.getLocale());
String[] temp = table_exist_null.split(";");
model
.getOneModel(groupValue)
.setErrorString(temp[0] + sb_null.toString() + temp[1] + Tools.getLineSeparator());
}
}
if (transformer.isTransform()) {
dropTable(newTableName);
}
st.close();
itsLogger.debug(LogUtils.exit(classLogInfo, "learn", model.toString()));
model.setGroupByColumn(groupbyColumn);
return model;
} catch (Exception e) {
itsLogger.error(e.getMessage(), e);
throw new OperatorException(e.getLocalizedMessage());
}
}
protected HashMap<String, ArrayList<String>> calculateNullGroup(DataSet dataSet, Columns atts)
throws OperatorException {
DatabaseConnection databaseConnection =
((DBTable) dataSet.getDBTable()).getDatabaseConnection();
String tableName = ((DBTable) dataSet.getDBTable()).getTableName();
HashMap<String, ArrayList<String>> resultList = new HashMap<String, ArrayList<String>>();
Iterator<Column> i = dataSet.getColumns().iterator();
StringBuilder sb_count = new StringBuilder("select ");
sb_count.append(StringHandler.doubleQ(groupbyColumn)).append(", count(*) ");
sb_count
.append(" from ")
.append(tableName)
.append(this.getWhere(atts))
.append(" group by ")
.append(StringHandler.doubleQ(groupbyColumn));
try {
Statement st = databaseConnection.createStatement(false);
itsLogger.debug("LinearRegressionImp.calculateNull():sql=" + sb_count.toString());
ResultSet rs = st.executeQuery(sb_count.toString());
while (rs.next()) {
String groupValue = "";
groupValue = rs.getString(1);
this.groupCount.put(groupValue, rs.getInt(2));
}
} catch (SQLException e) {
e.printStackTrace();
itsLogger.error(e.getMessage(), e);
throw new OperatorException(e.getLocalizedMessage());
}
sb_count =
new StringBuilder("select ").append(StringHandler.doubleQ(groupbyColumn)).append(", ");
while (i.hasNext()) {
Column att = i.next();
sb_count.append("count(").append(StringHandler.doubleQ(att.getName())).append(")");
sb_count.append(StringHandler.doubleQ(att.getName())).append(",");
}
sb_count = sb_count.deleteCharAt(sb_count.length() - 1);
sb_count
.append(" from ")
.append(tableName)
.append(this.getWhere(atts))
.append(" group by ")
.append(StringHandler.doubleQ(groupbyColumn));
try {
Statement st = databaseConnection.createStatement(false);
itsLogger.debug("LinearRegressionImp.calculateNull():sql=" + sb_count.toString());
ResultSet rs = st.executeQuery(sb_count.toString());
while (rs.next()) {
ArrayList<String> null_list = new ArrayList<String>();
String groupValue = "";
groupValue = rs.getString(1);
for (int j = 1; j < rs.getMetaData().getColumnCount(); j++) {
if (rs.getFloat(j + 1) != groupCount.get(groupValue)) {
null_list.add(
dataSet.getColumns().get(rs.getMetaData().getColumnName(j + 1)).getName());
}
}
resultList.put(groupValue, null_list);
}
} catch (SQLException e) {
e.printStackTrace();
itsLogger.error(e.getMessage(), e);
throw new OperatorException(e.getLocalizedMessage());
}
return resultList;
}
private void performOperation(
DatabaseConnection databaseConnection, DataSet dataSet, Locale locale)
throws AnalysisError, OperatorException {
String outputTableName = getQuotaedTableName(getOutputSchema(), getOutputTable());
String inputTableName = getQuotaedTableName(getInputSchema(), getInputTable());
Columns atts = dataSet.getColumns();
String dbType = databaseConnection.getProperties().getName();
IDataSourceInfo dataSourceInfo = DataSourceInfoFactory.createConnectionInfo(dbType);
IMultiDBUtility multiDBUtility = MultiDBUtilityFactory.createConnectionInfo(dbType);
ISqlGeneratorMultiDB sqlGenerator = SqlGeneratorMultiDBFactory.createConnectionInfo(dbType);
dropIfExist(dataSet);
DatabaseUtil.alterParallel(databaseConnection, getOutputType()); // for oracle
StringBuilder sb_create = new StringBuilder("create ");
StringBuilder insertTable = new StringBuilder();
if (getOutputType().equalsIgnoreCase("table")) {
sb_create.append(" table ");
} else {
sb_create.append(" view ");
}
sb_create.append(outputTableName);
sb_create.append(
getOutputType().equalsIgnoreCase(Resources.TableType) ? getAppendOnlyString() : "");
sb_create.append(DatabaseUtil.addParallel(databaseConnection, getOutputType())).append(" as (");
StringBuilder selectSql = new StringBuilder(" select ");
selectSql.append(StringHandler.doubleQ(groupColumn)).append(",");
Column att = atts.get(columnNames);
dataSet.computeColumnStatistics(att);
if (att.isNumerical()) {
logger.error("PivotTableAnalyzer cannot accept numeric type column");
throw new AnalysisError(
this,
AnalysisErrorName.Not_numeric,
locale,
SDKLanguagePack.getMessage(SDKLanguagePack.PIVOT_NAME, locale));
}
String attName = StringHandler.doubleQ(att.getName());
List<String> valueList = att.getMapping().getValues();
if (!useArray
&& valueList.size() > Integer.parseInt(AlpineMinerConfig.PIVOT_DISTINCTVALUE_THRESHOLD)) {
logger.error("Too many distinct value for column " + StringHandler.doubleQ(columnNames));
throw new AnalysisError(
this,
AnalysisErrorName.Too_Many_Distinct_value,
locale,
StringHandler.doubleQ(columnNames),
AlpineMinerConfig.PIVOT_DISTINCTVALUE_THRESHOLD);
}
if (valueList.size() <= 0) {
logger.error("Empty table");
throw new AnalysisError(this, AnalysisErrorName.Empty_table, locale);
}
String aggColumnName;
if (!StringUtil.isEmpty(aggColumn)) {
aggColumnName = StringHandler.doubleQ(aggColumn);
} else {
aggColumnName = "1";
}
Iterator<String> valueList_i = valueList.iterator();
if (useArray) {
if (dataSourceInfo.getDBType().equals(DataSourceInfoOracle.dBType)) {
ArrayList<String> array = new ArrayList<String>();
while (valueList_i.hasNext()) {
String value = StringHandler.escQ(valueList_i.next());
String newValue =
"alpine_miner_null_to_0("
+ aggrType
+ " (case when "
+ attName
+ "="
+ CommonUtility.quoteValue(dbType, att, value)
+ " then "
+ aggColumnName
+ " end )) ";
array.add(newValue);
}
selectSql.append(
CommonUtility.array2OracleArray(array, CommonUtility.OracleDataType.Float));
} else {
selectSql.append(multiDBUtility.floatArrayHead());
while (valueList_i.hasNext()) {
String value = valueList_i.next();
selectSql.append("alpine_miner_null_to_0(").append(aggrType);
selectSql.append(" (case when ").append(attName).append("=");
value = StringHandler.escQ(value);
selectSql
.append(CommonUtility.quoteValue(dbType, att, value))
.append(" then ")
.append(aggColumnName)
.append(" end )) "); // else 0
selectSql.append(",");
}
selectSql = selectSql.deleteCharAt(selectSql.length() - 1);
selectSql.append(multiDBUtility.floatArrayTail());
}
selectSql.append(" " + StringHandler.doubleQ(att.getName()));
} else {
if (((DBTable) dataSet.getDBTable())
.getDatabaseConnection()
.getProperties()
.getName()
.equals(DataSourceInfoNZ.dBType)) {
while (valueList_i.hasNext()) {
String value = valueList_i.next();
selectSql.append("(").append(aggrType);
selectSql.append(" (case when ").append(attName).append("=");
value = StringHandler.escQ(value);
selectSql
.append(CommonUtility.quoteValue(dbType, att, value))
.append(" then ")
.append(aggColumnName)
.append(" end )) "); // else 0
String colName = StringHandler.doubleQ(att.getName() + "_" + value);
selectSql.append(colName);
selectSql.append(",");
}
selectSql = selectSql.deleteCharAt(selectSql.length() - 1);
} else if (((DBTable) dataSet.getDBTable())
.getDatabaseConnection()
.getProperties()
.getName()
.equals(DataSourceInfoDB2.dBType)) {
while (valueList_i.hasNext()) {
String value = valueList_i.next();
selectSql.append("alpine_miner_null_to_0(").append(aggrType);
selectSql.append(" (double(case when ").append(attName).append("=");
value = StringHandler.escQ(value);
selectSql
.append(CommonUtility.quoteValue(dbType, att, value))
.append(" then ")
.append(aggColumnName)
.append(" end ))) "); // else 0
String colName = StringHandler.doubleQ(att.getName() + "_" + value);
selectSql.append(colName);
selectSql.append(",");
}
selectSql = selectSql.deleteCharAt(selectSql.length() - 1);
} else {
while (valueList_i.hasNext()) {
String value = valueList_i.next();
selectSql.append("alpine_miner_null_to_0(").append(aggrType);
selectSql.append(" (case when ").append(attName).append("=");
value = StringHandler.escQ(value);
selectSql
.append(CommonUtility.quoteValue(dbType, att, value))
.append(" then ")
.append(aggColumnName)
.append(" end )) "); // else 0
String colName = StringHandler.doubleQ(att.getName() + "_" + value);
selectSql.append(colName);
selectSql.append(",");
}
selectSql = selectSql.deleteCharAt(selectSql.length() - 1);
}
}
selectSql.append(" from ").append(inputTableName).append(" foo group by ");
selectSql.append(StringHandler.doubleQ(groupColumn));
if (((DBTable) dataSet.getDBTable())
.getDatabaseConnection()
.getProperties()
.getName()
.equals(DataSourceInfoNZ.dBType)) {
StringBuilder sb = new StringBuilder();
sb.append("select ").append(StringHandler.doubleQ(groupColumn)).append(",");
Iterator<String> valueList_new = valueList.iterator();
while (valueList_new.hasNext()) {
String value = valueList_new.next();
String colName = StringHandler.doubleQ(att.getName() + "_" + value);
sb.append("case when ").append(colName).append(" is null then 0 else ");
sb.append(colName).append(" end ").append(colName).append(",");
}
sb = sb.deleteCharAt(sb.length() - 1);
sb.append(" from (").append(selectSql).append(") foo ");
selectSql = sb;
}
sb_create.append(selectSql).append(" )");
if (getOutputType().equalsIgnoreCase("table")) {
sb_create.append(getEndingString());
insertTable.append(sqlGenerator.insertTable(selectSql.toString(), outputTableName));
}
try {
Statement st = databaseConnection.createStatement(false);
logger.debug("PivotTableAnalyzer.performOperation():sql=" + sb_create);
st.execute(sb_create.toString());
if (insertTable.length() > 0) {
st.execute(insertTable.toString());
logger.debug("PivotTableAnalyzer.performOperation():insertTableSql=" + insertTable);
}
} catch (SQLException e) {
logger.error(e);
if (e.getMessage().startsWith("ORA-03001")
|| e.getMessage().startsWith("ERROR: invalid identifier")) {
throw new AnalysisError(this, AnalysisErrorName.Invalid_Identifier, locale);
} else {
throw new OperatorException(e.getLocalizedMessage());
}
}
}