/**
* This method builds a decision tree model
*
* @param sparkContext JavaSparkContext initialized with the application
* @param modelID Model ID
* @param trainingData Training data as a JavaRDD of LabeledPoints
* @param testingData Testing data as a JavaRDD of LabeledPoints
* @param workflow Machine learning workflow
* @param mlModel Deployable machine learning model
* @throws MLModelBuilderException
*/
private ModelSummary buildDecisionTreeModel(
JavaSparkContext sparkContext,
long modelID,
JavaRDD<LabeledPoint> trainingData,
JavaRDD<LabeledPoint> testingData,
Workflow workflow,
MLModel mlModel,
SortedMap<Integer, String> includedFeatures,
Map<Integer, Integer> categoricalFeatureInfo)
throws MLModelBuilderException {
try {
Map<String, String> hyperParameters = workflow.getHyperParameters();
DecisionTree decisionTree = new DecisionTree();
DecisionTreeModel decisionTreeModel =
decisionTree.train(
trainingData,
getNoOfClasses(mlModel),
categoricalFeatureInfo,
hyperParameters.get(MLConstants.IMPURITY),
Integer.parseInt(hyperParameters.get(MLConstants.MAX_DEPTH)),
Integer.parseInt(hyperParameters.get(MLConstants.MAX_BINS)));
// remove from cache
trainingData.unpersist();
// add test data to cache
testingData.cache();
JavaPairRDD<Double, Double> predictionsAndLabels =
decisionTree.test(decisionTreeModel, testingData).cache();
ClassClassificationAndRegressionModelSummary classClassificationAndRegressionModelSummary =
SparkModelUtils.getClassClassificationModelSummary(
sparkContext, testingData, predictionsAndLabels);
// remove from cache
testingData.unpersist();
mlModel.setModel(new MLDecisionTreeModel(decisionTreeModel));
classClassificationAndRegressionModelSummary.setFeatures(
includedFeatures.values().toArray(new String[0]));
classClassificationAndRegressionModelSummary.setAlgorithm(
SUPERVISED_ALGORITHM.DECISION_TREE.toString());
MulticlassMetrics multiclassMetrics =
getMulticlassMetrics(sparkContext, predictionsAndLabels);
predictionsAndLabels.unpersist();
classClassificationAndRegressionModelSummary.setMulticlassConfusionMatrix(
getMulticlassConfusionMatrix(multiclassMetrics, mlModel));
Double modelAccuracy = getModelAccuracy(multiclassMetrics);
classClassificationAndRegressionModelSummary.setModelAccuracy(modelAccuracy);
classClassificationAndRegressionModelSummary.setDatasetVersion(workflow.getDatasetVersion());
return classClassificationAndRegressionModelSummary;
} catch (Exception e) {
throw new MLModelBuilderException(
"An error occurred while building decision tree model: " + e.getMessage(), e);
}
}
private Map<Integer, Integer> getCategoricalFeatureInfo(List<Map<String, Integer>> encodings) {
Map<Integer, Integer> info = new HashMap<Integer, Integer>();
// skip the response variable which is at last
for (int i = 0; i < encodings.size() - 1; i++) {
if (encodings.get(i).size() > 0) {
info.put(i, encodings.get(i).size());
}
}
return info;
}
/**
* This method builds a naive bayes model
*
* @param sparkContext JavaSparkContext initialized with the application
* @param modelID Model ID
* @param trainingData Training data as a JavaRDD of LabeledPoints
* @param testingData Testing data as a JavaRDD of LabeledPoints
* @param workflow Machine learning workflow
* @param mlModel Deployable machine learning model
* @throws MLModelBuilderException
*/
private ModelSummary buildNaiveBayesModel(
JavaSparkContext sparkContext,
long modelID,
JavaRDD<LabeledPoint> trainingData,
JavaRDD<LabeledPoint> testingData,
Workflow workflow,
MLModel mlModel,
SortedMap<Integer, String> includedFeatures)
throws MLModelBuilderException {
try {
Map<String, String> hyperParameters = workflow.getHyperParameters();
NaiveBayesClassifier naiveBayesClassifier = new NaiveBayesClassifier();
NaiveBayesModel naiveBayesModel =
naiveBayesClassifier.train(
trainingData, Double.parseDouble(hyperParameters.get(MLConstants.LAMBDA)));
// remove from cache
trainingData.unpersist();
// add test data to cache
testingData.cache();
JavaPairRDD<Double, Double> predictionsAndLabels =
naiveBayesClassifier.test(naiveBayesModel, testingData).cache();
ClassClassificationAndRegressionModelSummary classClassificationAndRegressionModelSummary =
SparkModelUtils.getClassClassificationModelSummary(
sparkContext, testingData, predictionsAndLabels);
// remove from cache
testingData.unpersist();
mlModel.setModel(new MLClassificationModel(naiveBayesModel));
classClassificationAndRegressionModelSummary.setFeatures(
includedFeatures.values().toArray(new String[0]));
classClassificationAndRegressionModelSummary.setAlgorithm(
SUPERVISED_ALGORITHM.NAIVE_BAYES.toString());
MulticlassMetrics multiclassMetrics =
getMulticlassMetrics(sparkContext, predictionsAndLabels);
predictionsAndLabels.unpersist();
classClassificationAndRegressionModelSummary.setMulticlassConfusionMatrix(
getMulticlassConfusionMatrix(multiclassMetrics, mlModel));
Double modelAccuracy = getModelAccuracy(multiclassMetrics);
classClassificationAndRegressionModelSummary.setModelAccuracy(modelAccuracy);
classClassificationAndRegressionModelSummary.setDatasetVersion(workflow.getDatasetVersion());
return classClassificationAndRegressionModelSummary;
} catch (Exception e) {
throw new MLModelBuilderException(
"An error occurred while building naive bayes model: " + e.getMessage(), e);
}
}
/**
* This method builds a lasso regression model
*
* @param sparkContext JavaSparkContext initialized with the application
* @param modelID Model ID
* @param trainingData Training data as a JavaRDD of LabeledPoints
* @param testingData Testing data as a JavaRDD of LabeledPoints
* @param workflow Machine learning workflow
* @param mlModel Deployable machine learning model
* @throws MLModelBuilderException
*/
private ModelSummary buildLassoRegressionModel(
JavaSparkContext sparkContext,
long modelID,
JavaRDD<LabeledPoint> trainingData,
JavaRDD<LabeledPoint> testingData,
Workflow workflow,
MLModel mlModel,
SortedMap<Integer, String> includedFeatures)
throws MLModelBuilderException {
try {
LassoRegression lassoRegression = new LassoRegression();
Map<String, String> hyperParameters = workflow.getHyperParameters();
LassoModel lassoModel =
lassoRegression.train(
trainingData,
Integer.parseInt(hyperParameters.get(MLConstants.ITERATIONS)),
Double.parseDouble(hyperParameters.get(MLConstants.LEARNING_RATE)),
Double.parseDouble(hyperParameters.get(MLConstants.REGULARIZATION_PARAMETER)),
Double.parseDouble(hyperParameters.get(MLConstants.SGD_DATA_FRACTION)));
// remove from cache
trainingData.unpersist();
// add test data to cache
testingData.cache();
Vector weights = lassoModel.weights();
if (!isValidWeights(weights)) {
throw new MLModelBuilderException(
"Weights of the model generated are null or infinity. [Weights] "
+ vectorToString(weights));
}
JavaRDD<Tuple2<Double, Double>> predictionsAndLabels =
lassoRegression.test(lassoModel, testingData).cache();
ClassClassificationAndRegressionModelSummary regressionModelSummary =
SparkModelUtils.generateRegressionModelSummary(
sparkContext, testingData, predictionsAndLabels);
// remove from cache
testingData.unpersist();
mlModel.setModel(new MLGeneralizedLinearModel(lassoModel));
List<FeatureImportance> featureWeights =
getFeatureWeights(includedFeatures, lassoModel.weights().toArray());
regressionModelSummary.setFeatures(includedFeatures.values().toArray(new String[0]));
regressionModelSummary.setAlgorithm(SUPERVISED_ALGORITHM.LASSO_REGRESSION.toString());
regressionModelSummary.setFeatureImportance(featureWeights);
RegressionMetrics regressionMetrics =
getRegressionMetrics(sparkContext, predictionsAndLabels);
predictionsAndLabels.unpersist();
Double meanSquaredError = regressionMetrics.meanSquaredError();
regressionModelSummary.setMeanSquaredError(meanSquaredError);
regressionModelSummary.setDatasetVersion(workflow.getDatasetVersion());
return regressionModelSummary;
} catch (Exception e) {
throw new MLModelBuilderException(
"An error occurred while building lasso regression model: " + e.getMessage(), e);
}
}
/**
* This method builds a support vector machine (SVM) model
*
* @param sparkContext JavaSparkContext initialized with the application
* @param modelID Model ID
* @param trainingData Training data as a JavaRDD of LabeledPoints
* @param testingData Testing data as a JavaRDD of LabeledPoints
* @param workflow Machine learning workflow
* @param mlModel Deployable machine learning model
* @throws MLModelBuilderException
*/
private ModelSummary buildSVMModel(
JavaSparkContext sparkContext,
long modelID,
JavaRDD<LabeledPoint> trainingData,
JavaRDD<LabeledPoint> testingData,
Workflow workflow,
MLModel mlModel,
SortedMap<Integer, String> includedFeatures)
throws MLModelBuilderException {
if (getNoOfClasses(mlModel) > 2) {
throw new MLModelBuilderException(
"A binary classification algorithm cannot have more than "
+ "two distinct values in response variable.");
}
try {
SVM svm = new SVM();
Map<String, String> hyperParameters = workflow.getHyperParameters();
SVMModel svmModel =
svm.train(
trainingData,
Integer.parseInt(hyperParameters.get(MLConstants.ITERATIONS)),
hyperParameters.get(MLConstants.REGULARIZATION_TYPE),
Double.parseDouble(hyperParameters.get(MLConstants.REGULARIZATION_PARAMETER)),
Double.parseDouble(hyperParameters.get(MLConstants.LEARNING_RATE)),
Double.parseDouble(hyperParameters.get(MLConstants.SGD_DATA_FRACTION)));
// remove from cache
trainingData.unpersist();
// add test data to cache
testingData.cache();
Vector weights = svmModel.weights();
if (!isValidWeights(weights)) {
throw new MLModelBuilderException(
"Weights of the model generated are null or infinity. [Weights] "
+ vectorToString(weights));
}
// getting scores and labels without clearing threshold to get confusion matrix
JavaRDD<Tuple2<Object, Object>> scoresAndLabelsThresholded = svm.test(svmModel, testingData);
MulticlassMetrics multiclassMetrics =
new MulticlassMetrics(JavaRDD.toRDD(scoresAndLabelsThresholded));
MulticlassConfusionMatrix multiclassConfusionMatrix =
getMulticlassConfusionMatrix(multiclassMetrics, mlModel);
svmModel.clearThreshold();
JavaRDD<Tuple2<Object, Object>> scoresAndLabels = svm.test(svmModel, testingData);
ProbabilisticClassificationModelSummary probabilisticClassificationModelSummary =
SparkModelUtils.generateProbabilisticClassificationModelSummary(
sparkContext, testingData, scoresAndLabels);
// remove from cache
testingData.unpersist();
mlModel.setModel(new MLClassificationModel(svmModel));
List<FeatureImportance> featureWeights =
getFeatureWeights(includedFeatures, svmModel.weights().toArray());
probabilisticClassificationModelSummary.setFeatures(
includedFeatures.values().toArray(new String[0]));
probabilisticClassificationModelSummary.setFeatureImportance(featureWeights);
probabilisticClassificationModelSummary.setAlgorithm(SUPERVISED_ALGORITHM.SVM.toString());
probabilisticClassificationModelSummary.setMulticlassConfusionMatrix(
multiclassConfusionMatrix);
Double modelAccuracy = getModelAccuracy(multiclassMetrics);
probabilisticClassificationModelSummary.setModelAccuracy(modelAccuracy);
probabilisticClassificationModelSummary.setDatasetVersion(workflow.getDatasetVersion());
return probabilisticClassificationModelSummary;
} catch (Exception e) {
throw new MLModelBuilderException(
"An error occurred while building SVM model: " + e.getMessage(), e);
}
}