/* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.Mapper#cleanup(org.apache.hadoop.mapreduce.Mapper.Context)
*/
protected void cleanup(Context context) throws IOException, InterruptedException {
if (isValidationMode) {
if (neighborhood.IsInClassificationMode()) {
context.getCounter("Validation", "TruePositive").increment(confMatrix.getTruePos());
context.getCounter("Validation", "FalseNegative").increment(confMatrix.getFalseNeg());
context.getCounter("Validation", "TrueNagative").increment(confMatrix.getTrueNeg());
context.getCounter("Validation", "FalsePositive").increment(confMatrix.getFalsePos());
context.getCounter("Validation", "Accuracy").increment(confMatrix.getAccuracy());
context.getCounter("Validation", "Recall").increment(confMatrix.getRecall());
context.getCounter("Validation", "Precision").increment(confMatrix.getPrecision());
}
}
}
/* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.Reducer#reduce(KEYIN, java.lang.Iterable, org.apache.hadoop.mapreduce.Reducer.Context)
*/
protected void reduce(Tuple key, Iterable<Tuple> values, Context context)
throws IOException, InterruptedException {
if (stBld.length() > 0) {
stBld.delete(0, stBld.length());
}
testEntityId = key.getString(0);
stBld.append(testEntityId);
// collect nearest neighbors
count = 0;
neighborhood.initialize();
for (Tuple value : values) {
int index = 0;
trainEntityId = value.getString(index++);
distance = value.getInt(index++);
trainClassValue = value.getString(index++);
if (classCondtionWeighted && neighborhood.IsInClassificationMode()) {
trainingFeaturePostProb = value.getDouble(index++);
if (inverseDistanceWeighted) {
neighborhood.addNeighbor(
trainEntityId, distance, trainClassValue, trainingFeaturePostProb, true);
} else {
neighborhood.addNeighbor(
trainEntityId, distance, trainClassValue, trainingFeaturePostProb);
}
} else {
Neighborhood.Neighbor neighbor =
neighborhood.addNeighbor(trainEntityId, distance, trainClassValue);
if (neighborhood.isInLinearRegressionMode()) {
neighbor.setRegrInputVar(Double.parseDouble(value.getString(index++)));
}
}
if (++count == topMatchCount) {
break;
}
}
if (neighborhood.isInLinearRegressionMode()) {
String testRegrNumFld = isValidationMode ? key.getString(2) : key.getString(1);
neighborhood.withRegrInputVar(Double.parseDouble(testRegrNumFld));
}
// class distribution
neighborhood.processClassDitribution();
if (outputClassDistr && neighborhood.IsInClassificationMode()) {
if (classCondtionWeighted) {
Map<String, Double> classDistr = neighborhood.getWeightedClassDitribution();
double thisScore;
for (String classVal : classDistr.keySet()) {
thisScore = classDistr.get(classVal);
// LOG.debug("classVal:" + classVal + " thisScore:" + thisScore);
stBld.append(fieldDelim).append(classVal).append(fieldDelim).append(thisScore);
}
} else {
Map<String, Integer> classDistr = neighborhood.getClassDitribution();
int thisScore;
for (String classVal : classDistr.keySet()) {
thisScore = classDistr.get(classVal);
stBld.append(classVal).append(fieldDelim).append(thisScore);
}
}
}
if (isValidationMode) {
// actual class attr value
testClassValActual = key.getString(1);
stBld.append(fieldDelim).append(testClassValActual);
}
// predicted class value
if (useCostBasedClassifier) {
// use cost based arbitrator
if (neighborhood.IsInClassificationMode()) {
posClassProbab = neighborhood.getClassProb(posClassAttrValue);
testClassValPredicted = costBasedArbitrator.classify(posClassProbab);
}
} else {
// get directly
if (neighborhood.IsInClassificationMode()) {
testClassValPredicted = neighborhood.classify();
} else {
testClassValPredicted = "" + neighborhood.getPredictedValue();
}
}
stBld.append(fieldDelim).append(testClassValPredicted);
if (isValidationMode) {
if (neighborhood.IsInClassificationMode()) {
confMatrix.report(testClassValPredicted, testClassValActual);
}
}
outVal.set(stBld.toString());
context.write(NullWritable.get(), outVal);
}
/* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.Reducer#setup(org.apache.hadoop.mapreduce.Reducer.Context)
*/
protected void setup(Context context) throws IOException, InterruptedException {
Configuration config = context.getConfiguration();
if (config.getBoolean("debug.on", false)) {
LOG.setLevel(Level.DEBUG);
System.out.println("in debug mode");
}
fieldDelim = config.get("field.delim", ",");
topMatchCount = config.getInt("top.match.count", 10);
isValidationMode = config.getBoolean("validation.mode", true);
kernelFunction = config.get("kernel.function", "none");
kernelParam = config.getInt("kernel.param", -1);
classCondtionWeighted = config.getBoolean("class.condtion.weighted", false);
neighborhood = new Neighborhood(kernelFunction, kernelParam, classCondtionWeighted);
outputClassDistr = config.getBoolean("output.class.distr", false);
inverseDistanceWeighted = config.getBoolean("inverse.distance.weighted", false);
// regression
String predictionMode = config.get("prediction.mode", "classification");
if (predictionMode.equals("regression")) {
neighborhood.withPredictionMode(PredictionMode.Regression);
String regressionMethod = config.get("regression.method", "average");
regressionMethod = WordUtils.capitalize(regressionMethod);
neighborhood.withRegressionMethod(RegressionMethod.valueOf(regressionMethod));
}
// decision threshold for classification
decisionThreshold = Double.parseDouble(config.get("decision.threshold", "-1.0"));
if (decisionThreshold > 0 && neighborhood.IsInClassificationMode()) {
String[] classAttrValues = config.get("class.attribute.values").split(",");
posClassAttrValue = classAttrValues[0];
negClassAttrValue = classAttrValues[1];
neighborhood.withDecisionThreshold(decisionThreshold).withPositiveClass(posClassAttrValue);
}
// using cost based arbitrator for classification
useCostBasedClassifier = config.getBoolean("use.cost.based.classifier", false);
if (useCostBasedClassifier && neighborhood.IsInClassificationMode()) {
if (null == posClassAttrValue) {
String[] classAttrValues = config.get("class.attribute.values").split(",");
posClassAttrValue = classAttrValues[0];
negClassAttrValue = classAttrValues[1];
}
int[] missclassificationCost =
Utility.intArrayFromString(config.get("misclassification.cost"));
falsePosCost = missclassificationCost[0];
falseNegCost = missclassificationCost[1];
costBasedArbitrator =
new CostBasedArbitrator(
negClassAttrValue, posClassAttrValue, falseNegCost, falsePosCost);
}
// confusion matrix for classification validation
if (isValidationMode) {
if (neighborhood.IsInClassificationMode()) {
InputStream fs =
Utility.getFileStream(context.getConfiguration(), "feature.schema.file.path");
ObjectMapper mapper = new ObjectMapper();
schema = mapper.readValue(fs, FeatureSchema.class);
classAttrField = schema.findClassAttrField();
List<String> cardinality = classAttrField.getCardinality();
predictingClasses = new String[2];
predictingClasses[0] = cardinality.get(0);
predictingClasses[1] = cardinality.get(1);
confMatrix = new ConfusionMatrix(predictingClasses[0], predictingClasses[1]);
}
}
LOG.debug(
"classCondtionWeighted:"
+ classCondtionWeighted
+ "outputClassDistr:"
+ outputClassDistr);
}
@Override
public Set<V> getSuccessors(V vertex) {
Neighborhood<V, VL, EL> neighborhood = mNeighbors.get(vertex);
return (neighborhood != null) ? neighborhood.getForwardLinks() : null;
}
@Override
public Set<V> getPredecessors(V vertex) {
Neighborhood<V, VL, EL> neighborhood = mNeighbors.get(vertex);
return (neighborhood != null) ? neighborhood.getReverseLinks() : null;
}
@Override
public EL getEdgeLabel(Object s, Object t) {
Neighborhood<V, VL, EL> s_neighborhood = mNeighbors.get(s);
return (s_neighborhood != null) ? s_neighborhood.getForwardLinkLabel(t) : null;
}
