/* (non-Javadoc) * @see org.apache.hadoop.mapreduce.Mapper#cleanup(org.apache.hadoop.mapreduce.Mapper.Context) */ protected void cleanup(Context context) throws IOException, InterruptedException { if (isValidationMode) { if (neighborhood.IsInClassificationMode()) { context.getCounter("Validation", "TruePositive").increment(confMatrix.getTruePos()); context.getCounter("Validation", "FalseNegative").increment(confMatrix.getFalseNeg()); context.getCounter("Validation", "TrueNagative").increment(confMatrix.getTrueNeg()); context.getCounter("Validation", "FalsePositive").increment(confMatrix.getFalsePos()); context.getCounter("Validation", "Accuracy").increment(confMatrix.getAccuracy()); context.getCounter("Validation", "Recall").increment(confMatrix.getRecall()); context.getCounter("Validation", "Precision").increment(confMatrix.getPrecision()); } } }
/* (non-Javadoc) * @see org.apache.hadoop.mapreduce.Reducer#reduce(KEYIN, java.lang.Iterable, org.apache.hadoop.mapreduce.Reducer.Context) */ protected void reduce(Tuple key, Iterable<Tuple> values, Context context) throws IOException, InterruptedException { if (stBld.length() > 0) { stBld.delete(0, stBld.length()); } testEntityId = key.getString(0); stBld.append(testEntityId); // collect nearest neighbors count = 0; neighborhood.initialize(); for (Tuple value : values) { int index = 0; trainEntityId = value.getString(index++); distance = value.getInt(index++); trainClassValue = value.getString(index++); if (classCondtionWeighted && neighborhood.IsInClassificationMode()) { trainingFeaturePostProb = value.getDouble(index++); if (inverseDistanceWeighted) { neighborhood.addNeighbor( trainEntityId, distance, trainClassValue, trainingFeaturePostProb, true); } else { neighborhood.addNeighbor( trainEntityId, distance, trainClassValue, trainingFeaturePostProb); } } else { Neighborhood.Neighbor neighbor = neighborhood.addNeighbor(trainEntityId, distance, trainClassValue); if (neighborhood.isInLinearRegressionMode()) { neighbor.setRegrInputVar(Double.parseDouble(value.getString(index++))); } } if (++count == topMatchCount) { break; } } if (neighborhood.isInLinearRegressionMode()) { String testRegrNumFld = isValidationMode ? key.getString(2) : key.getString(1); neighborhood.withRegrInputVar(Double.parseDouble(testRegrNumFld)); } // class distribution neighborhood.processClassDitribution(); if (outputClassDistr && neighborhood.IsInClassificationMode()) { if (classCondtionWeighted) { Map<String, Double> classDistr = neighborhood.getWeightedClassDitribution(); double thisScore; for (String classVal : classDistr.keySet()) { thisScore = classDistr.get(classVal); // LOG.debug("classVal:" + classVal + " thisScore:" + thisScore); stBld.append(fieldDelim).append(classVal).append(fieldDelim).append(thisScore); } } else { Map<String, Integer> classDistr = neighborhood.getClassDitribution(); int thisScore; for (String classVal : classDistr.keySet()) { thisScore = classDistr.get(classVal); stBld.append(classVal).append(fieldDelim).append(thisScore); } } } if (isValidationMode) { // actual class attr value testClassValActual = key.getString(1); stBld.append(fieldDelim).append(testClassValActual); } // predicted class value if (useCostBasedClassifier) { // use cost based arbitrator if (neighborhood.IsInClassificationMode()) { posClassProbab = neighborhood.getClassProb(posClassAttrValue); testClassValPredicted = costBasedArbitrator.classify(posClassProbab); } } else { // get directly if (neighborhood.IsInClassificationMode()) { testClassValPredicted = neighborhood.classify(); } else { testClassValPredicted = "" + neighborhood.getPredictedValue(); } } stBld.append(fieldDelim).append(testClassValPredicted); if (isValidationMode) { if (neighborhood.IsInClassificationMode()) { confMatrix.report(testClassValPredicted, testClassValActual); } } outVal.set(stBld.toString()); context.write(NullWritable.get(), outVal); }
/* (non-Javadoc) * @see org.apache.hadoop.mapreduce.Reducer#setup(org.apache.hadoop.mapreduce.Reducer.Context) */ protected void setup(Context context) throws IOException, InterruptedException { Configuration config = context.getConfiguration(); if (config.getBoolean("debug.on", false)) { LOG.setLevel(Level.DEBUG); System.out.println("in debug mode"); } fieldDelim = config.get("field.delim", ","); topMatchCount = config.getInt("top.match.count", 10); isValidationMode = config.getBoolean("validation.mode", true); kernelFunction = config.get("kernel.function", "none"); kernelParam = config.getInt("kernel.param", -1); classCondtionWeighted = config.getBoolean("class.condtion.weighted", false); neighborhood = new Neighborhood(kernelFunction, kernelParam, classCondtionWeighted); outputClassDistr = config.getBoolean("output.class.distr", false); inverseDistanceWeighted = config.getBoolean("inverse.distance.weighted", false); // regression String predictionMode = config.get("prediction.mode", "classification"); if (predictionMode.equals("regression")) { neighborhood.withPredictionMode(PredictionMode.Regression); String regressionMethod = config.get("regression.method", "average"); regressionMethod = WordUtils.capitalize(regressionMethod); neighborhood.withRegressionMethod(RegressionMethod.valueOf(regressionMethod)); } // decision threshold for classification decisionThreshold = Double.parseDouble(config.get("decision.threshold", "-1.0")); if (decisionThreshold > 0 && neighborhood.IsInClassificationMode()) { String[] classAttrValues = config.get("class.attribute.values").split(","); posClassAttrValue = classAttrValues[0]; negClassAttrValue = classAttrValues[1]; neighborhood.withDecisionThreshold(decisionThreshold).withPositiveClass(posClassAttrValue); } // using cost based arbitrator for classification useCostBasedClassifier = config.getBoolean("use.cost.based.classifier", false); if (useCostBasedClassifier && neighborhood.IsInClassificationMode()) { if (null == posClassAttrValue) { String[] classAttrValues = config.get("class.attribute.values").split(","); posClassAttrValue = classAttrValues[0]; negClassAttrValue = classAttrValues[1]; } int[] missclassificationCost = Utility.intArrayFromString(config.get("misclassification.cost")); falsePosCost = missclassificationCost[0]; falseNegCost = missclassificationCost[1]; costBasedArbitrator = new CostBasedArbitrator( negClassAttrValue, posClassAttrValue, falseNegCost, falsePosCost); } // confusion matrix for classification validation if (isValidationMode) { if (neighborhood.IsInClassificationMode()) { InputStream fs = Utility.getFileStream(context.getConfiguration(), "feature.schema.file.path"); ObjectMapper mapper = new ObjectMapper(); schema = mapper.readValue(fs, FeatureSchema.class); classAttrField = schema.findClassAttrField(); List<String> cardinality = classAttrField.getCardinality(); predictingClasses = new String[2]; predictingClasses[0] = cardinality.get(0); predictingClasses[1] = cardinality.get(1); confMatrix = new ConfusionMatrix(predictingClasses[0], predictingClasses[1]); } } LOG.debug( "classCondtionWeighted:" + classCondtionWeighted + "outputClassDistr:" + outputClassDistr); }
@Override public Set<V> getSuccessors(V vertex) { Neighborhood<V, VL, EL> neighborhood = mNeighbors.get(vertex); return (neighborhood != null) ? neighborhood.getForwardLinks() : null; }
@Override public Set<V> getPredecessors(V vertex) { Neighborhood<V, VL, EL> neighborhood = mNeighbors.get(vertex); return (neighborhood != null) ? neighborhood.getReverseLinks() : null; }
@Override public EL getEdgeLabel(Object s, Object t) { Neighborhood<V, VL, EL> s_neighborhood = mNeighbors.get(s); return (s_neighborhood != null) ? s_neighborhood.getForwardLinkLabel(t) : null; }