public static NaiveBayesModel materialize(Path output, Configuration conf) throws IOException {
FileSystem fs = output.getFileSystem(conf);
Vector weightsPerLabel = null;
Vector perLabelThetaNormalizer = null;
Vector weightsPerFeature = null;
Matrix weightsPerLabelAndFeature;
float alphaI;
FSDataInputStream in = fs.open(new Path(output, "naiveBayesModel.bin"));
try {
alphaI = in.readFloat();
weightsPerFeature = VectorWritable.readVector(in);
weightsPerLabel = VectorWritable.readVector(in);
perLabelThetaNormalizer = VectorWritable.readVector(in);
weightsPerLabelAndFeature =
new SparseMatrix(weightsPerLabel.size(), weightsPerFeature.size());
for (int label = 0; label < weightsPerLabelAndFeature.numRows(); label++) {
weightsPerLabelAndFeature.assignRow(label, VectorWritable.readVector(in));
}
} finally {
Closeables.closeQuietly(in);
}
NaiveBayesModel model =
new NaiveBayesModel(
weightsPerLabelAndFeature,
weightsPerFeature,
weightsPerLabel,
perLabelThetaNormalizer,
alphaI);
model.validate();
return model;
}
private float phase1(
int i, int j, String basePath, int numNodes, boolean useCombiner, boolean useInMapperCombiner)
throws Exception {
Job job = Job.getInstance(getConf());
job.setJobName("PageRank:Basic:iteration" + j + ":Phase1");
job.setJarByClass(RunPageRankBasic.class);
String in = basePath + "/iter" + formatter.format(i);
String out = basePath + "/iter" + formatter.format(j) + "t";
String outm = out + "-mass";
// We need to actually count the number of part files to get the number of partitions (because
// the directory might contain _log).
int numPartitions = 0;
for (FileStatus s : FileSystem.get(getConf()).listStatus(new Path(in))) {
if (s.getPath().getName().contains("part-")) numPartitions++;
}
LOG.info("PageRank: iteration " + j + ": Phase1");
LOG.info(" - input: " + in);
LOG.info(" - output: " + out);
LOG.info(" - nodeCnt: " + numNodes);
LOG.info(" - useCombiner: " + useCombiner);
LOG.info(" - useInmapCombiner: " + useInMapperCombiner);
LOG.info("computed number of partitions: " + numPartitions);
int numReduceTasks = numPartitions;
job.getConfiguration().setInt("NodeCount", numNodes);
job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
job.getConfiguration().setBoolean("mapred.reduce.tasks.speculative.execution", false);
// job.getConfiguration().set("mapred.child.java.opts", "-Xmx2048m");
job.getConfiguration().set("PageRankMassPath", outm);
job.setNumReduceTasks(numReduceTasks);
FileInputFormat.setInputPaths(job, new Path(in));
FileOutputFormat.setOutputPath(job, new Path(out));
job.setInputFormatClass(NonSplitableSequenceFileInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(PageRankNode.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(PageRankNode.class);
job.setMapperClass(useInMapperCombiner ? MapWithInMapperCombiningClass.class : MapClass.class);
if (useCombiner) {
job.setCombinerClass(CombineClass.class);
}
job.setReducerClass(ReduceClass.class);
FileSystem.get(getConf()).delete(new Path(out), true);
FileSystem.get(getConf()).delete(new Path(outm), true);
long startTime = System.currentTimeMillis();
job.waitForCompletion(true);
System.out.println(
"Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
float mass = Float.NEGATIVE_INFINITY;
FileSystem fs = FileSystem.get(getConf());
for (FileStatus f : fs.listStatus(new Path(outm))) {
FSDataInputStream fin = fs.open(f.getPath());
mass = sumLogProbs(mass, fin.readFloat());
fin.close();
}
return mass;
}