@Override
  public void write(HadoopTwitterTokenToolOptions opts, TwitterTokenMode completedMode)
      throws Exception {

    this.stages =
        new MultiStagedJob(
            HadoopToolsUtil.getInputPaths(
                completedMode.finalOutput(opts), CountWordsAcrossTimeperiod.WORDCOUNT_DIR),
            HadoopToolsUtil.getOutputPath(outputPath),
            opts.getArgs());
    // Three stage process
    // 1a. Write all the words (word per line)
    new WordIndex().stage(stages);
    final Path wordIndex = stages.runAll();

    HashMap<String, IndependentPair<Long, Long>> wordCountLines =
        WordIndex.readWordCountLines(wordIndex.toString(), "");
    StatsWordMatch matches = new StatsWordMatch();
    for (Entry<String, IndependentPair<Long, Long>> entry : wordCountLines.entrySet()) {
      String word = entry.getKey();
      IndependentPair<Long, Long> countLine = entry.getValue();
      Long count = countLine.firstObject();
      matches.updateStats(word, count);
    }

    System.out.println(matches);
  }
  @Override
  public void write(HadoopTwitterTokenToolOptions opts, TwitterTokenMode completedMode)
      throws Exception {
    MultiStagedJob stages;

    stages =
        new MultiStagedJob(
            HadoopToolsUtil.getInputPaths(
                completedMode.finalOutput(opts), CountWordsAcrossTimeperiod.WORDCOUNT_DIR),
            HadoopToolsUtil.getOutputPath(outputPath),
            opts.getArgs());
    stages.queueStage(new TimeWordJacardIndex());
    stages.runAll();
  }
 /**
  * from a report output path get the words
  *
  * @param path report output path
  * @param ext where the words are in the path
  * @return map of words to counts and index
  * @throws IOException
  */
 public static LinkedHashMap<String, IndependentPair<Long, Long>> readWordCountLines(
     String path, String ext) throws IOException {
   String wordPath = path + ext;
   Path p = HadoopToolsUtil.getInputPaths(wordPath)[0];
   FileSystem fs = HadoopToolsUtil.getFileSystem(p);
   FSDataInputStream toRead = fs.open(p);
   BufferedReader reader = new BufferedReader(new InputStreamReader(toRead, "UTF-8"));
   CSVParser csvreader = new CSVParser(reader);
   long lineN = 0;
   String[] next = null;
   LinkedHashMap<String, IndependentPair<Long, Long>> toRet =
       new LinkedHashMap<String, IndependentPair<Long, Long>>();
   while ((next = csvreader.getLine()) != null && next.length > 0) {
     if (next.length != 2) {
       System.out.println("PROBLEM READLINE LINE: " + Arrays.toString(next));
       continue;
     }
     toRet.put(next[0], IndependentPair.pair(Long.parseLong(next[1]), lineN));
     lineN++;
   }
   return toRet;
 }
  /**
   * Write a CSV wordIndex to a {@link MLCell} writen to a .mat data file
   *
   * @param path
   * @throws IOException
   */
  public static void writeToMatlab(String path) throws IOException {
    Path wordMatPath = new Path(path + "/words/wordIndex.mat");
    FileSystem fs = HadoopToolsUtil.getFileSystem(wordMatPath);
    LinkedHashMap<String, IndependentPair<Long, Long>> wordIndex = readWordCountLines(path);
    MLCell wordCell = new MLCell("words", new int[] {wordIndex.size(), 2});

    System.out.println("... reading words");
    for (Entry<String, IndependentPair<Long, Long>> ent : wordIndex.entrySet()) {
      String word = ent.getKey();
      int wordCellIndex = (int) (long) ent.getValue().secondObject();
      long count = ent.getValue().firstObject();
      wordCell.set(new MLChar(null, word), wordCellIndex, 0);
      wordCell.set(new MLDouble(null, new double[][] {new double[] {count}}), wordCellIndex, 1);
    }
    ArrayList<MLArray> list = new ArrayList<MLArray>();
    list.add(wordCell);
    new MatFileWriter(fs.create(wordMatPath), list);
  }