/**
* Write a CSV wordIndex to a {@link MLCell} writen to a .mat data file
*
* @param path
* @throws IOException
*/
public static void writeToMatlab(String path) throws IOException {
Path wordMatPath = new Path(path + "/words/wordIndex.mat");
FileSystem fs = HadoopToolsUtil.getFileSystem(wordMatPath);
LinkedHashMap<String, IndependentPair<Long, Long>> wordIndex = readWordCountLines(path);
MLCell wordCell = new MLCell("words", new int[] {wordIndex.size(), 2});
System.out.println("... reading words");
for (Entry<String, IndependentPair<Long, Long>> ent : wordIndex.entrySet()) {
String word = ent.getKey();
int wordCellIndex = (int) (long) ent.getValue().secondObject();
long count = ent.getValue().firstObject();
wordCell.set(new MLChar(null, word), wordCellIndex, 0);
wordCell.set(new MLDouble(null, new double[][] {new double[] {count}}), wordCellIndex, 1);
}
ArrayList<MLArray> list = new ArrayList<MLArray>();
list.add(wordCell);
new MatFileWriter(fs.create(wordMatPath), list);
}
/**
* from a report output path get the words
*
* @param path report output path
* @param ext where the words are in the path
* @return map of words to counts and index
* @throws IOException
*/
public static LinkedHashMap<String, IndependentPair<Long, Long>> readWordCountLines(
String path, String ext) throws IOException {
String wordPath = path + ext;
Path p = HadoopToolsUtil.getInputPaths(wordPath)[0];
FileSystem fs = HadoopToolsUtil.getFileSystem(p);
FSDataInputStream toRead = fs.open(p);
BufferedReader reader = new BufferedReader(new InputStreamReader(toRead, "UTF-8"));
CSVParser csvreader = new CSVParser(reader);
long lineN = 0;
String[] next = null;
LinkedHashMap<String, IndependentPair<Long, Long>> toRet =
new LinkedHashMap<String, IndependentPair<Long, Long>>();
while ((next = csvreader.getLine()) != null && next.length > 0) {
if (next.length != 2) {
System.out.println("PROBLEM READLINE LINE: " + Arrays.toString(next));
continue;
}
toRet.put(next[0], IndependentPair.pair(Long.parseLong(next[1]), lineN));
lineN++;
}
return toRet;
}
