@Override
public void write(HadoopTwitterTokenToolOptions opts, TwitterTokenMode completedMode)
throws Exception {
this.stages =
new MultiStagedJob(
HadoopToolsUtil.getInputPaths(
completedMode.finalOutput(opts), CountWordsAcrossTimeperiod.WORDCOUNT_DIR),
HadoopToolsUtil.getOutputPath(outputPath),
opts.getArgs());
// Three stage process
// 1a. Write all the words (word per line)
new WordIndex().stage(stages);
final Path wordIndex = stages.runAll();
HashMap<String, IndependentPair<Long, Long>> wordCountLines =
WordIndex.readWordCountLines(wordIndex.toString(), "");
StatsWordMatch matches = new StatsWordMatch();
for (Entry<String, IndependentPair<Long, Long>> entry : wordCountLines.entrySet()) {
String word = entry.getKey();
IndependentPair<Long, Long> countLine = entry.getValue();
Long count = countLine.firstObject();
matches.updateStats(word, count);
}
System.out.println(matches);
}
private List<IndependentPair<double[], double[]>> aspairs(Matrix x, Matrix y) {
List<IndependentPair<double[], double[]>> ret =
new ArrayList<IndependentPair<double[], double[]>>();
double[][] xd = x.getArray();
double[][] yd = y.getArray();
for (int i = 0; i < xd.length; i++) {
ret.add(IndependentPair.pair(xd[i], yd[i]));
}
return ret;
}
/**
* from a report output path get the words
*
* @param path report output path
* @param ext where the words are in the path
* @return map of words to counts and index
* @throws IOException
*/
public static LinkedHashMap<String, IndependentPair<Long, Long>> readWordCountLines(
String path, String ext) throws IOException {
String wordPath = path + ext;
Path p = HadoopToolsUtil.getInputPaths(wordPath)[0];
FileSystem fs = HadoopToolsUtil.getFileSystem(p);
FSDataInputStream toRead = fs.open(p);
BufferedReader reader = new BufferedReader(new InputStreamReader(toRead, "UTF-8"));
CSVParser csvreader = new CSVParser(reader);
long lineN = 0;
String[] next = null;
LinkedHashMap<String, IndependentPair<Long, Long>> toRet =
new LinkedHashMap<String, IndependentPair<Long, Long>>();
while ((next = csvreader.getLine()) != null && next.length > 0) {
if (next.length != 2) {
System.out.println("PROBLEM READLINE LINE: " + Arrays.toString(next));
continue;
}
toRet.put(next[0], IndependentPair.pair(Long.parseLong(next[1]), lineN));
lineN++;
}
return toRet;
}
/**
* Draws the first n frames of the audio streams on to a chart mapping the names given to each
* stream into the legend. Note that the legend will only be shown if there is more than one
* stream.
*
* @param numFrames The number of frames to draw
* @param colouredFrames Whether to colour individual frames
* @param streams The audio streams and their labels
*/
public static void drawChart(
final int numFrames,
final boolean colouredFrames,
final List<IndependentPair<AudioStream, String>> streams) {
final DefaultXYDataset ds = new DefaultXYDataset();
for (final IndependentPair<AudioStream, String> asl : streams) {
final AudioStream as = asl.firstObject();
final String label = asl.secondObject();
SampleChunk s = as.nextSampleChunk();
SampleBuffer b = s.getSampleBuffer();
int x = 0;
int y = 0;
double[][] data = new double[2][];
if (!colouredFrames) {
data[0] = new double[b.size() * numFrames]; // x
data[1] = new double[b.size() * numFrames]; // y
}
for (int n = 0; n < numFrames; n++) {
s = as.nextSampleChunk();
if (s == null) break;
// Convert sample to a XY data plot
if (colouredFrames) {
data = new double[2][];
data[0] = new double[b.size()]; // x
data[1] = new double[b.size()]; // y
x = 0;
}
System.out.println("Loop " + x + " to " + (x + b.size()) + ", y = " + y);
// Copy the value into the data series
for (int z = x; z < x + b.size(); z++) {
data[0][z] = b.get(z - x);
data[1][z] = z + y;
}
// Add as a series if we're using coloured frames
if (colouredFrames) {
y += b.size();
ds.addSeries(label + ", Frame " + n, data);
} else x += b.size();
// Get ready for next loop
b = s.getSampleBuffer();
}
if (!colouredFrames) ds.addSeries(label, data);
}
final JFreeChart chart =
ChartFactory.createXYLineChart(
"Sample",
"samples",
"amplitude",
ds,
PlotOrientation.HORIZONTAL,
streams.size() > 1,
false,
false);
final ChartPanel chartPanel = new ChartPanel(chart, false);
chartPanel.setPreferredSize(new Dimension(1280, 480));
final JFrame f = new JFrame();
f.add(chartPanel, BorderLayout.CENTER);
f.pack();
f.setVisible(true);
}