private <T> void assertExpectedTokens(LuceneAnalyzer analyzer, List<T> testData) {
JavaRDD<T> rdd = jsc.parallelize(testData);
Row[] pairs =
analyzer
.transform(jsql.createDataFrame(rdd, testData.get(0).getClass()))
.select("wantedTokens", "tokens")
.collect();
for (Row r : pairs) {
Assert.assertEquals(r.get(0), r.get(1));
}
}
