public final ImmutableList<AnalyzeToken> analyze(final String field, final String text)
throws IoExceptionT {
AnalyzeResponse response;
try {
response =
client
.admin()
.indices()
.prepareAnalyze(text)
.setField(field)
.setIndex(indexName)
.execute()
.actionGet();
} catch (final ElasticsearchException e) {
throw exceptionFactory.newIoException(
e, String.format("error analyzing text with index %s", indexName));
}
return ImmutableList.copyOf(response.getTokens());
}
public void testAnalyze() {
createIndexWithAlias();
assertAcked(
client()
.admin()
.indices()
.preparePutMapping("test")
.setType("test")
.setSource("field", "type=text,analyzer=keyword"));
ensureYellow("test");
AnalyzeResponse analyzeResponse =
client()
.admin()
.indices()
.prepareAnalyze("this is a test")
.setIndex(indexOrAlias())
.setField("field")
.get();
assertThat(analyzeResponse.getTokens().size(), equalTo(1));
assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("this is a test"));
}
/**
* Simple upgrade test for analyzers to make sure they analyze to the same tokens after upgrade
* TODO we need this for random tokenizers / tokenfilters as well
*/
@Test
public void testAnalyzerTokensAfterUpgrade()
throws IOException, ExecutionException, InterruptedException {
int numFields =
randomIntBetween(PreBuiltAnalyzers.values().length, PreBuiltAnalyzers.values().length * 10);
StringBuilder builder = new StringBuilder();
String[] fields = new String[numFields * 2];
int fieldId = 0;
for (int i = 0; i < fields.length; i++) {
fields[i++] = "field_" + fieldId++;
String analyzer = randomAnalyzer();
fields[i] = "type=string,analyzer=" + analyzer;
}
assertAcked(prepareCreate("test").addMapping("type", fields).setSettings(indexSettings()));
ensureYellow();
InputOutput[] inout = new InputOutput[numFields];
for (int i = 0; i < numFields; i++) {
String input;
Matcher matcher;
do {
// In Lucene 4.10, a bug was fixed in StandardTokenizer which was causing breaks on complex
// characters.
// The bug was fixed without backcompat Version handling, so testing between >=4.10 vs <=
// 4.9 can
// cause differences when the random string generated contains these complex characters. To
// mitigate
// the problem, we skip any strings containing these characters.
// TODO: only skip strings containing complex chars when comparing against ES <= 1.3.x
input = TestUtil.randomAnalysisString(getRandom(), 100, false);
matcher = complexUnicodeChars.matcher(input);
} while (matcher.find());
AnalyzeResponse test =
client().admin().indices().prepareAnalyze("test", input).setField("field_" + i).get();
inout[i] = new InputOutput(test, input, "field_" + i);
}
logClusterState();
boolean upgraded;
do {
logClusterState();
upgraded = backwardsCluster().upgradeOneNode();
ensureYellow();
} while (upgraded);
for (int i = 0; i < inout.length; i++) {
InputOutput inputOutput = inout[i];
AnalyzeResponse test =
client()
.admin()
.indices()
.prepareAnalyze("test", inputOutput.input)
.setField(inputOutput.field)
.get();
List<AnalyzeResponse.AnalyzeToken> tokens = test.getTokens();
List<AnalyzeResponse.AnalyzeToken> expectedTokens = inputOutput.response.getTokens();
assertThat(
"size mismatch field: "
+ fields[i * 2]
+ " analyzer: "
+ fields[i * 2 + 1]
+ " input: "
+ BaseTokenStreamTestCase.escape(inputOutput.input),
expectedTokens.size(),
equalTo(tokens.size()));
for (int j = 0; j < tokens.size(); j++) {
String msg =
"failed for term: "
+ expectedTokens.get(j).getTerm()
+ " field: "
+ fields[i * 2]
+ " analyzer: "
+ fields[i * 2 + 1]
+ " input: "
+ BaseTokenStreamTestCase.escape(inputOutput.input);
assertThat(
msg,
BaseTokenStreamTestCase.escape(expectedTokens.get(j).getTerm()),
equalTo(BaseTokenStreamTestCase.escape(tokens.get(j).getTerm())));
assertThat(msg, expectedTokens.get(j).getPosition(), equalTo(tokens.get(j).getPosition()));
assertThat(
msg, expectedTokens.get(j).getStartOffset(), equalTo(tokens.get(j).getStartOffset()));
assertThat(
msg, expectedTokens.get(j).getEndOffset(), equalTo(tokens.get(j).getEndOffset()));
assertThat(msg, expectedTokens.get(j).getType(), equalTo(tokens.get(j).getType()));
}
}
}