public void testKeepWordsPathSettings() {
Settings settings =
Settings.settingsBuilder()
.put("path.home", createTempDir().toString())
.put("index.analysis.filter.non_broken_keep_filter.type", "keep")
.put(
"index.analysis.filter.non_broken_keep_filter.keep_words_path",
"does/not/exists.txt")
.build();
try {
// test our none existing setup is picked up
AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
fail("expected an exception due to non existent keep_words_path");
} catch (IllegalArgumentException e) {
} catch (IOException e) {
fail("expected IAE");
}
settings =
Settings.settingsBuilder()
.put(settings)
.put("index.analysis.filter.non_broken_keep_filter.keep_words", new String[] {"test"})
.build();
try {
// test our none existing setup is picked up
AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
fail(
"expected an exception indicating that you can't use [keep_words_path] with [keep_words] ");
} catch (IllegalArgumentException e) {
} catch (IOException e) {
fail("expected IAE");
}
}
@Test
public void testSettings() throws IOException {
{
Settings settings =
ImmutableSettings.settingsBuilder()
.put("index.analysis.filter.limit_1.type", "limit")
.put("index.analysis.filter.limit_1.max_token_count", 3)
.put("index.analysis.filter.limit_1.consume_all_tokens", true)
.build();
AnalysisService analysisService =
AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
TokenFilterFactory tokenFilter = analysisService.tokenFilter("limit_1");
String source = "the quick brown fox";
String[] expected = new String[] {"the", "quick", "brown"};
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
}
{
Settings settings =
ImmutableSettings.settingsBuilder()
.put("index.analysis.filter.limit_1.type", "limit")
.put("index.analysis.filter.limit_1.max_token_count", 3)
.put("index.analysis.filter.limit_1.consume_all_tokens", false)
.build();
AnalysisService analysisService =
AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
TokenFilterFactory tokenFilter = analysisService.tokenFilter("limit_1");
String source = "the quick brown fox";
String[] expected = new String[] {"the", "quick", "brown"};
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
}
{
Settings settings =
ImmutableSettings.settingsBuilder()
.put("index.analysis.filter.limit_1.type", "limit")
.put("index.analysis.filter.limit_1.max_token_count", 17)
.put("index.analysis.filter.limit_1.consume_all_tokens", true)
.build();
AnalysisService analysisService =
AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
TokenFilterFactory tokenFilter = analysisService.tokenFilter("limit_1");
String source = "the quick brown fox";
String[] expected = new String[] {"the", "quick", "brown", "fox"};
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
}
}
@Test
public void testDefault() throws IOException {
Settings settings =
ImmutableSettings.settingsBuilder()
.put("index.analysis.filter.limit_default.type", "limit")
.build();
AnalysisService analysisService =
AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
{
TokenFilterFactory tokenFilter = analysisService.tokenFilter("limit_default");
String source = "the quick brown fox";
String[] expected = new String[] {"the"};
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
}
{
TokenFilterFactory tokenFilter = analysisService.tokenFilter("limit");
String source = "the quick brown fox";
String[] expected = new String[] {"the"};
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
}
}
public void testDefault() throws IOException {
AnalysisService analysisService =
AnalysisTestsHelper.createAnalysisServiceFromSettings(
settingsBuilder()
.put("path.home", createTempDir().toString())
.put("index.analysis.filter.my_ascii_folding.type", "asciifolding")
.build());
TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_ascii_folding");
String source = "Ansprüche";
String[] expected = new String[] {"Anspruche"};
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
}
public void testPreserveOriginal() throws IOException {
AnalysisService analysisService =
AnalysisTestsHelper.createAnalysisServiceFromSettings(
settingsBuilder()
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.put("index.analysis.filter.my_ascii_folding.type", "asciifolding")
.put("index.analysis.filter.my_ascii_folding.preserve_original", true)
.build());
TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_ascii_folding");
String source = "Ansprüche";
String[] expected = new String[] {"Anspruche", "Ansprüche"};
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
}
public void testLoadOverConfiguredSettings() {
Settings settings =
Settings.settingsBuilder()
.put("path.home", createTempDir().toString())
.put("index.analysis.filter.broken_keep_filter.type", "keep")
.put("index.analysis.filter.broken_keep_filter.keep_words_path", "does/not/exists.txt")
.put("index.analysis.filter.broken_keep_filter.keep_words", "[\"Hello\", \"worlD\"]")
.build();
try {
AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
Assert.fail("path and array are configured");
} catch (IllegalArgumentException e) {
} catch (IOException e) {
fail("expected IAE");
}
}