/**
* Analyzes the given value using the given Analyzer.
*
* @param value Value to analyze
* @param context The {@link AnalysisContext analysis context}.
* @return NamedList containing the tokens produced by analyzing the given value
*/
protected NamedList<? extends Object> analyzeValue(String value, AnalysisContext context) {
Analyzer analyzer = context.getAnalyzer();
if (!TokenizerChain.class.isInstance(analyzer)) {
TokenStream tokenStream = null;
try {
tokenStream = analyzer.reusableTokenStream(context.getFieldName(), new StringReader(value));
tokenStream.reset();
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
NamedList<List<NamedList>> namedList = new NamedList<List<NamedList>>();
namedList.add(
tokenStream.getClass().getName(),
convertTokensToNamedLists(analyzeTokenStream(tokenStream), context));
return namedList;
}
TokenizerChain tokenizerChain = (TokenizerChain) analyzer;
CharFilterFactory[] cfiltfacs = tokenizerChain.getCharFilterFactories();
TokenizerFactory tfac = tokenizerChain.getTokenizerFactory();
TokenFilterFactory[] filtfacs = tokenizerChain.getTokenFilterFactories();
NamedList<Object> namedList = new NamedList<Object>();
if (cfiltfacs != null) {
String source = value;
for (CharFilterFactory cfiltfac : cfiltfacs) {
CharStream reader = CharReader.get(new StringReader(source));
reader = cfiltfac.create(reader);
source = writeCharStream(namedList, reader);
}
}
TokenStream tokenStream = tfac.create(tokenizerChain.charStream(new StringReader(value)));
List<AttributeSource> tokens = analyzeTokenStream(tokenStream);
namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context));
ListBasedTokenStream listBasedTokenStream = new ListBasedTokenStream(tokens);
for (TokenFilterFactory tokenFilterFactory : filtfacs) {
tokenStream = tokenFilterFactory.create(listBasedTokenStream);
List<AttributeSource> tokenList = analyzeTokenStream(tokenStream);
namedList.add(
tokenStream.getClass().getName(), convertTokensToNamedLists(tokenList, context));
listBasedTokenStream = new ListBasedTokenStream(tokenList);
}
return namedList;
}
private static AnalysisContext.AnalysisResult analyze(String query) {
try {
AnalysisContext analysisCtxt = new AnalysisContext(catalog_, TestUtils.createQueryContext());
analysisCtxt.analyze(query);
AnalysisContext.AnalysisResult analysisResult = analysisCtxt.getAnalysisResult();
Preconditions.checkNotNull(analysisResult.getStmt());
return analysisResult;
} catch (Exception e) {
e.printStackTrace();
fail("Failed to analyze query: " + query + "\n" + e.getMessage());
}
return null;
}
@Before
public void setup() {
Bootstraps.inject(this);
analysisContext = analysisContextFactory.buildContext();
analysisContext
.getInjectionNodeBuilders()
.putType(
B.class,
variableInjectionBuilderFactory.buildVariableInjectionNodeBuilder(
astClassFactory.getType(BImpl.class)));
}
/**
* Converts the list of Tokens to a list of NamedLists representing the tokens.
*
* @param tokens Tokens to convert
* @param context The analysis context
* @return List of NamedLists containing the relevant information taken from the tokens
*/
private List<NamedList> convertTokensToNamedLists(
final List<AttributeSource> tokens, AnalysisContext context) {
final List<NamedList> tokensNamedLists = new ArrayList<NamedList>();
final int[] positions = new int[tokens.size()];
int position = 0;
for (int i = 0, c = tokens.size(); i < c; i++) {
AttributeSource token = tokens.get(i);
position += token.addAttribute(PositionIncrementAttribute.class).getPositionIncrement();
positions[i] = position;
}
// sort the tokens by absoulte position
new SorterTemplate() {
@Override
protected void swap(int i, int j) {
final int p = positions[i];
positions[i] = positions[j];
positions[j] = p;
Collections.swap(tokens, i, j);
}
@Override
protected int compare(int i, int j) {
return positions[i] - positions[j];
}
@Override
protected void setPivot(int i) {
pivot = positions[i];
}
@Override
protected int comparePivot(int j) {
return pivot - positions[j];
}
private int pivot;
}.mergeSort(0, tokens.size() - 1);
FieldType fieldType = context.getFieldType();
final CharArr textBuf = new CharArr();
for (int i = 0, c = tokens.size(); i < c; i++) {
AttributeSource token = tokens.get(i);
final NamedList<Object> tokenNamedList = new SimpleOrderedMap<Object>();
final TermToBytesRefAttribute termAtt = token.getAttribute(TermToBytesRefAttribute.class);
BytesRef rawBytes = termAtt.getBytesRef();
termAtt.fillBytesRef();
textBuf.reset();
fieldType.indexedToReadable(rawBytes, textBuf);
final String text = textBuf.toString();
tokenNamedList.add("text", text);
if (token.hasAttribute(CharTermAttribute.class)) {
final String rawText = token.getAttribute(CharTermAttribute.class).toString();
if (!rawText.equals(text)) {
tokenNamedList.add("raw_text", rawText);
}
}
tokenNamedList.add("raw_bytes", rawBytes.toString());
if (context.getTermsToMatch().contains(rawBytes)) {
tokenNamedList.add("match", true);
}
tokenNamedList.add("position", positions[i]);
token.reflectWith(
new AttributeReflector() {
public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
// leave out position and bytes term
if (TermToBytesRefAttribute.class.isAssignableFrom(attClass)) return;
if (CharTermAttribute.class.isAssignableFrom(attClass)) return;
if (PositionIncrementAttribute.class.isAssignableFrom(attClass)) return;
String k = attClass.getName() + '#' + key;
// map keys for "standard attributes":
if (ATTRIBUTE_MAPPING.containsKey(k)) {
k = ATTRIBUTE_MAPPING.get(k);
}
if (value instanceof Payload) {
final Payload p = (Payload) value;
value = new BytesRef(p.getData()).toString();
}
tokenNamedList.add(k, value);
}
});
tokensNamedLists.add(tokenNamedList);
}
return tokensNamedLists;
}
@Override
protected Type visitFunctionCall(FunctionCall node, AnalysisContext context) {
if (node.getWindow().isPresent()) {
for (Expression expression : node.getWindow().get().getPartitionBy()) {
process(expression, context);
Type type = expressionTypes.get(expression);
if (!type.isComparable()) {
throw new SemanticException(
TYPE_MISMATCH,
node,
"%s is not comparable, and therefore cannot be used in window function PARTITION BY",
type);
}
}
for (SortItem sortItem : node.getWindow().get().getOrderBy()) {
process(sortItem.getSortKey(), context);
Type type = expressionTypes.get(sortItem.getSortKey());
if (!type.isOrderable()) {
throw new SemanticException(
TYPE_MISMATCH,
node,
"%s is not orderable, and therefore cannot be used in window function ORDER BY",
type);
}
}
if (node.getWindow().get().getFrame().isPresent()) {
WindowFrame frame = node.getWindow().get().getFrame().get();
if (frame.getStart().getValue().isPresent()) {
Type type = process(frame.getStart().getValue().get(), context);
if (!type.equals(BIGINT)) {
throw new SemanticException(
TYPE_MISMATCH,
node,
"Window frame start value type must be BIGINT (actual %s)",
type);
}
}
if (frame.getEnd().isPresent() && frame.getEnd().get().getValue().isPresent()) {
Type type = process(frame.getEnd().get().getValue().get(), context);
if (!type.equals(BIGINT)) {
throw new SemanticException(
TYPE_MISMATCH,
node,
"Window frame end value type must be BIGINT (actual %s)",
type);
}
}
}
}
ImmutableList.Builder<TypeSignature> argumentTypes = ImmutableList.builder();
for (Expression expression : node.getArguments()) {
argumentTypes.add(process(expression, context).getTypeSignature());
}
Signature function =
functionRegistry.resolveFunction(
node.getName(), argumentTypes.build(), context.isApproximate());
for (int i = 0; i < node.getArguments().size(); i++) {
Expression expression = node.getArguments().get(i);
Type type = typeManager.getType(function.getArgumentTypes().get(i));
requireNonNull(type, format("Type %s not found", function.getArgumentTypes().get(i)));
if (node.isDistinct() && !type.isComparable()) {
throw new SemanticException(
TYPE_MISMATCH,
node,
"DISTINCT can only be applied to comparable types (actual: %s)",
type);
}
coerceType(
context, expression, type, String.format("Function %s argument %d", function, i));
}
resolvedFunctions.put(node, function);
Type type = typeManager.getType(function.getReturnType());
expressionTypes.put(node, type);
return type;
}