public static class Transformers {
public static CollectionTransformer<ScoredDocument, String> ToIds =
Expressive.Transformers.transformAllUsing(
new ETransformer<ScoredDocument, String>() {
@Override
public String from(ScoredDocument from) {
return from.getId();
}
});
}
/**
* A common base class for {@link TextSearchService} and {@link IdTextSearchService} implementations
* that use the Appengine Full Text Search API.
*
* @param <T>
* @param <K>
* @see GaeSearchService
* @see IdGaeSearchService
*/
public abstract class BaseGaeSearchService<T, K> implements SearchExecutor<T, K, SearchImpl<T, K>> {
private static final int IntLow = Integer.MIN_VALUE + 1;
private static final int IntHigh = Integer.MAX_VALUE;
private static final Date DateLow = new Date(0);
private static final Date DateHigh = new Date(Long.MAX_VALUE);
private static final String StringLow = "";
private static final String StringHigh = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
private static final Map<Is, String> IsSymbols = createIsSymbolMap();
protected Class<T> type;
protected SearchMetadata<T, K> metadata;
protected FieldMediatorSet fieldMediators;
protected TransformerManager transformerManager;
protected IndexTypeLookup indexTypeLookup;
protected IndexNamingStrategy indexNamingStrategy;
protected String indexName;
protected BaseGaeSearchService(
Class<T> type, SearchConfig searchConfig, IndexNamingStrategy indexNamingStrategy) {
this(
type,
new SearchMetadata<T, K>(type, searchConfig.getIndexTypeLookup()),
searchConfig,
indexNamingStrategy);
}
protected BaseGaeSearchService(
Class<T> type,
Class<K> keyType,
SearchConfig searchConfig,
IndexNamingStrategy indexNamingStrategy) {
this(
type,
new SearchMetadata<T, K>(type, keyType, searchConfig.getIndexTypeLookup()),
searchConfig,
indexNamingStrategy);
}
protected BaseGaeSearchService(
Class<T> type,
SearchMetadata<T, K> metadata,
SearchConfig searchConfig,
IndexNamingStrategy indexNamingStrategy) {
this.type = type;
this.fieldMediators = searchConfig.getFieldMediators();
this.transformerManager = searchConfig.getTransformerManager();
this.indexTypeLookup = searchConfig.getIndexTypeLookup();
this.indexNamingStrategy = indexNamingStrategy;
this.indexName = indexNamingStrategy.getName(type);
this.metadata = metadata;
}
public boolean hasIndexableFields() {
return metadata.hasIndexableFields();
}
protected IndexOperation index(T object, K id) {
Map<String, Object> data = metadata.getData(object);
Document document = buildDocument(id, data);
Future<PutResponse> putAsync = getIndex().putAsync(document);
return new IndexOperation(putAsync);
}
protected IndexOperation index(Collection<T> objects) {
if (Expressive.isEmpty(objects)) {
return new IndexOperation(null);
}
List<Document> documents = new ArrayList<Document>(objects.size());
for (T t : objects) {
Map<String, Object> data = metadata.getData(t);
K id = metadata.getId(t);
Document document = buildDocument(id, data);
documents.add(document);
}
return new IndexOperation(getIndex().putAsync(documents));
}
protected IndexOperation index(Map<K, T> objects) {
if (Expressive.isEmpty(objects)) {
return new IndexOperation(null);
}
List<Document> documents = new ArrayList<Document>(objects.size());
for (Map.Entry<K, T> entry : objects.entrySet()) {
Map<String, Object> data = metadata.getData(entry.getValue());
Document document = buildDocument(entry.getKey(), data);
documents.add(document);
}
return new IndexOperation(getIndex().putAsync(documents));
}
protected IndexOperation removeById(K id) {
String stringId = convert(id, String.class);
Future<Void> deleteAsync = getIndex().deleteAsync(stringId);
return new IndexOperation(deleteAsync);
}
protected IndexOperation removeById(Iterable<K> ids) {
List<String> stringIds = convert(ids, String.class);
Future<Void> deleteAsync = getIndex().deleteAsync(stringIds);
return new IndexOperation(deleteAsync);
}
protected int removeAll() {
int count = 0;
Index index = getIndex();
GetRequest request = GetRequest.newBuilder().setReturningIdsOnly(true).setLimit(200).build();
GetResponse<Document> response = index.getRange(request);
// can only delete documents in blocks of 200 so we need to iterate until they're all gone
while (!response.getResults().isEmpty()) {
List<String> ids = new ArrayList<String>();
for (Document document : response) {
ids.add(document.getId());
}
index.delete(ids);
count += ids.size();
response = index.getRange(request);
}
return count;
}
protected SearchImpl<T, K> search() {
return new SearchImpl<T, K>(this);
}
@Override
public Result<T, K> createSearchResult(SearchImpl<T, K> searchRequest) {
String queryString = buildQueryString(searchRequest.query());
SortOptions.Builder sortOptions = buildSortOptions(searchRequest.order());
QueryOptions.Builder queryOptions = QueryOptions.newBuilder();
Integer limit = searchRequest.limit();
int offset = 0;
if (limit != null) {
offset = searchRequest.offset() == null ? 0 : searchRequest.offset();
int effectiveLimit = limit + offset;
if (effectiveLimit > 1000) {
Logger.warn(
"Currently the Google Search API does not support queries with a limit over 1000. With an offset of %d and a limit of %d, you have an effective limit of %d",
offset, limit, effectiveLimit);
}
limit = effectiveLimit;
/* Note, this can't be more than 1000 (Crashes) */
queryOptions = queryOptions.setLimit(limit);
}
if (searchRequest.accuracy() != null) {
queryOptions.setNumberFoundAccuracy(searchRequest.accuracy());
}
queryOptions.setSortOptions(sortOptions);
Query query = Query.newBuilder().setOptions(queryOptions).build(queryString);
Future<Results<ScoredDocument>> searchAsync = getIndex().searchAsync(query);
return new ResultImpl<T, K>(this, searchAsync, searchRequest.offset());
}
protected SortOptions.Builder buildSortOptions(List<OrderComponent> order) {
SortOptions.Builder sortOptions = SortOptions.newBuilder();
for (OrderComponent sort : order) {
String fieldName = getEncodedFieldName(sort.getField());
SortExpression.Builder expression = SortExpression.newBuilder().setExpression(fieldName);
expression =
expression.setDirection(
sort.isAscending()
? SortExpression.SortDirection.ASCENDING
: SortExpression.SortDirection.DESCENDING);
IndexType indexType = metadata.getIndexType(sort.getField());
if (IndexType.SmallDecimal == indexType || IndexType.BigDecimal == indexType) {
expression = expression.setDefaultValueNumeric(sort.isDescending() ? IntLow : IntHigh);
} else if (IndexType.Date == indexType) {
expression = expression.setDefaultValueDate(sort.isDescending() ? DateLow : DateHigh);
} else {
expression = expression.setDefaultValue(sort.isDescending() ? StringLow : StringHigh);
}
sortOptions = sortOptions.addSortExpression(expression);
}
return sortOptions;
}
protected String buildQueryString(List<QueryComponent> queryComponents) {
List<String> stringQueryComponents = new ArrayList<>();
for (QueryComponent queryComponent : queryComponents) {
String fragmentString = convertQueryComponentToQueryFragment(queryComponent);
stringQueryComponents.add(fragmentString);
}
return StringUtils.join(stringQueryComponents, " ");
}
protected String convertQueryComponentToQueryFragment(QueryComponent queryComponent) {
if (!queryComponent.isFieldedQuery()) {
return queryComponent.getQuery();
}
String field = this.getEncodedFieldName(queryComponent.getField());
if (field == null) {
throw new SearchException(
"Unable to build query string - there is no field named '%s' on %s",
queryComponent.getField(), type.getSimpleName());
}
String operation = IsSymbols.get(queryComponent.getIs());
if (queryComponent.isCollectionQuery()) {
List<String> values = convertValuesToString(field, queryComponent.getCollectionValue());
String stringValue = StringUtils.join(values, " OR ");
return String.format("%s:(%s)", field, stringValue);
} else {
String value = convertValueToString(field, queryComponent.getValue());
return String.format("%s%s%s", field, operation, value);
}
}
protected Document buildDocument(K id, Map<String, Object> fields) {
String stringId = convert(id, String.class);
Builder documentBuilder = Document.newBuilder();
documentBuilder.setId(stringId);
for (Map.Entry<String, Object> fieldData : fields.entrySet()) {
Object value = fieldData.getValue();
String fieldName = fieldData.getKey();
for (Object object : getCollectionValues(value)) {
try {
Field field = buildField(metadata, fieldName, object);
documentBuilder.addField(field);
} catch (Exception e) {
throw new SearchException(
e,
"Failed to add field '%s' with value '%s' to document with id '%s': %s",
fieldName,
value.toString(),
id,
e.getMessage());
}
}
}
return documentBuilder.build();
}
<F> Field buildField(SearchMetadata<T, K> metadata, String field, Object value) {
com.google.appengine.api.search.Field.Builder fieldBuilder =
Field.newBuilder().setName(metadata.getEncodedFieldName(field));
IndexType indexType = metadata.getIndexType(field);
FieldMediator<F> fieldMediator = fieldMediators.get(indexType);
F normalised = fieldMediator.normalise(transformerManager, value);
fieldMediator.setValue(fieldBuilder, normalised);
return fieldBuilder.build();
}
@SuppressWarnings("unchecked")
<O, R> R convert(O input, Class<R> type) {
Class<O> inputType = (Class<O>) input.getClass();
if (inputType == type) {
return (R) input;
}
ETransformer<O, R> transformer = transformerManager.getTransformerSafe(inputType, type);
return transformer.from(input);
}
@SuppressWarnings("unchecked")
<O, R> List<R> convert(Iterable<O> inputs, Class<R> type) {
Iterator<O> iterator = inputs.iterator();
if (!iterator.hasNext()) {
return Collections.emptyList();
}
List<R> results = new ArrayList<R>();
ETransformer<O, R> transformer = null;
for (O t : inputs) {
if (transformer == null) {
Class<O> inputType = (Class<O>) t.getClass();
transformer = transformerManager.getTransformerSafe(inputType, type);
}
results.add(transformer.from(t));
}
return results;
}
/**
* We treat all values as collections. Nulls are treated as an empty collection, Non-collections
* are treated as a collection of length 1
*
* @param value
* @return
*/
@SuppressWarnings("unchecked")
private Collection<Object> getCollectionValues(Object value) {
if (value == null) {
return Collections.emptyList();
}
Collection<Object> collection = Cast.as(value, Collection.class);
return collection == null ? Collections.singleton(value) : collection;
}
@Override
public List<K> getResultsAsIds(List<ScoredDocument> results) {
EList<String> stringIds = Transformers.ToIds.from(results);
return transformerManager.transformAll(String.class, metadata.getKeyType(), stringIds);
}
@Override
public List<T> getResults(List<ScoredDocument> results) {
EList<Map<String, Object>> data = toMap.from(results);
EList<T> objects = toBean.from(data);
return objects;
}
protected String convertValueToString(String field, Object value) {
IndexType indexType = metadata.getIndexType(field);
FieldMediator<?> indexTypeFieldBuilder = fieldMediators.get(indexType);
return convertSingleValueToString(field, value, metadata, indexTypeFieldBuilder);
}
protected List<String> convertValuesToString(String field, Collection<Object> values) {
IndexType indexType = metadata.getIndexType(field);
FieldMediator<?> indexTypeFieldBuilder = fieldMediators.get(indexType);
List<String> stringValues = new ArrayList<>();
for (Object value : values) {
stringValues.add(convertSingleValueToString(field, value, metadata, indexTypeFieldBuilder));
}
return stringValues;
}
protected String getEncodedFieldName(String field) {
return metadata.getEncodedFieldName(field);
}
/** Extension point allowing the Index implementation to be modified */
protected Index getIndex() {
SearchService searchService = SearchServiceFactory.getSearchService();
return searchService.getIndex(IndexSpec.newBuilder().setName(indexName));
}
private <V> String convertSingleValueToString(
String field, Object value, SearchMetadata<T, ?> metadata, FieldMediator<V> fieldMediator) {
try {
V normalised = fieldMediator.normalise(transformerManager, value);
return fieldMediator.stringify(normalised);
} catch (Exception e) {
throw new SearchException(
"Cannot query the field %s %s - cannot convert the query value %s %s to a %s. You can register extra conversions using the %s",
metadata.getFieldType(field).getSimpleName(),
field,
value.getClass().getSimpleName(),
value,
fieldMediator.getTargetType().getSimpleName(),
TransformerManager.class.getSimpleName());
}
}
private static Map<Is, String> createIsSymbolMap() {
Map<Is, String> map = new HashMap<>();
map.put(Is.EqualTo, "=");
map.put(Is.GreaterThan, ">");
map.put(Is.GreaterThanOrEqualTo, ">=");
map.put(Is.Is, ":");
map.put(Is.LessThan, "<");
map.put(Is.LessThanOrEqualTo, "<=");
map.put(Is.Like, ":~");
return map;
}
public static class Transformers {
public static CollectionTransformer<ScoredDocument, String> ToIds =
Expressive.Transformers.transformAllUsing(
new ETransformer<ScoredDocument, String>() {
@Override
public String from(ScoredDocument from) {
return from.getId();
}
});
}
// TODO - NAO - This is a very naive implementation, a wider more flexible strategy would work
// well here
private CollectionTransformer<Map<String, Object>, T> toBean =
Expressive.Transformers.transformAllUsing(
new ETransformer<Map<String, Object>, T>() {
@Override
public T from(Map<String, Object> from) {
try {
T instance = type.newInstance();
for (Map.Entry<String, Object> entry : from.entrySet()) {
String field = metadata.getDecodedFieldName(entry.getKey());
BeanUtil.setDeclaredPropertyForcedSilent(instance, field, entry.getValue());
}
return instance;
} catch (Exception e) {
throw new SearchException(
e,
"Failed to create a new instance of %s for search results: %s",
type.getName(),
e.getMessage());
}
}
});
private CollectionTransformer<ScoredDocument, Map<String, Object>> toMap =
Expressive.Transformers.transformAllUsing(
new ETransformer<ScoredDocument, Map<String, Object>>() {
@Override
public Map<String, Object> from(ScoredDocument from) {
Map<String, Object> results = new HashMap<String, Object>();
results.put("___id___", from.getId());
for (Field field : from.getFields()) {
FieldType fieldType = field.getType();
Object value = null;
if (FieldType.TEXT.equals(fieldType)) {
value = field.getText();
} else if (FieldType.NUMBER.equals(fieldType)) {
value = field.getNumber();
} else if (FieldType.DATE.equals(fieldType)) {
value = field.getDate();
} else if (FieldType.ATOM.equals(fieldType)) {
value = field.getAtom();
} else if (FieldType.HTML.equals(fieldType)) {
value = field.getHTML();
} else if (FieldType.GEO_POINT.equals(fieldType)) {
value = field.getGeoPoint();
}
results.put(field.getName(), value);
}
return results;
}
});
}