protected static void parseNested(
String name, Map<String, Object> node, ObjectMapper.Builder builder) {
boolean nested = false;
boolean nestedIncludeInParent = false;
boolean nestedIncludeInRoot = false;
Object fieldNode = node.get("type");
if (fieldNode != null) {
String type = fieldNode.toString();
if (type.equals(CONTENT_TYPE)) {
builder.nested = Nested.NO;
} else if (type.equals(NESTED_CONTENT_TYPE)) {
nested = true;
} else {
throw new MapperParsingException(
"Trying to parse an object but has a different type ["
+ type
+ "] for ["
+ name
+ "]");
}
}
fieldNode = node.get("include_in_parent");
if (fieldNode != null) {
nestedIncludeInParent = nodeBooleanValue(fieldNode);
}
fieldNode = node.get("include_in_root");
if (fieldNode != null) {
nestedIncludeInRoot = nodeBooleanValue(fieldNode);
}
if (nested) {
builder.nested = Nested.newNested(nestedIncludeInParent, nestedIncludeInRoot);
}
}
@Override
public Y build(BuilderContext context) {
ContentPath.Type origPathType = context.path().pathType();
context.path().pathType(pathType);
context.path().add(name);
Map<String, Mapper> mappers = new HashMap<String, Mapper>();
for (Mapper.Builder builder : mappersBuilders) {
Mapper mapper = builder.build(context);
mappers.put(mapper.name(), mapper);
}
context.path().pathType(origPathType);
context.path().remove();
ObjectMapper objectMapper =
createMapper(
name,
context.path().fullPathAsText(name),
enabled,
nested,
dynamic,
pathType,
mappers);
objectMapper.includeInAllIfNotSet(includeInAll);
return (Y) objectMapper;
}
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
Map<String, Object> objectNode = node;
ObjectMapper.Builder builder = createBuilder(name);
boolean nested = false;
boolean nestedIncludeInParent = false;
boolean nestedIncludeInRoot = false;
for (Map.Entry<String, Object> entry : objectNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("dynamic")) {
String value = fieldNode.toString();
if (value.equalsIgnoreCase("strict")) {
builder.dynamic(Dynamic.STRICT);
} else {
builder.dynamic(nodeBooleanValue(fieldNode) ? Dynamic.TRUE : Dynamic.FALSE);
}
} else if (fieldName.equals("type")) {
String type = fieldNode.toString();
if (type.equals(CONTENT_TYPE)) {
builder.nested = Nested.NO;
} else if (type.equals(NESTED_CONTENT_TYPE)) {
nested = true;
} else {
throw new MapperParsingException(
"Trying to parse an object but has a different type ["
+ type
+ "] for ["
+ name
+ "]");
}
} else if (fieldName.equals("include_in_parent")) {
nestedIncludeInParent = nodeBooleanValue(fieldNode);
} else if (fieldName.equals("include_in_root")) {
nestedIncludeInRoot = nodeBooleanValue(fieldNode);
} else if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("path")) {
builder.pathType(parsePathType(name, fieldNode.toString()));
} else if (fieldName.equals("properties")) {
parseProperties(builder, (Map<String, Object>) fieldNode, parserContext);
} else if (fieldName.equals("include_in_all")) {
builder.includeInAll(nodeBooleanValue(fieldNode));
} else {
processField(builder, fieldName, fieldNode);
}
}
if (nested) {
builder.nested = Nested.newNested(nestedIncludeInParent, nestedIncludeInRoot);
}
return builder;
}
@Override
public void run() {
OntologyHelper helper = helpers.get(threadKey);
if (helper != null) {
// Check if the last call time was longer ago than the maximum
if (System.currentTimeMillis() - DELETE_CHECK_DELAY_MS > helper.getLastCallTime()) {
// Assume helper is out of use - dispose of it to allow memory to be freed
helper.dispose();
helpers.remove(threadKey);
}
}
}
private Analyzer getAnalyzerAtField(
String field, @Nullable Map<String, String> perFieldAnalyzer) {
MapperService mapperService = indexShard.mapperService();
Analyzer analyzer;
if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) {
analyzer = mapperService.analysisService().analyzer(perFieldAnalyzer.get(field).toString());
} else {
analyzer = mapperService.smartNameFieldMapper(field).indexAnalyzer();
}
if (analyzer == null) {
analyzer = mapperService.analysisService().defaultIndexAnalyzer();
}
return analyzer;
}
protected boolean processField(
ObjectMapper.Builder builder, String fieldName, Object fieldNode) {
if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) {
List<FormatDateTimeFormatter> dateTimeFormatters = newArrayList();
if (fieldNode instanceof List) {
for (Object node1 : (List) fieldNode) {
dateTimeFormatters.add(parseDateTimeFormatter(fieldName, node1));
}
} else if ("none".equals(fieldNode.toString())) {
dateTimeFormatters = null;
} else {
dateTimeFormatters.add(parseDateTimeFormatter(fieldName, fieldNode));
}
if (dateTimeFormatters == null) {
((Builder) builder).noDynamicDateTimeFormatter();
} else {
((Builder) builder).dynamicDateTimeFormatter(dateTimeFormatters);
}
return true;
} else if (fieldName.equals("dynamic_templates")) {
// "dynamic_templates" : [
// {
// "template_1" : {
// "match" : "*_test",
// "match_mapping_type" : "string",
// "mapping" : { "type" : "string", "store" : "yes" }
// }
// }
// ]
List tmplNodes = (List) fieldNode;
for (Object tmplNode : tmplNodes) {
Map<String, Object> tmpl = (Map<String, Object>) tmplNode;
if (tmpl.size() != 1) {
throw new MapperParsingException("A dynamic template must be defined with a name");
}
Map.Entry<String, Object> entry = tmpl.entrySet().iterator().next();
((Builder) builder)
.add(DynamicTemplate.parse(entry.getKey(), (Map<String, Object>) entry.getValue()));
}
return true;
} else if (fieldName.equals("date_detection")) {
((Builder) builder).dateDetection = nodeBooleanValue(fieldNode);
return true;
} else if (fieldName.equals("numeric_detection")) {
((Builder) builder).numericDetection = nodeBooleanValue(fieldNode);
return true;
}
return false;
}
public static OntologyHelper getHelper(OntologySettings settings, ThreadPool threadPool)
throws OntologyHelperException {
String helperKey = buildHelperKey(settings);
OntologyHelper helper = helpers.get(helperKey);
if (helper == null) {
helper = new ElasticOntologyHelperFactory(settings).buildOntologyHelper();
OntologyCheckRunnable checker = new OntologyCheckRunnable(helperKey);
threadPool.scheduleWithFixedDelay(checker, TimeValue.timeValueMillis(DELETE_CHECK_DELAY_MS));
helpers.put(helperKey, helper);
helper.updateLastCallTime();
}
return helper;
}
@Override
public OntologyMapper build(BuilderContext context) {
ContentPath.Type origPathType = context.path().pathType();
context.path().pathType(pathType);
Map<String, FieldMapper<String>> fieldMappers = Maps.newHashMap();
context.path().add(name);
if (propertyBuilders != null) {
for (String property : propertyBuilders.keySet()) {
StringFieldMapper sfm = propertyBuilders.get(property).build(context);
fieldMappers.put(sfm.names().indexName(), sfm);
}
}
// Initialise field mappers for the pre-defined fields
for (FieldMappings mapping : ontologySettings.getFieldMappings()) {
if (!fieldMappers.containsKey(context.path().fullPathAsText(mapping.getFieldName()))) {
StringFieldMapper mapper =
MapperBuilders.stringField(mapping.getFieldName())
.store(true)
.index(true)
.tokenized(!mapping.isUriField())
.build(context);
fieldMappers.put(mapper.names().indexName(), mapper);
}
}
context.path().remove(); // remove name
context.path().pathType(origPathType);
return new OntologyMapper(
buildNames(context),
fieldType,
docValues,
indexAnalyzer,
searchAnalyzer,
postingsProvider,
docValuesProvider,
similarity,
fieldDataSettings,
context.indexSettings(),
new MultiFields.Builder().build(this, context),
ontologySettings,
fieldMappers,
threadPool);
}
private Map<String, StringFieldMapper.Builder> parseProperties(
Map<String, Object> propertiesNode, ParserContext parserContext) {
Map<String, StringFieldMapper.Builder> propertyMap = new HashMap<>();
for (Iterator<Map.Entry<String, Object>> iterator = propertiesNode.entrySet().iterator();
iterator.hasNext(); ) {
Entry<String, Object> entry = iterator.next();
String name = entry.getKey();
@SuppressWarnings("unchecked")
Mapper.Builder builder =
new StringFieldMapper.TypeParser()
.parse(entry.getKey(), (Map<String, Object>) entry.getValue(), parserContext);
propertyMap.put(name, (StringFieldMapper.Builder) builder);
}
return propertyMap;
}
private Set<String> getFieldsToGenerate(
Map<String, String> perAnalyzerField, Fields fieldsObject) {
Set<String> selectedFields = new HashSet<>();
for (String fieldName : fieldsObject) {
if (perAnalyzerField.containsKey(fieldName)) {
selectedFields.add(fieldName);
}
}
return selectedFields;
}
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
ObjectMapper.Builder builder = createBuilder(name);
for (Map.Entry<String, Object> entry : node.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder);
parseObjectProperties(name, fieldName, fieldNode, builder);
}
parseNested(name, node, builder);
return builder;
}
private Mapper.Builder<?, ?> findMapperBuilder(
Map<String, Object> propNode, String propName, ParserContext parserContext) {
String type;
Object typeNode = propNode.get("type");
if (typeNode != null) {
type = typeNode.toString();
} else {
type = "string";
}
Mapper.TypeParser typeParser = parserContext.typeParser(type);
Mapper.Builder<?, ?> mapperBuilder =
typeParser.parse(propName, (Map<String, Object>) propNode, parserContext);
return mapperBuilder;
}
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
ObjectMapper.Builder builder = createBuilder(name);
Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)
|| processField(builder, fieldName, fieldNode)) {
iterator.remove();
}
}
return builder;
}
protected static void parseProperties(
ObjectMapper.Builder objBuilder,
Map<String, Object> propsNode,
ParserContext parserContext) {
for (Map.Entry<String, Object> entry : propsNode.entrySet()) {
String propName = entry.getKey();
// Should accept empty arrays, as a work around for when the user can't provide an empty
// Map. (PHP for example)
boolean isEmptyList =
entry.getValue() instanceof List && ((List<?>) entry.getValue()).isEmpty();
if (entry.getValue() instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, Object> propNode = (Map<String, Object>) entry.getValue();
String type;
Object typeNode = propNode.get("type");
if (typeNode != null) {
type = typeNode.toString();
} else {
// lets see if we can derive this...
if (propNode.get("properties") != null) {
type = ObjectMapper.CONTENT_TYPE;
} else if (propNode.size() == 1 && propNode.get("enabled") != null) {
// if there is a single property with the enabled flag on it, make it an object
// (usually, setting enabled to false to not index any type, including core values,
// which
// non enabled object type supports).
type = ObjectMapper.CONTENT_TYPE;
} else {
throw new MapperParsingException("No type specified for property [" + propName + "]");
}
}
Mapper.TypeParser typeParser = parserContext.typeParser(type);
if (typeParser == null) {
throw new MapperParsingException(
"No handler for type [" + type + "] declared on field [" + propName + "]");
}
objBuilder.add(typeParser.parse(propName, propNode, parserContext));
} else if (!isEmptyList) {
throw new MapperParsingException(
"Expected map for property [fields] on field ["
+ propName
+ "] but got a "
+ propName.getClass());
}
}
}
/**
* Parse the mapping definition for the ontology type.
*
* @param name the field name
* @param node the JSON node holding the mapping definitions.
* @param parserContext the parser context object.
* @return a Builder for an OntologyMapper.
*/
@SuppressWarnings("unchecked")
@Override
public Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
OntologySettings ontologySettings = null;
Builder builder = new Builder(name, threadPool);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Entry<String, Object> entry = iterator.next();
if (entry.getKey().equals(OntologySettings.ONTOLOGY_SETTINGS_KEY)) {
ontologySettings =
new OntologySettingsBuilder()
.settingsNode((Map<String, Object>) entry.getValue())
.build();
iterator.remove();
} else if (entry.getKey().equals(ONTOLOGY_PROPERTIES)) {
Map<String, StringFieldMapper.Builder> builders =
parseProperties((Map<String, Object>) entry.getValue(), parserContext);
builder.propertyBuilders(builders);
iterator.remove();
}
}
if (ontologySettings == null) {
throw new MapperParsingException("No ontology settings supplied");
} else if (StringUtils.isBlank(ontologySettings.getOntologyUri())
&& StringUtils.isBlank(ontologySettings.getOlsBaseUrl())) {
throw new MapperParsingException("No ontology URI or OLS details supplied");
} else {
builder = builder.ontologySettings(ontologySettings);
}
return builder;
}
private void parseProperties(
ObjectMapper.Builder objBuilder,
Map<String, Object> propsNode,
ParserContext parserContext) {
for (Map.Entry<String, Object> entry : propsNode.entrySet()) {
String propName = entry.getKey();
Map<String, Object> propNode = (Map<String, Object>) entry.getValue();
String type;
Object typeNode = propNode.get("type");
if (typeNode != null) {
type = typeNode.toString();
} else {
// lets see if we can derive this...
if (propNode.get("properties") != null) {
type = ObjectMapper.CONTENT_TYPE;
} else if (propNode.get("fields") != null) {
type = MultiFieldMapper.CONTENT_TYPE;
} else if (propNode.size() == 1 && propNode.get("enabled") != null) {
// if there is a single property with the enabled flag on it, make it an object
// (usually, setting enabled to false to not index any type, including core values,
// which
// non enabled object type supports).
type = ObjectMapper.CONTENT_TYPE;
} else {
throw new MapperParsingException("No type specified for property [" + propName + "]");
}
}
Mapper.TypeParser typeParser = parserContext.typeParser(type);
if (typeParser == null) {
throw new MapperParsingException(
"No handler for type [" + type + "] declared on field [" + propName + "]");
}
objBuilder.add(typeParser.parse(propName, propNode, parserContext));
}
}
@Override
public Iterator<String> iterator() {
return Collections.unmodifiableSet(fields.keySet()).iterator();
}
void addField(String fieldName, Terms terms) {
fields.put(fieldName, terms);
}
@Override
public Terms terms(String field) {
return fields.get(field);
}
@SuppressWarnings({"unchecked"})
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
AttachmentMapper.Builder builder = new AttachmentMapper.Builder(name);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = entry.getKey();
Object fieldNode = entry.getValue();
if (fieldName.equals("path")
&& parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) {
builder.pathType(parsePathType(name, fieldNode.toString()));
iterator.remove();
} else if (fieldName.equals("fields")) {
Map<String, Object> fieldsNode = (Map<String, Object>) fieldNode;
for (Iterator<Map.Entry<String, Object>> fieldsIterator =
fieldsNode.entrySet().iterator();
fieldsIterator.hasNext(); ) {
Map.Entry<String, Object> entry1 = fieldsIterator.next();
String propName = entry1.getKey();
Map<String, Object> propNode = (Map<String, Object>) entry1.getValue();
Mapper.Builder<?, ?> mapperBuilder =
findMapperBuilder(propNode, propName, parserContext);
if (parseMultiField(
(FieldMapper.Builder) mapperBuilder,
fieldName,
parserContext,
propName,
propNode)) {
fieldsIterator.remove();
} else if (propName.equals(name)
&& parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) {
builder.content(mapperBuilder);
fieldsIterator.remove();
} else {
switch (propName) {
case FieldNames.CONTENT:
builder.content(mapperBuilder);
fieldsIterator.remove();
break;
case FieldNames.DATE:
builder.date(mapperBuilder);
fieldsIterator.remove();
break;
case FieldNames.AUTHOR:
builder.author(mapperBuilder);
fieldsIterator.remove();
break;
case FieldNames.CONTENT_LENGTH:
builder.contentLength(mapperBuilder);
fieldsIterator.remove();
break;
case FieldNames.CONTENT_TYPE:
builder.contentType(mapperBuilder);
fieldsIterator.remove();
break;
case FieldNames.KEYWORDS:
builder.keywords(mapperBuilder);
fieldsIterator.remove();
break;
case FieldNames.LANGUAGE:
builder.language(mapperBuilder);
fieldsIterator.remove();
break;
case FieldNames.TITLE:
builder.title(mapperBuilder);
fieldsIterator.remove();
break;
case FieldNames.NAME:
builder.name(mapperBuilder);
fieldsIterator.remove();
break;
}
}
}
DocumentMapperParser.checkNoRemainingFields(
fieldName, fieldsNode, parserContext.indexVersionCreated());
iterator.remove();
}
}
return builder;
}
@Override
public int size() {
return fields.size();
}
