private Fields addGeneratedTermVectors(
Engine.GetResult get,
Fields termVectorsByField,
TermVectorRequest request,
Set<String> selectedFields)
throws IOException {
/* only keep valid fields */
Set<String> validFields = new HashSet<>();
for (String field : selectedFields) {
FieldMapper fieldMapper = indexShard.mapperService().smartNameFieldMapper(field);
if (!isValidField(fieldMapper)) {
continue;
}
// already retrieved, only if the analyzer hasn't been overridden at the field
if (fieldMapper.fieldType().storeTermVectors()
&& (request.perFieldAnalyzer() == null
|| !request.perFieldAnalyzer().containsKey(field))) {
continue;
}
validFields.add(field);
}
if (validFields.isEmpty()) {
return termVectorsByField;
}
/* generate term vectors from fetched document fields */
GetResult getResult =
indexShard
.getService()
.get(
get,
request.id(),
request.type(),
validFields.toArray(Strings.EMPTY_ARRAY),
null,
false);
Fields generatedTermVectors =
generateTermVectors(
getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer());
/* merge with existing Fields */
if (termVectorsByField == null) {
return generatedTermVectors;
} else {
return mergeFields(termVectorsByField, generatedTermVectors);
}
}
public static GetResult readGetResult(StreamInput in) throws IOException {
GetResult result = new GetResult();
result.readFrom(in);
return result;
}
protected void shardOperation(
final UpdateRequest request,
final ActionListener<UpdateResponse> listener,
final int retryCount)
throws ElasticSearchException {
IndexService indexService = indicesService.indexServiceSafe(request.index());
IndexShard indexShard = indexService.shardSafe(request.shardId());
long getDate = System.currentTimeMillis();
final GetResult getResult =
indexShard
.getService()
.get(
request.type(),
request.id(),
new String[] {
SourceFieldMapper.NAME,
RoutingFieldMapper.NAME,
ParentFieldMapper.NAME,
TTLFieldMapper.NAME
},
true);
// no doc, what to do, what to do...
if (!getResult.isExists()) {
if (request.upsertRequest() == null) {
listener.onFailure(
new DocumentMissingException(
new ShardId(request.index(), request.shardId()), request.type(), request.id()));
return;
}
final IndexRequest indexRequest = request.upsertRequest();
indexRequest
.index(request.index())
.type(request.type())
.id(request.id())
// it has to be a "create!"
.create(true)
.routing(request.routing())
.percolate(request.percolate())
.refresh(request.refresh())
.replicationType(request.replicationType())
.consistencyLevel(request.consistencyLevel());
indexRequest.operationThreaded(false);
// we fetch it from the index request so we don't generate the bytes twice, its already done
// in the index request
final BytesReference updateSourceBytes = indexRequest.source();
indexAction.execute(
indexRequest,
new ActionListener<IndexResponse>() {
@Override
public void onResponse(IndexResponse response) {
UpdateResponse update =
new UpdateResponse(
response.getIndex(),
response.getType(),
response.getId(),
response.getVersion());
update.setMatches(response.getMatches());
if (request.fields() != null && request.fields().length > 0) {
Tuple<XContentType, Map<String, Object>> sourceAndContent =
XContentHelper.convertToMap(updateSourceBytes, true);
update.setGetResult(
extractGetResult(
request,
response.getVersion(),
sourceAndContent.v2(),
sourceAndContent.v1(),
updateSourceBytes));
} else {
update.setGetResult(null);
}
listener.onResponse(update);
}
@Override
public void onFailure(Throwable e) {
e = ExceptionsHelper.unwrapCause(e);
if (e instanceof VersionConflictEngineException
|| e instanceof DocumentAlreadyExistsException) {
if (retryCount < request.retryOnConflict()) {
threadPool
.executor(executor())
.execute(
new Runnable() {
@Override
public void run() {
shardOperation(request, listener, retryCount + 1);
}
});
return;
}
}
listener.onFailure(e);
}
});
return;
}
if (getResult.internalSourceRef() == null) {
// no source, we can't do nothing, through a failure...
listener.onFailure(
new DocumentSourceMissingException(
new ShardId(request.index(), request.shardId()), request.type(), request.id()));
return;
}
Tuple<XContentType, Map<String, Object>> sourceAndContent =
XContentHelper.convertToMap(getResult.internalSourceRef(), true);
String operation = null;
String timestamp = null;
Long ttl = null;
Object fetchedTTL = null;
final Map<String, Object> updatedSourceAsMap;
final XContentType updateSourceContentType = sourceAndContent.v1();
String routing =
getResult.getFields().containsKey(RoutingFieldMapper.NAME)
? getResult.field(RoutingFieldMapper.NAME).getValue().toString()
: null;
String parent =
getResult.getFields().containsKey(ParentFieldMapper.NAME)
? getResult.field(ParentFieldMapper.NAME).getValue().toString()
: null;
if (request.script() == null && request.doc() != null) {
IndexRequest indexRequest = request.doc();
updatedSourceAsMap = sourceAndContent.v2();
if (indexRequest.ttl() > 0) {
ttl = indexRequest.ttl();
}
timestamp = indexRequest.timestamp();
if (indexRequest.routing() != null) {
routing = indexRequest.routing();
}
if (indexRequest.parent() != null) {
parent = indexRequest.parent();
}
XContentHelper.update(updatedSourceAsMap, indexRequest.sourceAsMap());
} else {
Map<String, Object> ctx = new HashMap<String, Object>(2);
ctx.put("_source", sourceAndContent.v2());
try {
ExecutableScript script =
scriptService.executable(request.scriptLang, request.script, request.scriptParams);
script.setNextVar("ctx", ctx);
script.run();
// we need to unwrap the ctx...
ctx = (Map<String, Object>) script.unwrap(ctx);
} catch (Exception e) {
throw new ElasticSearchIllegalArgumentException("failed to execute script", e);
}
operation = (String) ctx.get("op");
timestamp = (String) ctx.get("_timestamp");
fetchedTTL = ctx.get("_ttl");
if (fetchedTTL != null) {
if (fetchedTTL instanceof Number) {
ttl = ((Number) fetchedTTL).longValue();
} else {
ttl = TimeValue.parseTimeValue((String) fetchedTTL, null).millis();
}
}
updatedSourceAsMap = (Map<String, Object>) ctx.get("_source");
}
// apply script to update the source
// No TTL has been given in the update script so we keep previous TTL value if there is one
if (ttl == null) {
ttl =
getResult.getFields().containsKey(TTLFieldMapper.NAME)
? (Long) getResult.field(TTLFieldMapper.NAME).getValue()
: null;
if (ttl != null) {
ttl =
ttl
- (System.currentTimeMillis()
- getDate); // It is an approximation of exact TTL value, could be improved
}
}
// TODO: external version type, does it make sense here? does not seem like it...
if (operation == null || "index".equals(operation)) {
final IndexRequest indexRequest =
Requests.indexRequest(request.index())
.type(request.type())
.id(request.id())
.routing(routing)
.parent(parent)
.source(updatedSourceAsMap, updateSourceContentType)
.version(getResult.getVersion())
.replicationType(request.replicationType())
.consistencyLevel(request.consistencyLevel())
.timestamp(timestamp)
.ttl(ttl)
.percolate(request.percolate())
.refresh(request.refresh());
indexRequest.operationThreaded(false);
// we fetch it from the index request so we don't generate the bytes twice, its already done
// in the index request
final BytesReference updateSourceBytes = indexRequest.source();
indexAction.execute(
indexRequest,
new ActionListener<IndexResponse>() {
@Override
public void onResponse(IndexResponse response) {
UpdateResponse update =
new UpdateResponse(
response.getIndex(),
response.getType(),
response.getId(),
response.getVersion());
update.setMatches(response.getMatches());
update.setGetResult(
extractGetResult(
request,
response.getVersion(),
updatedSourceAsMap,
updateSourceContentType,
updateSourceBytes));
listener.onResponse(update);
}
@Override
public void onFailure(Throwable e) {
e = ExceptionsHelper.unwrapCause(e);
if (e instanceof VersionConflictEngineException) {
if (retryCount < request.retryOnConflict()) {
threadPool
.executor(executor())
.execute(
new Runnable() {
@Override
public void run() {
shardOperation(request, listener, retryCount + 1);
}
});
return;
}
}
listener.onFailure(e);
}
});
} else if ("delete".equals(operation)) {
DeleteRequest deleteRequest =
Requests.deleteRequest(request.index())
.type(request.type())
.id(request.id())
.routing(routing)
.parent(parent)
.version(getResult.getVersion())
.replicationType(request.replicationType())
.consistencyLevel(request.consistencyLevel());
deleteRequest.operationThreaded(false);
deleteAction.execute(
deleteRequest,
new ActionListener<DeleteResponse>() {
@Override
public void onResponse(DeleteResponse response) {
UpdateResponse update =
new UpdateResponse(
response.getIndex(),
response.getType(),
response.getId(),
response.getVersion());
update.setGetResult(
extractGetResult(
request,
response.getVersion(),
updatedSourceAsMap,
updateSourceContentType,
null));
listener.onResponse(update);
}
@Override
public void onFailure(Throwable e) {
e = ExceptionsHelper.unwrapCause(e);
if (e instanceof VersionConflictEngineException) {
if (retryCount < request.retryOnConflict()) {
threadPool
.executor(executor())
.execute(
new Runnable() {
@Override
public void run() {
shardOperation(request, listener, retryCount + 1);
}
});
return;
}
}
listener.onFailure(e);
}
});
} else if ("none".equals(operation)) {
UpdateResponse update =
new UpdateResponse(
getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion());
update.setGetResult(
extractGetResult(
request, getResult.getVersion(), updatedSourceAsMap, updateSourceContentType, null));
listener.onResponse(update);
} else {
logger.warn(
"Used update operation [{}] for script [{}], doing nothing...",
operation,
request.script);
listener.onResponse(
new UpdateResponse(
getResult.getIndex(),
getResult.getType(),
getResult.getId(),
getResult.getVersion()));
}
}
/**
* Prepares an update request by converting it into an index or delete request or an update
* response (no action).
*/
@SuppressWarnings("unchecked")
protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult getResult) {
long getDateNS = System.nanoTime();
if (!getResult.isExists()) {
if (request.upsertRequest() == null && !request.docAsUpsert()) {
throw new DocumentMissingException(shardId, request.type(), request.id());
}
IndexRequest indexRequest = request.docAsUpsert() ? request.doc() : request.upsertRequest();
TimeValue ttl = indexRequest.ttl();
if (request.scriptedUpsert() && request.script() != null) {
// Run the script to perform the create logic
IndexRequest upsert = request.upsertRequest();
Map<String, Object> upsertDoc = upsert.sourceAsMap();
Map<String, Object> ctx = new HashMap<>(2);
// Tell the script that this is a create and not an update
ctx.put("op", "create");
ctx.put("_source", upsertDoc);
ctx = executeScript(request.script, ctx);
// Allow the script to set TTL using ctx._ttl
if (ttl == null) {
ttl = getTTLFromScriptContext(ctx);
}
// Allow the script to abort the create by setting "op" to "none"
String scriptOpChoice = (String) ctx.get("op");
// Only valid options for an upsert script are "create"
// (the default) or "none", meaning abort upsert
if (!"create".equals(scriptOpChoice)) {
if (!"none".equals(scriptOpChoice)) {
logger.warn(
"Used upsert operation [{}] for script [{}], doing nothing...",
scriptOpChoice,
request.script.getScript());
}
UpdateResponse update =
new UpdateResponse(
shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false);
update.setGetResult(getResult);
return new Result(update, Operation.NONE, upsertDoc, XContentType.JSON);
}
indexRequest.source((Map) ctx.get("_source"));
}
indexRequest
.index(request.index())
.type(request.type())
.id(request.id())
// it has to be a "create!"
.create(true)
.ttl(ttl)
.refresh(request.refresh())
.routing(request.routing())
.parent(request.parent())
.consistencyLevel(request.consistencyLevel());
if (request.versionType() != VersionType.INTERNAL) {
// in all but the internal versioning mode, we want to create the new document using the
// given version.
indexRequest.version(request.version()).versionType(request.versionType());
}
return new Result(indexRequest, Operation.UPSERT, null, null);
}
long updateVersion = getResult.getVersion();
if (request.versionType() != VersionType.INTERNAL) {
assert request.versionType() == VersionType.FORCE;
updateVersion = request.version(); // remember, match_any is excluded by the conflict test
}
if (getResult.internalSourceRef() == null) {
// no source, we can't do nothing, through a failure...
throw new DocumentSourceMissingException(shardId, request.type(), request.id());
}
Tuple<XContentType, Map<String, Object>> sourceAndContent =
XContentHelper.convertToMap(getResult.internalSourceRef(), true);
String operation = null;
String timestamp = null;
TimeValue ttl = null;
final Map<String, Object> updatedSourceAsMap;
final XContentType updateSourceContentType = sourceAndContent.v1();
String routing =
getResult.getFields().containsKey(RoutingFieldMapper.NAME)
? getResult.field(RoutingFieldMapper.NAME).getValue().toString()
: null;
String parent =
getResult.getFields().containsKey(ParentFieldMapper.NAME)
? getResult.field(ParentFieldMapper.NAME).getValue().toString()
: null;
if (request.script() == null && request.doc() != null) {
IndexRequest indexRequest = request.doc();
updatedSourceAsMap = sourceAndContent.v2();
if (indexRequest.ttl() != null) {
ttl = indexRequest.ttl();
}
timestamp = indexRequest.timestamp();
if (indexRequest.routing() != null) {
routing = indexRequest.routing();
}
if (indexRequest.parent() != null) {
parent = indexRequest.parent();
}
boolean noop =
!XContentHelper.update(
updatedSourceAsMap, indexRequest.sourceAsMap(), request.detectNoop());
// noop could still be true even if detectNoop isn't because update detects empty maps as
// noops. BUT we can only
// actually turn the update into a noop if detectNoop is true to preserve backwards
// compatibility and to handle
// cases where users repopulating multi-fields or adding synonyms, etc.
if (request.detectNoop() && noop) {
operation = "none";
}
} else {
Map<String, Object> ctx = new HashMap<>(16);
Long originalTtl =
getResult.getFields().containsKey(TTLFieldMapper.NAME)
? (Long) getResult.field(TTLFieldMapper.NAME).getValue()
: null;
Long originalTimestamp =
getResult.getFields().containsKey(TimestampFieldMapper.NAME)
? (Long) getResult.field(TimestampFieldMapper.NAME).getValue()
: null;
ctx.put("_index", getResult.getIndex());
ctx.put("_type", getResult.getType());
ctx.put("_id", getResult.getId());
ctx.put("_version", getResult.getVersion());
ctx.put("_routing", routing);
ctx.put("_parent", parent);
ctx.put("_timestamp", originalTimestamp);
ctx.put("_ttl", originalTtl);
ctx.put("_source", sourceAndContent.v2());
ctx = executeScript(request.script, ctx);
operation = (String) ctx.get("op");
Object fetchedTimestamp = ctx.get("_timestamp");
if (fetchedTimestamp != null) {
timestamp = fetchedTimestamp.toString();
} else if (originalTimestamp != null) {
// No timestamp has been given in the update script, so we keep the previous timestamp if
// there is one
timestamp = originalTimestamp.toString();
}
ttl = getTTLFromScriptContext(ctx);
updatedSourceAsMap = (Map<String, Object>) ctx.get("_source");
}
// apply script to update the source
// No TTL has been given in the update script so we keep previous TTL value if there is one
if (ttl == null) {
Long ttlAsLong =
getResult.getFields().containsKey(TTLFieldMapper.NAME)
? (Long) getResult.field(TTLFieldMapper.NAME).getValue()
: null;
if (ttlAsLong != null) {
ttl =
new TimeValue(
ttlAsLong
- TimeValue.nsecToMSec(
System.nanoTime()
- getDateNS)); // It is an approximation of exact TTL value, could be
// improved
}
}
if (operation == null || "index".equals(operation)) {
final IndexRequest indexRequest =
Requests.indexRequest(request.index())
.type(request.type())
.id(request.id())
.routing(routing)
.parent(parent)
.source(updatedSourceAsMap, updateSourceContentType)
.version(updateVersion)
.versionType(request.versionType())
.consistencyLevel(request.consistencyLevel())
.timestamp(timestamp)
.ttl(ttl)
.refresh(request.refresh());
return new Result(indexRequest, Operation.INDEX, updatedSourceAsMap, updateSourceContentType);
} else if ("delete".equals(operation)) {
DeleteRequest deleteRequest =
Requests.deleteRequest(request.index())
.type(request.type())
.id(request.id())
.routing(routing)
.parent(parent)
.version(updateVersion)
.versionType(request.versionType())
.consistencyLevel(request.consistencyLevel());
return new Result(
deleteRequest, Operation.DELETE, updatedSourceAsMap, updateSourceContentType);
} else if ("none".equals(operation)) {
UpdateResponse update =
new UpdateResponse(
shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false);
update.setGetResult(
extractGetResult(
request,
request.index(),
getResult.getVersion(),
updatedSourceAsMap,
updateSourceContentType,
getResult.internalSourceRef()));
return new Result(update, Operation.NONE, updatedSourceAsMap, updateSourceContentType);
} else {
logger.warn(
"Used update operation [{}] for script [{}], doing nothing...",
operation,
request.script.getScript());
UpdateResponse update =
new UpdateResponse(
shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false);
return new Result(update, Operation.NONE, updatedSourceAsMap, updateSourceContentType);
}
}
/**
* Prepares an update request by converting it into an index request.
*
* <p>TODO: detect a NOOP and return an update response if true
*/
@SuppressWarnings("unchecked")
public IndexRequest prepareUpdate(
DocTableInfo tableInfo,
ShardUpsertRequest request,
ShardUpsertRequest.Item item,
ShardId shardId)
throws ElasticsearchException {
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
IndexShard indexShard = indexService.shardSafe(shardId.id());
final GetResult getResult =
indexShard
.getService()
.get(
request.type(),
item.id(),
new String[] {RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TTLFieldMapper.NAME},
true,
item.version(),
VersionType.INTERNAL,
FetchSourceContext.FETCH_SOURCE,
false);
if (!getResult.isExists()) {
throw new DocumentMissingException(
new ShardId(request.index(), request.shardId()), request.type(), item.id());
}
if (getResult.internalSourceRef() == null) {
// no source, we can't do nothing, through a failure...
throw new DocumentSourceMissingException(
new ShardId(request.index(), request.shardId()), request.type(), item.id());
}
Tuple<XContentType, Map<String, Object>> sourceAndContent =
XContentHelper.convertToMap(getResult.internalSourceRef(), true);
final Map<String, Object> updatedSourceAsMap;
final XContentType updateSourceContentType = sourceAndContent.v1();
String routing =
getResult.getFields().containsKey(RoutingFieldMapper.NAME)
? getResult.field(RoutingFieldMapper.NAME).getValue().toString()
: null;
String parent =
getResult.getFields().containsKey(ParentFieldMapper.NAME)
? getResult.field(ParentFieldMapper.NAME).getValue().toString()
: null;
updatedSourceAsMap = sourceAndContent.v2();
SymbolToFieldExtractorContext ctx =
new SymbolToFieldExtractorContext(functions, item.insertValues());
Map<String, Object> pathsToUpdate = new LinkedHashMap<>();
Map<String, Object> updatedGeneratedColumns = new LinkedHashMap<>();
for (int i = 0; i < request.updateColumns().length; i++) {
/**
* NOTE: mapping isn't applied. So if an Insert was done using the ES Rest Endpoint the data
* might be returned in the wrong format (date as string instead of long)
*/
String columnPath = request.updateColumns()[i];
Object value =
SYMBOL_TO_FIELD_EXTRACTOR.convert(item.updateAssignments()[i], ctx).extract(getResult);
ReferenceInfo referenceInfo = tableInfo.getReferenceInfo(ColumnIdent.fromPath(columnPath));
if (referenceInfo instanceof GeneratedReferenceInfo) {
updatedGeneratedColumns.put(columnPath, value);
} else {
pathsToUpdate.put(columnPath, value);
}
}
processGeneratedColumns(
tableInfo,
pathsToUpdate,
updatedGeneratedColumns,
request.validateGeneratedColumns(),
getResult);
updateSourceByPaths(updatedSourceAsMap, pathsToUpdate);
final IndexRequest indexRequest =
Requests.indexRequest(request.index())
.type(request.type())
.id(item.id())
.routing(routing)
.parent(parent)
.source(updatedSourceAsMap, updateSourceContentType)
.version(getResult.getVersion());
indexRequest.operationThreaded(false);
return indexRequest;
}