@Test public void delete_active_rules_from_profile() throws Exception { esSetup .client() .prepareBulk() // On profile 1 .add( Requests.indexRequest() .index("rules") .type("active_rule") .parent("1") .source(testFileAsString("delete_from_profile/active_rule25.json"))) .add( Requests.indexRequest() .index("rules") .type("active_rule") .parent("3") .source(testFileAsString("delete_from_profile/active_rule2702.json"))) // On profile 2 .add( Requests.indexRequest() .index("rules") .type("active_rule") .parent("2") .source(testFileAsString("delete_from_profile/active_rule523.json"))) .setRefresh(true) .execute() .actionGet(); esActiveRule.deleteActiveRulesFromProfile(1); assertThat(!esSetup.exists("rules", "active_rule", "25")); assertThat(!esSetup.exists("rules", "active_rule", "2702")); assertThat(esSetup.exists("rules", "active_rule", "523")); }
@Override public void handleRequest(final InteractiveRequest request, final InteractiveChannel channel) { String index = request.paramAsString("index"); String type = request.paramAsString("type"); String id = request.paramAsString("id"); try { if (index == null) { channel.sendResponse(TYPE, new IllegalArgumentException("index is null")); return; } if (type == null) { channel.sendResponse(TYPE, new IllegalArgumentException("type is null")); return; } if (id == null) { channel.sendResponse(TYPE, new IllegalArgumentException("id is null")); return; } IndexRequest indexRequest = Requests.indexRequest(index) .type(type) .id(id) .source((Map<String, Object>) request.asMap().get("data")); add(indexRequest, channel); } catch (IOException ex) { try { channel.sendResponse(TYPE, ex); } catch (IOException ex1) { logger.error("error while sending exception"); } } }
@Override public void run() { while (!closed) { try { for (int i = 0; i < simpleNumber; i++) { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.field(fieldName, i); builder.endObject(); currentRequest.add( Requests.indexRequest(indexName) .type(typeName) .id(UUID.randomUUID().toString()) .create(true) .source(builder)); processBulkIfNeeded(); } if (currentRequest.numberOfActions() > 0) { currentRequest.execute().get(); currentRequest = client.prepareBulk(); } delay(); } catch (Exception e) { logger.error(e.getMessage(), e, (Object) null); closed = true; } if (closed) { return; } } }
@Override public void run() { try { barrier1.await(); barrier2.await(); for (; counter < max; counter++) { Client client = client(counter); long id = idGenerator.incrementAndGet(); client .index( Requests.indexRequest() .index("test") .type("type1") .id(Long.toString(id)) .source( XContentFactory.jsonBuilder() .startObject() .field("num", id % fieldNumLimit) .endObject())) .actionGet(); } System.out.println("Indexer [" + id + "]: Done"); } catch (Exception e) { System.err.println("Failed to index:"); e.printStackTrace(); } finally { latch.countDown(); } }
@Override public void run() { while (!closed) { try { for (int i = 0; i < simpleNumber; i++) { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.field(fieldName, i); builder.endObject(); bulkProcessor.add( Requests.indexRequest(indexName) .type(typeName) .id(UUID.randomUUID().toString()) .create(true) .source(builder)); } // in this case we force the bulking, but it should seldom done bulkProcessor.flush(); delay(); } catch (Exception e) { logger.error(e.getMessage(), e, (Object) null); closed = true; } if (closed) { return; } } }
private IndexRequest prepareInsert( DocTableInfo tableInfo, ShardUpsertRequest request, ShardUpsertRequest.Item item) throws IOException { List<GeneratedReferenceInfo> generatedReferencesWithValue = new ArrayList<>(); BytesReference source; if (request.isRawSourceInsert()) { assert item.insertValues().length > 0 : "empty insert values array"; source = new BytesArray((BytesRef) item.insertValues()[0]); } else { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); for (int i = 0; i < item.insertValues().length; i++) { Reference ref = request.insertColumns()[i]; if (ref.info().granularity() == RowGranularity.DOC) { // don't include values for partitions in the _source // ideally columns with partition granularity shouldn't be part of the request builder.field(ref.ident().columnIdent().fqn(), item.insertValues()[i]); if (ref.info() instanceof GeneratedReferenceInfo) { generatedReferencesWithValue.add((GeneratedReferenceInfo) ref.info()); } } } source = builder.bytes(); } int generatedColumnSize = 0; for (GeneratedReferenceInfo generatedReferenceInfo : tableInfo.generatedColumns()) { if (!tableInfo.partitionedByColumns().contains(generatedReferenceInfo)) { generatedColumnSize++; } } int numMissingGeneratedColumns = generatedColumnSize - generatedReferencesWithValue.size(); if (numMissingGeneratedColumns > 0 || (generatedReferencesWithValue.size() > 0 && request.validateGeneratedColumns())) { // we need to evaluate some generated column expressions Map<String, Object> sourceMap = processGeneratedColumnsOnInsert( tableInfo, request.insertColumns(), item.insertValues(), request.isRawSourceInsert(), request.validateGeneratedColumns()); source = XContentFactory.jsonBuilder().map(sourceMap).bytes(); } IndexRequest indexRequest = Requests.indexRequest(request.index()) .type(request.type()) .id(item.id()) .routing(request.routing()) .source(source) .create(!request.overwriteDuplicates()) .operationThreaded(false); if (logger.isTraceEnabled()) { logger.trace( "Inserting document with id {}, source: {}", item.id(), indexRequest.source().toUtf8()); } return indexRequest; }
@Test public void should_delete_from_integer_ids() throws Exception { esSetup .client() .prepareBulk() .add( Requests.indexRequest() .index("rules") .type("active_rule") .parent("1") .source(testFileAsString("delete_from_profile/active_rule25.json"))) .add( Requests.indexRequest() .index("rules") .type("active_rule") .parent("3") .source(testFileAsString("delete_from_profile/active_rule2702.json"))) .setRefresh(true) .execute() .actionGet(); esActiveRule.deleteActiveRules(ImmutableList.of(25, 2702)); }
@Test public void testCopyHeadersRequest() { Map<String, String> transportHeaders = randomHeaders(randomIntBetween(0, 10)); Map<String, String> restHeaders = randomHeaders(randomIntBetween(0, 10)); Map<String, String> copiedHeaders = randomHeadersFrom(restHeaders); Set<String> usefulRestHeaders = new HashSet<>(copiedHeaders.keySet()); usefulRestHeaders.addAll(randomMap(randomIntBetween(0, 10), "useful-").keySet()); Map<String, String> restContext = randomContext(randomIntBetween(0, 10)); Map<String, String> transportContext = Maps.difference(randomContext(randomIntBetween(0, 10)), restContext).entriesOnlyOnLeft(); Map<String, String> expectedHeaders = new HashMap<>(); expectedHeaders.putAll(transportHeaders); expectedHeaders.putAll(copiedHeaders); Map<String, String> expectedContext = new HashMap<>(); expectedContext.putAll(transportContext); expectedContext.putAll(restContext); Client client = client(new NoOpClient(), new FakeRestRequest(restHeaders, restContext), usefulRestHeaders); SearchRequest searchRequest = Requests.searchRequest(); putHeaders(searchRequest, transportHeaders); putContext(searchRequest, transportContext); assertHeaders(searchRequest, transportHeaders); client.search(searchRequest); assertHeaders(searchRequest, expectedHeaders); assertContext(searchRequest, expectedContext); GetRequest getRequest = Requests.getRequest("index"); putHeaders(getRequest, transportHeaders); putContext(getRequest, transportContext); assertHeaders(getRequest, transportHeaders); client.get(getRequest); assertHeaders(getRequest, expectedHeaders); assertContext(getRequest, expectedContext); IndexRequest indexRequest = Requests.indexRequest(); putHeaders(indexRequest, transportHeaders); putContext(indexRequest, transportContext); assertHeaders(indexRequest, transportHeaders); client.index(indexRequest); assertHeaders(indexRequest, expectedHeaders); assertContext(indexRequest, expectedContext); }
/** * Updates the specified objects * * @return the id's of the failed objects (e.g. due to versioning) */ public Collection<Integer> bulkUpdate( Collection<T> objects, String indexName, boolean refresh, boolean enableVersioning) { // now using bulk API instead of feeding each doc separate with feedDoc BulkRequestBuilder brb = client.prepareBulk(); // this works differently then the direct call to refresh!? maybe refresh is not async? // brb.setRefresh(refresh); for (T o : objects) { if (o.getId() == null) { logger.warn("Skipped object without id when bulkUpdate:" + o); continue; } try { XContentBuilder source = createDoc(o); IndexRequest indexReq = Requests.indexRequest(indexName).type(getIndexType()).id(o.getId()).source(source); if (enableVersioning) indexReq.version(o.getVersion()); brb.add(indexReq); } catch (IOException ex) { logger.warn("Cannot add object:" + o + " to bulkIndexing action." + ex.getMessage()); } } if (brb.numberOfActions() > 0) { BulkResponse rsp = brb.execute().actionGet(); if (rsp.hasFailures()) { List<Integer> list = new ArrayList<Integer>(rsp.items().length); for (BulkItemResponse br : rsp.items()) { if (br.isFailed()) { // logger.info("Error:" + br.failureMessage()); list.add(br.itemId()); } } return list; } if (refresh) refresh(indexName); } return Collections.emptyList(); }
@Test @Slow public void testSnapshotOperations() throws Exception { startNode("server1", getClassDefaultSettings()); // get the environment, so we can clear the work dir when needed Environment environment = ((InternalNode) node("server1")).injector().getInstance(Environment.class); logger.info("Running Cluster Health (waiting for node to startup properly)"); ClusterHealthResponse clusterHealth = client("server1") .admin() .cluster() .health(clusterHealthRequest().waitForGreenStatus()) .actionGet(); logger.info("Done Cluster Health, status " + clusterHealth.getStatus()); assertThat(clusterHealth.isTimedOut(), equalTo(false)); assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN)); // Translog tests logger.info("Creating index [{}]", "test"); client("server1").admin().indices().prepareCreate("test").execute().actionGet(); // create a mapping PutMappingResponse putMappingResponse = client("server1") .admin() .indices() .preparePutMapping("test") .setType("type1") .setSource(mappingSource()) .execute() .actionGet(); assertThat(putMappingResponse.isAcknowledged(), equalTo(true)); // verify that mapping is there ClusterStateResponse clusterState = client("server1").admin().cluster().state(clusterStateRequest()).actionGet(); assertThat(clusterState.getState().metaData().index("test").mapping("type1"), notNullValue()); // create two and delete the first logger.info("Indexing #1"); client("server1") .index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"))) .actionGet(); logger.info("Indexing #2"); client("server1") .index(Requests.indexRequest("test").type("type1").id("2").source(source("2", "test"))) .actionGet(); // perform snapshot to the index logger.info("Gateway Snapshot"); client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet(); logger.info("Deleting #1"); client("server1").delete(deleteRequest("test").type("type1").id("1")).actionGet(); // perform snapshot to the index logger.info("Gateway Snapshot"); client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet(); logger.info("Gateway Snapshot (should be a no op)"); // do it again, it should be a no op client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet(); logger.info("Closing the server"); closeNode("server1"); logger.info( "Starting the server, should recover from the gateway (only translog should be populated)"); startNode("server1"); logger.info("Running Cluster Health (wait for the shards to startup)"); clusterHealth = client("server1") .admin() .cluster() .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1)) .actionGet(); logger.info("Done Cluster Health, status " + clusterHealth.getStatus()); assertThat(clusterHealth.isTimedOut(), equalTo(false)); assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); // verify that mapping is there clusterState = client("server1").admin().cluster().state(clusterStateRequest()).actionGet(); assertThat(clusterState.getState().metaData().index("test").mapping("type1"), notNullValue()); logger.info("Getting #1, should not exists"); GetResponse getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet(); assertThat(getResponse.isExists(), equalTo(false)); logger.info("Getting #2"); getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet(); assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test"))); // Now flush and add some data (so we have index recovery as well) logger.info( "Flushing, so we have actual content in the index files (#2 should be in the index)"); client("server1").admin().indices().flush(flushRequest("test")).actionGet(); logger.info("Indexing #3, so we have something in the translog as well"); client("server1") .index(Requests.indexRequest("test").type("type1").id("3").source(source("3", "test"))) .actionGet(); logger.info("Gateway Snapshot"); client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet(); logger.info("Gateway Snapshot (should be a no op)"); client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet(); logger.info("Closing the server"); closeNode("server1"); logger.info( "Starting the server, should recover from the gateway (both index and translog) and reuse work dir"); startNode("server1"); logger.info("Running Cluster Health (wait for the shards to startup)"); clusterHealth = client("server1") .admin() .cluster() .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1)) .actionGet(); logger.info("Done Cluster Health, status " + clusterHealth.getStatus()); assertThat(clusterHealth.isTimedOut(), equalTo(false)); assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); logger.info("Getting #1, should not exists"); getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet(); assertThat(getResponse.isExists(), equalTo(false)); logger.info("Getting #2 (not from the translog, but from the index)"); getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet(); assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test"))); logger.info("Getting #3 (from the translog)"); getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet(); assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test"))); logger.info("Closing the server"); closeNode("server1"); logger.info("Clearing cluster data dir, so there will be a full recovery from the gateway"); FileSystemUtils.deleteRecursively(environment.dataWithClusterFiles()); logger.info( "Starting the server, should recover from the gateway (both index and translog) without reusing work dir"); startNode("server1"); logger.info("Running Cluster Health (wait for the shards to startup)"); clusterHealth = client("server1") .admin() .cluster() .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1)) .actionGet(); logger.info("Done Cluster Health, status " + clusterHealth.getStatus()); assertThat(clusterHealth.isTimedOut(), equalTo(false)); assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); logger.info("Getting #1, should not exists"); getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet(); assertThat(getResponse.isExists(), equalTo(false)); logger.info("Getting #2 (not from the translog, but from the index)"); getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet(); assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test"))); logger.info("Getting #3 (from the translog)"); getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet(); assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test"))); logger.info( "Flushing, so we have actual content in the index files (#3 should be in the index now as well)"); client("server1").admin().indices().flush(flushRequest("test")).actionGet(); logger.info("Gateway Snapshot"); client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet(); logger.info("Gateway Snapshot (should be a no op)"); client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet(); logger.info("Closing the server"); closeNode("server1"); logger.info( "Starting the server, should recover from the gateway (just from the index, nothing in the translog)"); startNode("server1"); logger.info("Running Cluster Health (wait for the shards to startup)"); clusterHealth = client("server1") .admin() .cluster() .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1)) .actionGet(); logger.info("Done Cluster Health, status " + clusterHealth.getStatus()); assertThat(clusterHealth.isTimedOut(), equalTo(false)); assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); logger.info("Getting #1, should not exists"); getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet(); assertThat(getResponse.isExists(), equalTo(false)); logger.info("Getting #2 (not from the translog, but from the index)"); getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet(); assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test"))); logger.info("Getting #3 (not from the translog, but from the index)"); getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet(); assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test"))); logger.info("Deleting the index"); client("server1").admin().indices().delete(deleteIndexRequest("test")).actionGet(); }
public static void main(String[] args) throws Exception { Natives.tryMlockall(); Settings settings = settingsBuilder() .put("refresh_interval", "-1") .put(SETTING_NUMBER_OF_SHARDS, 1) .put(SETTING_NUMBER_OF_REPLICAS, 0) .build(); String clusterName = TermsAggregationSearchAndIndexingBenchmark.class.getSimpleName(); nodes = new InternalNode[1]; for (int i = 0; i < nodes.length; i++) { nodes[i] = (InternalNode) nodeBuilder() .settings(settingsBuilder().put(settings).put("name", "node1")) .clusterName(clusterName) .node(); } Client client = nodes[0].client(); client .admin() .cluster() .prepareHealth(indexName) .setWaitForGreenStatus() .setTimeout("10s") .execute() .actionGet(); try { client .admin() .indices() .prepareCreate(indexName) .addMapping(typeName, generateMapping("eager", "lazy")) .get(); Thread.sleep(5000); long startTime = System.currentTimeMillis(); ObjectOpenHashSet<String> uniqueTerms = ObjectOpenHashSet.newInstance(); for (int i = 0; i < NUMBER_OF_TERMS; i++) { boolean added; do { added = uniqueTerms.add(RandomStrings.randomAsciiOfLength(random, STRING_TERM_SIZE)); } while (!added); } String[] sValues = uniqueTerms.toArray(String.class); long ITERS = COUNT / BATCH; long i = 1; int counter = 0; for (; i <= ITERS; i++) { BulkRequestBuilder request = client.prepareBulk(); for (int j = 0; j < BATCH; j++) { counter++; XContentBuilder builder = jsonBuilder().startObject(); builder.field("id", Integer.toString(counter)); final String sValue = sValues[counter % sValues.length]; builder.field("s_value", sValue); builder.field("s_value_dv", sValue); for (String field : new String[] {"sm_value", "sm_value_dv"}) { builder.startArray(field); for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) { builder.value(sValues[ThreadLocalRandom.current().nextInt(sValues.length)]); } builder.endArray(); } request.add( Requests.indexRequest(indexName) .type("type1") .id(Integer.toString(counter)) .source(builder)); } BulkResponse response = request.execute().actionGet(); if (response.hasFailures()) { System.err.println("--> failures..."); } if (((i * BATCH) % 10000) == 0) { System.out.println("--> Indexed " + (i * BATCH)); } } System.out.println( "--> Indexing took " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds."); } catch (IndexAlreadyExistsException e) { System.out.println("--> Index already exists, ignoring indexing phase, waiting for green"); ClusterHealthResponse clusterHealthResponse = client .admin() .cluster() .prepareHealth(indexName) .setWaitForGreenStatus() .setTimeout("10m") .execute() .actionGet(); if (clusterHealthResponse.isTimedOut()) { System.err.println("--> Timed out waiting for cluster health"); } } client .admin() .indices() .preparePutMapping(indexName) .setType(typeName) .setSource(generateMapping("lazy", "lazy")) .get(); client.admin().indices().prepareRefresh().execute().actionGet(); System.out.println( "--> Number of docs in index: " + client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount()); String[] nodeIds = new String[nodes.length]; for (int i = 0; i < nodeIds.length; i++) { nodeIds[i] = nodes[i].injector().getInstance(Discovery.class).localNode().getId(); } List<TestRun> testRuns = new ArrayList<>(); testRuns.add(new TestRun("Regular field ordinals", "eager", "lazy", "s_value", "ordinals")); testRuns.add( new TestRun("Docvalues field ordinals", "lazy", "eager", "s_value_dv", "ordinals")); testRuns.add( new TestRun( "Regular field global ordinals", "eager_global_ordinals", "lazy", "s_value", null)); testRuns.add( new TestRun("Docvalues field global", "lazy", "eager_global_ordinals", "s_value_dv", null)); List<TestResult> testResults = new ArrayList<>(); for (TestRun testRun : testRuns) { client .admin() .indices() .preparePutMapping(indexName) .setType(typeName) .setSource( generateMapping(testRun.indexedFieldEagerLoading, testRun.docValuesEagerLoading)) .get(); client.admin().indices().prepareClearCache(indexName).setFieldDataCache(true).get(); SearchThread searchThread = new SearchThread(client, testRun.termsAggsField, testRun.termsAggsExecutionHint); RefreshThread refreshThread = new RefreshThread(client); System.out.println("--> Running '" + testRun.name + "' round..."); new Thread(refreshThread).start(); new Thread(searchThread).start(); Thread.sleep(2 * 60 * 1000); refreshThread.stop(); searchThread.stop(); System.out.println("--> Avg refresh time: " + refreshThread.avgRefreshTime + " ms"); System.out.println("--> Avg query time: " + searchThread.avgQueryTime + " ms"); ClusterStatsResponse clusterStateResponse = client.admin().cluster().prepareClusterStats().setNodesIds(nodeIds).get(); System.out.println( "--> Heap used: " + clusterStateResponse.getNodesStats().getJvm().getHeapUsed()); ByteSizeValue fieldDataMemoryUsed = clusterStateResponse.getIndicesStats().getFieldData().getMemorySize(); System.out.println("--> Fielddata memory size: " + fieldDataMemoryUsed); testResults.add( new TestResult( testRun.name, refreshThread.avgRefreshTime, searchThread.avgQueryTime, fieldDataMemoryUsed)); } System.out.println( "----------------------------------------- SUMMARY ----------------------------------------------"); System.out.format( Locale.ENGLISH, "%30s%18s%15s%15s\n", "name", "avg refresh time", "avg query time", "fieldata size"); for (TestResult testResult : testResults) { System.out.format( Locale.ENGLISH, "%30s%18s%15s%15s\n", testResult.name, testResult.avgRefreshTime, testResult.avgQueryTime, testResult.fieldDataSizeInMemory); } System.out.println( "----------------------------------------- SUMMARY ----------------------------------------------"); client.close(); for (InternalNode node : nodes) { node.close(); } }
/** * Prepares an update request by converting it into an index or delete request or an update * response (no action). */ @SuppressWarnings("unchecked") protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult getResult) { long getDateNS = System.nanoTime(); if (!getResult.isExists()) { if (request.upsertRequest() == null && !request.docAsUpsert()) { throw new DocumentMissingException(shardId, request.type(), request.id()); } IndexRequest indexRequest = request.docAsUpsert() ? request.doc() : request.upsertRequest(); TimeValue ttl = indexRequest.ttl(); if (request.scriptedUpsert() && request.script() != null) { // Run the script to perform the create logic IndexRequest upsert = request.upsertRequest(); Map<String, Object> upsertDoc = upsert.sourceAsMap(); Map<String, Object> ctx = new HashMap<>(2); // Tell the script that this is a create and not an update ctx.put("op", "create"); ctx.put("_source", upsertDoc); ctx = executeScript(request.script, ctx); // Allow the script to set TTL using ctx._ttl if (ttl == null) { ttl = getTTLFromScriptContext(ctx); } // Allow the script to abort the create by setting "op" to "none" String scriptOpChoice = (String) ctx.get("op"); // Only valid options for an upsert script are "create" // (the default) or "none", meaning abort upsert if (!"create".equals(scriptOpChoice)) { if (!"none".equals(scriptOpChoice)) { logger.warn( "Used upsert operation [{}] for script [{}], doing nothing...", scriptOpChoice, request.script.getScript()); } UpdateResponse update = new UpdateResponse( shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false); update.setGetResult(getResult); return new Result(update, Operation.NONE, upsertDoc, XContentType.JSON); } indexRequest.source((Map) ctx.get("_source")); } indexRequest .index(request.index()) .type(request.type()) .id(request.id()) // it has to be a "create!" .create(true) .ttl(ttl) .refresh(request.refresh()) .routing(request.routing()) .parent(request.parent()) .consistencyLevel(request.consistencyLevel()); if (request.versionType() != VersionType.INTERNAL) { // in all but the internal versioning mode, we want to create the new document using the // given version. indexRequest.version(request.version()).versionType(request.versionType()); } return new Result(indexRequest, Operation.UPSERT, null, null); } long updateVersion = getResult.getVersion(); if (request.versionType() != VersionType.INTERNAL) { assert request.versionType() == VersionType.FORCE; updateVersion = request.version(); // remember, match_any is excluded by the conflict test } if (getResult.internalSourceRef() == null) { // no source, we can't do nothing, through a failure... throw new DocumentSourceMissingException(shardId, request.type(), request.id()); } Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true); String operation = null; String timestamp = null; TimeValue ttl = null; final Map<String, Object> updatedSourceAsMap; final XContentType updateSourceContentType = sourceAndContent.v1(); String routing = getResult.getFields().containsKey(RoutingFieldMapper.NAME) ? getResult.field(RoutingFieldMapper.NAME).getValue().toString() : null; String parent = getResult.getFields().containsKey(ParentFieldMapper.NAME) ? getResult.field(ParentFieldMapper.NAME).getValue().toString() : null; if (request.script() == null && request.doc() != null) { IndexRequest indexRequest = request.doc(); updatedSourceAsMap = sourceAndContent.v2(); if (indexRequest.ttl() != null) { ttl = indexRequest.ttl(); } timestamp = indexRequest.timestamp(); if (indexRequest.routing() != null) { routing = indexRequest.routing(); } if (indexRequest.parent() != null) { parent = indexRequest.parent(); } boolean noop = !XContentHelper.update( updatedSourceAsMap, indexRequest.sourceAsMap(), request.detectNoop()); // noop could still be true even if detectNoop isn't because update detects empty maps as // noops. BUT we can only // actually turn the update into a noop if detectNoop is true to preserve backwards // compatibility and to handle // cases where users repopulating multi-fields or adding synonyms, etc. if (request.detectNoop() && noop) { operation = "none"; } } else { Map<String, Object> ctx = new HashMap<>(16); Long originalTtl = getResult.getFields().containsKey(TTLFieldMapper.NAME) ? (Long) getResult.field(TTLFieldMapper.NAME).getValue() : null; Long originalTimestamp = getResult.getFields().containsKey(TimestampFieldMapper.NAME) ? (Long) getResult.field(TimestampFieldMapper.NAME).getValue() : null; ctx.put("_index", getResult.getIndex()); ctx.put("_type", getResult.getType()); ctx.put("_id", getResult.getId()); ctx.put("_version", getResult.getVersion()); ctx.put("_routing", routing); ctx.put("_parent", parent); ctx.put("_timestamp", originalTimestamp); ctx.put("_ttl", originalTtl); ctx.put("_source", sourceAndContent.v2()); ctx = executeScript(request.script, ctx); operation = (String) ctx.get("op"); Object fetchedTimestamp = ctx.get("_timestamp"); if (fetchedTimestamp != null) { timestamp = fetchedTimestamp.toString(); } else if (originalTimestamp != null) { // No timestamp has been given in the update script, so we keep the previous timestamp if // there is one timestamp = originalTimestamp.toString(); } ttl = getTTLFromScriptContext(ctx); updatedSourceAsMap = (Map<String, Object>) ctx.get("_source"); } // apply script to update the source // No TTL has been given in the update script so we keep previous TTL value if there is one if (ttl == null) { Long ttlAsLong = getResult.getFields().containsKey(TTLFieldMapper.NAME) ? (Long) getResult.field(TTLFieldMapper.NAME).getValue() : null; if (ttlAsLong != null) { ttl = new TimeValue( ttlAsLong - TimeValue.nsecToMSec( System.nanoTime() - getDateNS)); // It is an approximation of exact TTL value, could be // improved } } if (operation == null || "index".equals(operation)) { final IndexRequest indexRequest = Requests.indexRequest(request.index()) .type(request.type()) .id(request.id()) .routing(routing) .parent(parent) .source(updatedSourceAsMap, updateSourceContentType) .version(updateVersion) .versionType(request.versionType()) .consistencyLevel(request.consistencyLevel()) .timestamp(timestamp) .ttl(ttl) .refresh(request.refresh()); return new Result(indexRequest, Operation.INDEX, updatedSourceAsMap, updateSourceContentType); } else if ("delete".equals(operation)) { DeleteRequest deleteRequest = Requests.deleteRequest(request.index()) .type(request.type()) .id(request.id()) .routing(routing) .parent(parent) .version(updateVersion) .versionType(request.versionType()) .consistencyLevel(request.consistencyLevel()); return new Result( deleteRequest, Operation.DELETE, updatedSourceAsMap, updateSourceContentType); } else if ("none".equals(operation)) { UpdateResponse update = new UpdateResponse( shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false); update.setGetResult( extractGetResult( request, request.index(), getResult.getVersion(), updatedSourceAsMap, updateSourceContentType, getResult.internalSourceRef())); return new Result(update, Operation.NONE, updatedSourceAsMap, updateSourceContentType); } else { logger.warn( "Used update operation [{}] for script [{}], doing nothing...", operation, request.script.getScript()); UpdateResponse update = new UpdateResponse( shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false); return new Result(update, Operation.NONE, updatedSourceAsMap, updateSourceContentType); } }
protected void shardOperation( final UpdateRequest request, final ActionListener<UpdateResponse> listener, final int retryCount) throws ElasticSearchException { IndexService indexService = indicesService.indexServiceSafe(request.index()); IndexShard indexShard = indexService.shardSafe(request.shardId()); long getDate = System.currentTimeMillis(); final GetResult getResult = indexShard .getService() .get( request.type(), request.id(), new String[] { SourceFieldMapper.NAME, RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TTLFieldMapper.NAME }, true); // no doc, what to do, what to do... if (!getResult.isExists()) { if (request.upsertRequest() == null) { listener.onFailure( new DocumentMissingException( new ShardId(request.index(), request.shardId()), request.type(), request.id())); return; } final IndexRequest indexRequest = request.upsertRequest(); indexRequest .index(request.index()) .type(request.type()) .id(request.id()) // it has to be a "create!" .create(true) .routing(request.routing()) .percolate(request.percolate()) .refresh(request.refresh()) .replicationType(request.replicationType()) .consistencyLevel(request.consistencyLevel()); indexRequest.operationThreaded(false); // we fetch it from the index request so we don't generate the bytes twice, its already done // in the index request final BytesReference updateSourceBytes = indexRequest.source(); indexAction.execute( indexRequest, new ActionListener<IndexResponse>() { @Override public void onResponse(IndexResponse response) { UpdateResponse update = new UpdateResponse( response.getIndex(), response.getType(), response.getId(), response.getVersion()); update.setMatches(response.getMatches()); if (request.fields() != null && request.fields().length > 0) { Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(updateSourceBytes, true); update.setGetResult( extractGetResult( request, response.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), updateSourceBytes)); } else { update.setGetResult(null); } listener.onResponse(update); } @Override public void onFailure(Throwable e) { e = ExceptionsHelper.unwrapCause(e); if (e instanceof VersionConflictEngineException || e instanceof DocumentAlreadyExistsException) { if (retryCount < request.retryOnConflict()) { threadPool .executor(executor()) .execute( new Runnable() { @Override public void run() { shardOperation(request, listener, retryCount + 1); } }); return; } } listener.onFailure(e); } }); return; } if (getResult.internalSourceRef() == null) { // no source, we can't do nothing, through a failure... listener.onFailure( new DocumentSourceMissingException( new ShardId(request.index(), request.shardId()), request.type(), request.id())); return; } Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true); String operation = null; String timestamp = null; Long ttl = null; Object fetchedTTL = null; final Map<String, Object> updatedSourceAsMap; final XContentType updateSourceContentType = sourceAndContent.v1(); String routing = getResult.getFields().containsKey(RoutingFieldMapper.NAME) ? getResult.field(RoutingFieldMapper.NAME).getValue().toString() : null; String parent = getResult.getFields().containsKey(ParentFieldMapper.NAME) ? getResult.field(ParentFieldMapper.NAME).getValue().toString() : null; if (request.script() == null && request.doc() != null) { IndexRequest indexRequest = request.doc(); updatedSourceAsMap = sourceAndContent.v2(); if (indexRequest.ttl() > 0) { ttl = indexRequest.ttl(); } timestamp = indexRequest.timestamp(); if (indexRequest.routing() != null) { routing = indexRequest.routing(); } if (indexRequest.parent() != null) { parent = indexRequest.parent(); } XContentHelper.update(updatedSourceAsMap, indexRequest.sourceAsMap()); } else { Map<String, Object> ctx = new HashMap<String, Object>(2); ctx.put("_source", sourceAndContent.v2()); try { ExecutableScript script = scriptService.executable(request.scriptLang, request.script, request.scriptParams); script.setNextVar("ctx", ctx); script.run(); // we need to unwrap the ctx... ctx = (Map<String, Object>) script.unwrap(ctx); } catch (Exception e) { throw new ElasticSearchIllegalArgumentException("failed to execute script", e); } operation = (String) ctx.get("op"); timestamp = (String) ctx.get("_timestamp"); fetchedTTL = ctx.get("_ttl"); if (fetchedTTL != null) { if (fetchedTTL instanceof Number) { ttl = ((Number) fetchedTTL).longValue(); } else { ttl = TimeValue.parseTimeValue((String) fetchedTTL, null).millis(); } } updatedSourceAsMap = (Map<String, Object>) ctx.get("_source"); } // apply script to update the source // No TTL has been given in the update script so we keep previous TTL value if there is one if (ttl == null) { ttl = getResult.getFields().containsKey(TTLFieldMapper.NAME) ? (Long) getResult.field(TTLFieldMapper.NAME).getValue() : null; if (ttl != null) { ttl = ttl - (System.currentTimeMillis() - getDate); // It is an approximation of exact TTL value, could be improved } } // TODO: external version type, does it make sense here? does not seem like it... if (operation == null || "index".equals(operation)) { final IndexRequest indexRequest = Requests.indexRequest(request.index()) .type(request.type()) .id(request.id()) .routing(routing) .parent(parent) .source(updatedSourceAsMap, updateSourceContentType) .version(getResult.getVersion()) .replicationType(request.replicationType()) .consistencyLevel(request.consistencyLevel()) .timestamp(timestamp) .ttl(ttl) .percolate(request.percolate()) .refresh(request.refresh()); indexRequest.operationThreaded(false); // we fetch it from the index request so we don't generate the bytes twice, its already done // in the index request final BytesReference updateSourceBytes = indexRequest.source(); indexAction.execute( indexRequest, new ActionListener<IndexResponse>() { @Override public void onResponse(IndexResponse response) { UpdateResponse update = new UpdateResponse( response.getIndex(), response.getType(), response.getId(), response.getVersion()); update.setMatches(response.getMatches()); update.setGetResult( extractGetResult( request, response.getVersion(), updatedSourceAsMap, updateSourceContentType, updateSourceBytes)); listener.onResponse(update); } @Override public void onFailure(Throwable e) { e = ExceptionsHelper.unwrapCause(e); if (e instanceof VersionConflictEngineException) { if (retryCount < request.retryOnConflict()) { threadPool .executor(executor()) .execute( new Runnable() { @Override public void run() { shardOperation(request, listener, retryCount + 1); } }); return; } } listener.onFailure(e); } }); } else if ("delete".equals(operation)) { DeleteRequest deleteRequest = Requests.deleteRequest(request.index()) .type(request.type()) .id(request.id()) .routing(routing) .parent(parent) .version(getResult.getVersion()) .replicationType(request.replicationType()) .consistencyLevel(request.consistencyLevel()); deleteRequest.operationThreaded(false); deleteAction.execute( deleteRequest, new ActionListener<DeleteResponse>() { @Override public void onResponse(DeleteResponse response) { UpdateResponse update = new UpdateResponse( response.getIndex(), response.getType(), response.getId(), response.getVersion()); update.setGetResult( extractGetResult( request, response.getVersion(), updatedSourceAsMap, updateSourceContentType, null)); listener.onResponse(update); } @Override public void onFailure(Throwable e) { e = ExceptionsHelper.unwrapCause(e); if (e instanceof VersionConflictEngineException) { if (retryCount < request.retryOnConflict()) { threadPool .executor(executor()) .execute( new Runnable() { @Override public void run() { shardOperation(request, listener, retryCount + 1); } }); return; } } listener.onFailure(e); } }); } else if ("none".equals(operation)) { UpdateResponse update = new UpdateResponse( getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion()); update.setGetResult( extractGetResult( request, getResult.getVersion(), updatedSourceAsMap, updateSourceContentType, null)); listener.onResponse(update); } else { logger.warn( "Used update operation [{}] for script [{}], doing nothing...", operation, request.script); listener.onResponse( new UpdateResponse( getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion())); } }
/** * Prepares an update request by converting it into an index request. * * <p>TODO: detect a NOOP and return an update response if true */ @SuppressWarnings("unchecked") public IndexRequest prepareUpdate( DocTableInfo tableInfo, ShardUpsertRequest request, ShardUpsertRequest.Item item, ShardId shardId) throws ElasticsearchException { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.shardSafe(shardId.id()); final GetResult getResult = indexShard .getService() .get( request.type(), item.id(), new String[] {RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TTLFieldMapper.NAME}, true, item.version(), VersionType.INTERNAL, FetchSourceContext.FETCH_SOURCE, false); if (!getResult.isExists()) { throw new DocumentMissingException( new ShardId(request.index(), request.shardId()), request.type(), item.id()); } if (getResult.internalSourceRef() == null) { // no source, we can't do nothing, through a failure... throw new DocumentSourceMissingException( new ShardId(request.index(), request.shardId()), request.type(), item.id()); } Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true); final Map<String, Object> updatedSourceAsMap; final XContentType updateSourceContentType = sourceAndContent.v1(); String routing = getResult.getFields().containsKey(RoutingFieldMapper.NAME) ? getResult.field(RoutingFieldMapper.NAME).getValue().toString() : null; String parent = getResult.getFields().containsKey(ParentFieldMapper.NAME) ? getResult.field(ParentFieldMapper.NAME).getValue().toString() : null; updatedSourceAsMap = sourceAndContent.v2(); SymbolToFieldExtractorContext ctx = new SymbolToFieldExtractorContext(functions, item.insertValues()); Map<String, Object> pathsToUpdate = new LinkedHashMap<>(); Map<String, Object> updatedGeneratedColumns = new LinkedHashMap<>(); for (int i = 0; i < request.updateColumns().length; i++) { /** * NOTE: mapping isn't applied. So if an Insert was done using the ES Rest Endpoint the data * might be returned in the wrong format (date as string instead of long) */ String columnPath = request.updateColumns()[i]; Object value = SYMBOL_TO_FIELD_EXTRACTOR.convert(item.updateAssignments()[i], ctx).extract(getResult); ReferenceInfo referenceInfo = tableInfo.getReferenceInfo(ColumnIdent.fromPath(columnPath)); if (referenceInfo instanceof GeneratedReferenceInfo) { updatedGeneratedColumns.put(columnPath, value); } else { pathsToUpdate.put(columnPath, value); } } processGeneratedColumns( tableInfo, pathsToUpdate, updatedGeneratedColumns, request.validateGeneratedColumns(), getResult); updateSourceByPaths(updatedSourceAsMap, pathsToUpdate); final IndexRequest indexRequest = Requests.indexRequest(request.index()) .type(request.type()) .id(item.id()) .routing(routing) .parent(parent) .source(updatedSourceAsMap, updateSourceContentType) .version(getResult.getVersion()); indexRequest.operationThreaded(false); return indexRequest; }