@Override public String importQuestionnareOds(String filename, String questionnaireName) throws Exception { SpreadsheetDocument document = SpreadsheetDocument.loadDocument(new File(filename)); Table sheet = document.getSheetByIndex(0); List<String> questions = new ArrayList<>(); for (int i = 1; i < sheet.getRowList().size(); i++) { Row row = sheet.getRowList().get(i); String question = getCellStringValue(row, 0); if (StringUtils.isBlank(question)) { break; } String levelString = getCellStringValue(row, 1); long level = Long.valueOf(StringUtils.replace(levelString, "D", "")); String tagsString = getCellStringValue(row, 2); List<String> tags = Collections.emptyList(); if (StringUtils.isNotBlank(tagsString)) { tags = Lists.newArrayList(StringUtils.split(tagsString, ", ")); } String tip = StringUtils.defaultIfBlank(getCellStringValue(row, 3), null); XContentBuilder builder = jsonBuilder() .startObject() .field("title", question) .field("level", level) .field("tags", tags) .field("tip", tip) .endObject(); IndexResponse indexResponse = client .prepareIndex(domainResolver.resolveQuestionIndex(), Types.question) .setSource(builder) .execute() .actionGet(); questions.add(indexResponse.getId()); } XContentBuilder questionnaireBuilder = jsonBuilder() .startObject() .field("name", questionnaireName) .field("questions", questions) .endObject(); IndexResponse indexResponse = client .prepareIndex(domainResolver.resolveQuestionIndex(), Types.questionnaire) .setSource(questionnaireBuilder) .execute() .actionGet(); return indexResponse.getId(); }
@Override public void notify(DelegateExecution execution) throws Exception { HistoryService historyService = execution.getProcessEngineServices().getHistoryService(); HistoricDecisionInstance historicDecisionInstance = historyService .createHistoricDecisionInstanceQuery() .includeInputs() .includeOutputs() .decisionDefinitionKey((String) execution.getVariableLocal("tableName")) .processInstanceId(execution.getProcessInstanceId()) .singleResult(); // Fill a new object with stuff... FraudScoreTableObject fraudData = new FraudScoreTableObject(); fraudData.setFraudInstanceID(historicDecisionInstance.getId()); List<HistoricDecisionInputInstance> inputs = historicDecisionInstance.getInputs(); for (HistoricDecisionInputInstance historicDecisionInputInstance : inputs) { String inputName = historicDecisionInputInstance.getTypeName(); if (inputName.equals("paymentRejected")) { fraudData.setPaymentRejected((Boolean) historicDecisionInputInstance.getValue()); } else if (inputName.equals("numberOfPayouts")) { fraudData.setNumberOfPayouts((Integer) historicDecisionInputInstance.getValue()); } else if (inputName.equals("historyOfFraud")) { fraudData.setHistoryOfFraud((Boolean) historicDecisionInputInstance.getValue()); } else if (inputName.equals("claimAmount")) { fraudData.setCalimAmount((Long) historicDecisionInputInstance.getValue()); } } List<HistoricDecisionOutputInstance> outputs = historicDecisionInstance.getOutputs(); for (HistoricDecisionOutputInstance historicDecisionOutputInstance : outputs) { String inputName = historicDecisionOutputInstance.getTypeName(); if (inputName.equals("frausScore")) { fraudData.setFraudScore((Integer) historicDecisionOutputInstance.getValue()); } } ObjectMapper mapper = new ObjectMapper(); String serializedHistoricDecisionInstance = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(fraudData); Client client = TransportClient.builder() .build() .addTransportAddress( new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300)); IndexResponse response = client .prepareIndex("camunda", "fraudData", historicDecisionInstance.getId()) .setSource(serializedHistoricDecisionInstance) .get(); LOGGER.info(response.getId()); }
@Test public void testIndexAndDelete() throws Exception { prepareIndex(1); IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}").get(); assertShardInfo(indexResponse); DeleteResponse deleteResponse = client().prepareDelete("idx", "type", indexResponse.getId()).get(); assertShardInfo(deleteResponse); }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("id="); sb.append(id); sb.append(" version="); sb.append(version); sb.append(" delete?="); sb.append(delete); sb.append(" threadID="); sb.append(threadID); sb.append(" indexStartTime="); sb.append(indexStartTime); sb.append(" indexFinishTime="); sb.append(indexFinishTime); sb.append(" versionConflict="); sb.append(versionConflict); sb.append(" alreadyExists?="); sb.append(alreadyExists); if (response != null) { if (response instanceof DeleteResponse) { DeleteResponse deleteResponse = (DeleteResponse) response; sb.append(" response:"); sb.append(" index="); sb.append(deleteResponse.getIndex()); sb.append(" id="); sb.append(deleteResponse.getId()); sb.append(" type="); sb.append(deleteResponse.getType()); sb.append(" version="); sb.append(deleteResponse.getVersion()); sb.append(" found="); sb.append(deleteResponse.getOperation() == DocWriteResponse.Operation.DELETE); } else if (response instanceof IndexResponse) { IndexResponse indexResponse = (IndexResponse) response; sb.append(" index="); sb.append(indexResponse.getIndex()); sb.append(" id="); sb.append(indexResponse.getId()); sb.append(" type="); sb.append(indexResponse.getType()); sb.append(" version="); sb.append(indexResponse.getVersion()); sb.append(" created="); sb.append(indexResponse.getOperation() == DocWriteResponse.Operation.CREATE); } else { sb.append(" response: " + response); } } else { sb.append(" response: null"); } return sb.toString(); }
public static String save( ElasticSearchContextListener es, String userId, String memeId, String value) { try { String pendingItemId = getPendingItemId(es, userId, memeId); Map<String, Object> data = getRatingsObjectMap(userId, memeId, value); if (pendingItemId != null) { UpdateResponse response = es.updateRequest(INDEX_NAME, pendingItemId, data).actionGet(); return response.getId(); } else { IndexResponse response = es.indexRequest(INDEX_NAME, data).actionGet(); return response.getId(); } } catch (Exception e) { return null; } }
/** * Saves an entity into elastic search. * * @param object The entity to save. */ public void save(T object) { try { IndexRequestBuilder indexRequestBuilder = object.getId() == null ? client.prepareIndex(index, entity) : client.prepareIndex(index, entity, object.getId()); IndexResponse response = indexRequestBuilder.setSource(mapper.writeValueAsString(object)).execute().actionGet(); // Force refresh. object.setId(response.getId()); client.admin().indices().prepareRefresh(index).execute().actionGet(); if (logger.isDebugEnabled()) { BytesStreamOutput out = new BytesStreamOutput(1024 * 1024); response.writeTo(out); logger.debug("Obtained response: {}", new String(out.bytes().array())); } } catch (ElasticSearchException | IOException e) { logger.error("Error while saving entity", e); } }
public void testIndexGetAndDelete() throws ExecutionException, InterruptedException { createIndexWithAlias(); ensureYellow("test"); int numDocs = iterations(10, 50); for (int i = 0; i < numDocs; i++) { IndexResponse indexResponse = client() .prepareIndex(indexOrAlias(), "type", Integer.toString(i)) .setSource("field", "value-" + i) .get(); assertThat(indexResponse.isCreated(), equalTo(true)); assertThat(indexResponse.getIndex(), equalTo("test")); assertThat(indexResponse.getType(), equalTo("type")); assertThat(indexResponse.getId(), equalTo(Integer.toString(i))); } refresh(); String docId = Integer.toString(randomIntBetween(0, numDocs - 1)); GetResponse getResponse = client().prepareGet(indexOrAlias(), "type", docId).get(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getIndex(), equalTo("test")); assertThat(getResponse.getType(), equalTo("type")); assertThat(getResponse.getId(), equalTo(docId)); DeleteResponse deleteResponse = client().prepareDelete(indexOrAlias(), "type", docId).get(); assertThat(deleteResponse.isFound(), equalTo(true)); assertThat(deleteResponse.getIndex(), equalTo("test")); assertThat(deleteResponse.getType(), equalTo("type")); assertThat(deleteResponse.getId(), equalTo(docId)); getResponse = client().prepareGet(indexOrAlias(), "type", docId).get(); assertThat(getResponse.isExists(), equalTo(false)); refresh(); SearchResponse searchResponse = client().prepareSearch(indexOrAlias()).get(); assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1)); }
@Test public void testIndex() { IndexResponse response = null; try { response = client .prepareIndex("asdf2014", "asdf", "1") .setSource( jsonBuilder().startObject().field("age", 22).field("sex", "male").endObject()) .execute() .actionGet(); } catch (ElasticsearchException | IOException e) { e.printStackTrace(); throw new RuntimeException(e); } System.out.println("Index: " + response.getIndex()); System.out.println("Type: " + response.getType()); System.out.println("Id: " + response.getId()); System.err.println("Version: " + response.getVersion()); System.out.println("IsCreated: " + response.isCreated()); }
@Override protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary( ClusterState clusterState, PrimaryOperationRequest shardRequest) { final BulkShardRequest request = shardRequest.request; final IndexService indexService = indicesService.indexServiceSafe(request.index()); final IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id()); long[] preVersions = new long[request.items().length]; VersionType[] preVersionTypes = new VersionType[request.items().length]; Translog.Location location = null; for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) { BulkItemRequest item = request.items()[requestIndex]; if (item.request() instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) item.request(); preVersions[requestIndex] = indexRequest.version(); preVersionTypes[requestIndex] = indexRequest.versionType(); try { WriteResult<IndexResponse> result = shardIndexOperation(request, indexRequest, clusterState, indexShard, true); location = locationToSync(location, result.location); // add the response IndexResponse indexResponse = result.response(); setResponse( item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse)); } catch (Throwable e) { // rethrow the failure if we are going to retry on primary and let parent failure to // handle it if (retryPrimaryException(e)) { // restore updated versions... for (int j = 0; j < requestIndex; j++) { applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); } throw (ElasticsearchException) e; } if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) { logger.trace( "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest); } else { logger.debug( "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest); } // if its a conflict failure, and we already executed the request on a primary (and we // execute it // again, due to primary relocation and only processing up to N bulk items when the shard // gets closed) // then just use the response we got from the successful execution if (item.getPrimaryResponse() != null && isConflictException(e)) { setResponse(item, item.getPrimaryResponse()); } else { setResponse( item, new BulkItemResponse( item.id(), indexRequest.opType().lowercase(), new BulkItemResponse.Failure( request.index(), indexRequest.type(), indexRequest.id(), e))); } } } else if (item.request() instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) item.request(); preVersions[requestIndex] = deleteRequest.version(); preVersionTypes[requestIndex] = deleteRequest.versionType(); try { // add the response final WriteResult<DeleteResponse> writeResult = shardDeleteOperation(request, deleteRequest, indexShard); DeleteResponse deleteResponse = writeResult.response(); location = locationToSync(location, writeResult.location); setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse)); } catch (Throwable e) { // rethrow the failure if we are going to retry on primary and let parent failure to // handle it if (retryPrimaryException(e)) { // restore updated versions... for (int j = 0; j < requestIndex; j++) { applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); } throw (ElasticsearchException) e; } if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) { logger.trace( "{} failed to execute bulk item (delete) {}", e, shardRequest.shardId, deleteRequest); } else { logger.debug( "{} failed to execute bulk item (delete) {}", e, shardRequest.shardId, deleteRequest); } // if its a conflict failure, and we already executed the request on a primary (and we // execute it // again, due to primary relocation and only processing up to N bulk items when the shard // gets closed) // then just use the response we got from the successful execution if (item.getPrimaryResponse() != null && isConflictException(e)) { setResponse(item, item.getPrimaryResponse()); } else { setResponse( item, new BulkItemResponse( item.id(), OP_TYPE_DELETE, new BulkItemResponse.Failure( request.index(), deleteRequest.type(), deleteRequest.id(), e))); } } } else if (item.request() instanceof UpdateRequest) { UpdateRequest updateRequest = (UpdateRequest) item.request(); preVersions[requestIndex] = updateRequest.version(); preVersionTypes[requestIndex] = updateRequest.versionType(); // We need to do the requested retries plus the initial attempt. We don't do < // 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE for (int updateAttemptsCount = 0; updateAttemptsCount <= updateRequest.retryOnConflict(); updateAttemptsCount++) { UpdateResult updateResult; try { updateResult = shardUpdateOperation(clusterState, request, updateRequest, indexShard); } catch (Throwable t) { updateResult = new UpdateResult(null, null, false, t, null); } if (updateResult.success()) { if (updateResult.writeResult != null) { location = locationToSync(location, updateResult.writeResult.location); } switch (updateResult.result.operation()) { case UPSERT: case INDEX: WriteResult<IndexResponse> result = updateResult.writeResult; IndexRequest indexRequest = updateResult.request(); BytesReference indexSourceAsBytes = indexRequest.source(); // add the response IndexResponse indexResponse = result.response(); UpdateResponse updateResponse = new UpdateResponse( indexResponse.getShardInfo(), indexResponse.getIndex(), indexResponse.getType(), indexResponse.getId(), indexResponse.getVersion(), indexResponse.isCreated()); if (updateRequest.fields() != null && updateRequest.fields().length > 0) { Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(indexSourceAsBytes, true); updateResponse.setGetResult( updateHelper.extractGetResult( updateRequest, shardRequest.request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes)); } item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), indexRequest); setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse)); break; case DELETE: WriteResult<DeleteResponse> writeResult = updateResult.writeResult; DeleteResponse response = writeResult.response(); DeleteRequest deleteRequest = updateResult.request(); updateResponse = new UpdateResponse( response.getShardInfo(), response.getIndex(), response.getType(), response.getId(), response.getVersion(), false); updateResponse.setGetResult( updateHelper.extractGetResult( updateRequest, shardRequest.request.index(), response.getVersion(), updateResult.result.updatedSourceAsMap(), updateResult.result.updateSourceContentType(), null)); // Replace the update request to the translated delete request to execute on the // replica. item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest); setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse)); break; case NONE: setResponse( item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult)); item.setIgnoreOnReplica(); // no need to go to the replica break; } // NOTE: Breaking out of the retry_on_conflict loop! break; } else if (updateResult.failure()) { Throwable t = updateResult.error; if (updateResult.retry) { // updateAttemptCount is 0 based and marks current attempt, if it's equal to // retryOnConflict we are going out of the iteration if (updateAttemptsCount >= updateRequest.retryOnConflict()) { setResponse( item, new BulkItemResponse( item.id(), OP_TYPE_UPDATE, new BulkItemResponse.Failure( request.index(), updateRequest.type(), updateRequest.id(), t))); } } else { // rethrow the failure if we are going to retry on primary and let parent failure to // handle it if (retryPrimaryException(t)) { // restore updated versions... for (int j = 0; j < requestIndex; j++) { applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); } throw (ElasticsearchException) t; } // if its a conflict failure, and we already executed the request on a primary (and we // execute it // again, due to primary relocation and only processing up to N bulk items when the // shard gets closed) // then just use the response we got from the successful execution if (item.getPrimaryResponse() != null && isConflictException(t)) { setResponse(item, item.getPrimaryResponse()); } else if (updateResult.result == null) { setResponse( item, new BulkItemResponse( item.id(), OP_TYPE_UPDATE, new BulkItemResponse.Failure( shardRequest.request.index(), updateRequest.type(), updateRequest.id(), t))); } else { switch (updateResult.result.operation()) { case UPSERT: case INDEX: IndexRequest indexRequest = updateResult.request(); if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) { logger.trace( "{} failed to execute bulk item (index) {}", t, shardRequest.shardId, indexRequest); } else { logger.debug( "{} failed to execute bulk item (index) {}", t, shardRequest.shardId, indexRequest); } setResponse( item, new BulkItemResponse( item.id(), OP_TYPE_UPDATE, new BulkItemResponse.Failure( request.index(), indexRequest.type(), indexRequest.id(), t))); break; case DELETE: DeleteRequest deleteRequest = updateResult.request(); if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) { logger.trace( "{} failed to execute bulk item (delete) {}", t, shardRequest.shardId, deleteRequest); } else { logger.debug( "{} failed to execute bulk item (delete) {}", t, shardRequest.shardId, deleteRequest); } setResponse( item, new BulkItemResponse( item.id(), OP_TYPE_DELETE, new BulkItemResponse.Failure( request.index(), deleteRequest.type(), deleteRequest.id(), t))); break; } } // NOTE: Breaking out of the retry_on_conflict loop! break; } } } } else { throw new IllegalStateException("Unexpected index operation: " + item.request()); } assert item.getPrimaryResponse() != null; assert preVersionTypes[requestIndex] != null; } processAfter(request.refresh(), indexShard, location); BulkItemResponse[] responses = new BulkItemResponse[request.items().length]; BulkItemRequest[] items = request.items(); for (int i = 0; i < items.length; i++) { responses[i] = items[i].getPrimaryResponse(); } return new Tuple<>( new BulkShardResponse(shardRequest.shardId, responses), shardRequest.request); }
@Test public void testNetworkPartitionDuringReplicaIndexOp() throws Exception { final String INDEX = "testidx"; List<String> nodes = internalCluster().startNodesAsync(2, nodeSettings).get(); // Create index test with 1 shard, 1 replica and ensure it is green createIndex(INDEX); ensureGreen(INDEX); // Disable allocation so the replica cannot be reallocated when it fails Settings s = ImmutableSettings.builder().put("cluster.routing.allocation.enable", "none").build(); client().admin().cluster().prepareUpdateSettings().setTransientSettings(s).get(); // Determine which node holds the primary shard ClusterState state = getNodeClusterState(nodes.get(0)); IndexShardRoutingTable shard = state.getRoutingTable().index(INDEX).shard(0); String primaryNode; String replicaNode; if (shard.getShards().get(0).primary()) { primaryNode = nodes.get(0); replicaNode = nodes.get(1); } else { primaryNode = nodes.get(1); replicaNode = nodes.get(0); } logger.info("--> primary shard is on {}", primaryNode); // Index a document to make sure everything works well IndexResponse resp = internalCluster() .client(primaryNode) .prepareIndex(INDEX, "doc") .setSource("foo", "bar") .get(); assertThat( "document exists on primary node", internalCluster() .client(primaryNode) .prepareGet(INDEX, "doc", resp.getId()) .setPreference("_only_local") .get() .isExists(), equalTo(true)); assertThat( "document exists on replica node", internalCluster() .client(replicaNode) .prepareGet(INDEX, "doc", resp.getId()) .setPreference("_only_local") .get() .isExists(), equalTo(true)); // Disrupt the network so indexing requests fail to replicate logger.info("--> preventing index/replica operations"); TransportService mockTransportService = internalCluster().getInstance(TransportService.class, primaryNode); ((MockTransportService) mockTransportService) .addFailToSendNoConnectRule( internalCluster().getInstance(Discovery.class, replicaNode).localNode(), ImmutableSet.of(IndexAction.NAME + "[r]")); mockTransportService = internalCluster().getInstance(TransportService.class, replicaNode); ((MockTransportService) mockTransportService) .addFailToSendNoConnectRule( internalCluster().getInstance(Discovery.class, primaryNode).localNode(), ImmutableSet.of(IndexAction.NAME + "[r]")); logger.info("--> indexing into primary"); // the replica shard should now be marked as failed because the replication operation will fail resp = internalCluster() .client(primaryNode) .prepareIndex(INDEX, "doc") .setSource("foo", "baz") .get(); // wait until the cluster reaches an exact yellow state, meaning replica has failed assertBusy( new Runnable() { @Override public void run() { assertThat( client().admin().cluster().prepareHealth().get().getStatus(), equalTo(ClusterHealthStatus.YELLOW)); } }); assertThat( "document should still be indexed and available", client().prepareGet(INDEX, "doc", resp.getId()).get().isExists(), equalTo(true)); state = getNodeClusterState(randomFrom(nodes.toArray(Strings.EMPTY_ARRAY))); RoutingNodes rn = state.routingNodes(); logger.info( "--> counts: total: {}, unassigned: {}, initializing: {}, relocating: {}, started: {}", rn.shards( new Predicate<MutableShardRouting>() { @Override public boolean apply( org.elasticsearch.cluster.routing.MutableShardRouting input) { return true; } }) .size(), rn.shardsWithState(UNASSIGNED).size(), rn.shardsWithState(INITIALIZING).size(), rn.shardsWithState(RELOCATING).size(), rn.shardsWithState(STARTED).size()); logger.info( "--> unassigned: {}, initializing: {}, relocating: {}, started: {}", rn.shardsWithState(UNASSIGNED), rn.shardsWithState(INITIALIZING), rn.shardsWithState(RELOCATING), rn.shardsWithState(STARTED)); assertThat( "only a single shard is now active (replica should be failed and not reallocated)", rn.shardsWithState(STARTED).size(), equalTo(1)); }