@Override
public final String esGet(String index, String type, String id) {
try {
final GetResponse gr = client.prepareGet(index, type, id).execute().actionGet();
if (!gr.exists()) return "Doesn't exist";
return gr.getSourceAsString();
} catch (ElasticSearchException e) {
log.debug("ElasticSearchException {}", e);
return e.getMessage();
}
}
void insertIndexes(String str, String filename, int frameno, Client client) {
// System.err.println(str);
StringTokenizer token = new StringTokenizer(str, "\t");
try {
String color = token.nextToken();
IndexResponse response =
client
.prepareIndex("movie", color, filename + "_" + frameno)
.setSource(
jsonBuilder()
.startObject()
.field("filename", filename.replace(".", ""))
.field("color", color)
.field("frameno", String.valueOf(frameno))
.field("face", token.nextToken())
.field("corner", token.nextToken())
.field("apidq1", token.nextToken())
.field("apidq2", token.nextToken())
.field("apidq3", token.nextToken())
.field("apidq4", token.nextToken())
.field("apidq5", token.nextToken())
.field("apidq6", token.nextToken())
.field("apidq7", token.nextToken())
.field("apidq8", token.nextToken())
.field("hsv1", token.nextToken())
.field("hsv2", token.nextToken())
.field("hsv3", token.nextToken())
.field("hsv4", token.nextToken())
.field("edge1", token.nextToken())
.field("edge2", token.nextToken())
.field("edge3", token.nextToken())
.field("edge4", token.nextToken())
.field("edge5", token.nextToken())
.field("edge6", token.nextToken())
.field("edge7", token.nextToken())
.field("edge8", token.nextToken())
.endObject())
.execute()
.actionGet();
} catch (ElasticSearchException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Test
public void testExternalVersioningInitialDelete() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").execute().actionGet();
client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
DeleteResponse deleteResponse =
client2
.prepareDelete("test", "type", "1")
.setVersion(17)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertThat(deleteResponse.isNotFound(), equalTo(true));
try {
client
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(13)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
} catch (ElasticSearchException e) {
assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
}
client
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(18)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
}
@Test
public void testFastVectorHighlighterManyDocs() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (ElasticSearchException e) {
assertThat(e.unwrapCause(), instanceOf(IndexMissingException.class));
}
client
.admin()
.indices()
.prepareCreate("test")
.addMapping("type1", type1TermVectorMapping())
.execute()
.actionGet();
client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
int COUNT = 100;
logger.info("--> indexing docs");
for (int i = 0; i < COUNT; i++) {
client
.prepareIndex("test", "type1", Integer.toString(i))
.setSource("field1", "test " + i)
.execute()
.actionGet();
if (i % 5 == 0) {
// flush so we get updated readers and segmented readers
client.admin().indices().prepareFlush().execute().actionGet();
}
}
client.admin().indices().prepareRefresh().execute().actionGet();
logger.info("--> searching explicitly on field1 and highlighting on it");
SearchResponse searchResponse =
client
.prepareSearch()
.setSize(COUNT)
.setQuery(termQuery("field1", "test"))
.addHighlightedField("field1", 100, 0)
.execute()
.actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo((long) COUNT));
assertThat(searchResponse.hits().hits().length, equalTo(COUNT));
for (SearchHit hit : searchResponse.hits()) {
// LUCENE 3.1 UPGRADE: Caused adding the space at the end...
assertThat(
hit.highlightFields().get("field1").fragments()[0],
equalTo("<em>test</em> " + hit.id() + " "));
}
logger.info("--> searching explicitly on field1 and highlighting on it, with DFS");
searchResponse =
client
.prepareSearch()
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setSize(COUNT)
.setQuery(termQuery("field1", "test"))
.addHighlightedField("field1", 100, 0)
.execute()
.actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo((long) COUNT));
assertThat(searchResponse.hits().hits().length, equalTo(COUNT));
for (SearchHit hit : searchResponse.hits()) {
// LUCENE 3.1 UPGRADE: Caused adding the space at the end...
assertThat(
hit.highlightFields().get("field1").fragments()[0],
equalTo("<em>test</em> " + hit.id() + " "));
}
logger.info("--> searching explicitly _all and highlighting on _all");
searchResponse =
client
.prepareSearch()
.setSize(COUNT)
.setQuery(termQuery("_all", "test"))
.addHighlightedField("_all", 100, 0)
.execute()
.actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo((long) COUNT));
assertThat(searchResponse.hits().hits().length, equalTo(COUNT));
for (SearchHit hit : searchResponse.hits()) {
// LUCENE 3.1 UPGRADE: Caused adding the space at the end...
assertThat(
hit.highlightFields().get("_all").fragments()[0],
equalTo("<em>test</em> " + hit.id() + " "));
}
}
@Test
public void testSimpleVersioningWithFlush() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (IndexMissingException e) {
// its ok
}
client.admin().indices().prepareCreate("test").execute().actionGet();
client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
IndexResponse indexResponse =
client
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.execute()
.actionGet();
assertThat(indexResponse.version(), equalTo(1l));
client.admin().indices().prepareFlush().execute().actionGet();
indexResponse =
client
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_2")
.setVersion(1)
.execute()
.actionGet();
assertThat(indexResponse.version(), equalTo(2l));
client.admin().indices().prepareFlush().execute().actionGet();
try {
client
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(1)
.execute()
.actionGet();
} catch (ElasticSearchException e) {
assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
}
try {
client2
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(1)
.execute()
.actionGet();
} catch (ElasticSearchException e) {
assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
}
try {
client
.prepareIndex("test", "type", "1")
.setCreate(true)
.setSource("field1", "value1_1")
.setVersion(1)
.execute()
.actionGet();
} catch (ElasticSearchException e) {
assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
}
try {
client2
.prepareIndex("test", "type", "1")
.setCreate(true)
.setSource("field1", "value1_1")
.setVersion(1)
.execute()
.actionGet();
} catch (ElasticSearchException e) {
assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
}
try {
client.prepareDelete("test", "type", "1").setVersion(1).execute().actionGet();
} catch (ElasticSearchException e) {
assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
}
try {
client2.prepareDelete("test", "type", "1").setVersion(1).execute().actionGet();
} catch (ElasticSearchException e) {
assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
}
client.admin().indices().prepareRefresh().execute().actionGet();
for (int i = 0; i < 10; i++) {
assertThat(
client.prepareGet("test", "type", "1").execute().actionGet().getVersion(), equalTo(2l));
}
for (int i = 0; i < 10; i++) {
SearchResponse searchResponse =
client.prepareSearch().setQuery(matchAllQuery()).setVersion(true).execute().actionGet();
assertThat(searchResponse.getHits().getAt(0).version(), equalTo(2l));
}
}
@Test
public void testExternalVersioning() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (IndexMissingException e) {
// its ok
}
client.admin().indices().prepareCreate("test").execute().actionGet();
client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
IndexResponse indexResponse =
client
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(12)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertThat(indexResponse.version(), equalTo(12l));
indexResponse =
client
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(14)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertThat(indexResponse.version(), equalTo(14l));
try {
client
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(13)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
} catch (ElasticSearchException e) {
assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
}
client.admin().indices().prepareRefresh().execute().actionGet();
for (int i = 0; i < 10; i++) {
assertThat(
client.prepareGet("test", "type", "1").execute().actionGet().getVersion(), equalTo(14l));
}
DeleteResponse deleteResponse =
client2
.prepareDelete("test", "type", "1")
.setVersion(17)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertThat(deleteResponse.isNotFound(), equalTo(false));
assertThat(deleteResponse.getVersion(), equalTo(17l));
try {
client2
.prepareDelete("test", "type", "1")
.setVersion(2)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
} catch (ElasticSearchException e) {
assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
}
deleteResponse =
client2
.prepareDelete("test", "type", "1")
.setVersion(18)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertThat(deleteResponse.isNotFound(), equalTo(true));
assertThat(deleteResponse.getVersion(), equalTo(18l));
}
private void executeBulk(
final BulkRequest bulkRequest,
final long startTime,
final ActionListener<BulkResponse> listener) {
ClusterState clusterState = clusterService.state();
for (ActionRequest request : bulkRequest.requests) {
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
indexRequest.routing(
clusterState
.metaData()
.resolveIndexRouting(indexRequest.routing(), indexRequest.index()));
indexRequest.index(clusterState.metaData().concreteIndex(indexRequest.index()));
if (allowIdGeneration) {
if (indexRequest.id() == null) {
indexRequest.id(UUID.randomBase64UUID());
// since we generate the id, change it to CREATE
indexRequest.opType(IndexRequest.OpType.CREATE);
}
}
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index()));
}
}
final BulkItemResponse[] responses = new BulkItemResponse[bulkRequest.requests.size()];
// first, go over all the requests and create a ShardId -> Operations mapping
Map<ShardId, List<BulkItemRequest>> requestsByShard = Maps.newHashMap();
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
// handle routing
MappingMetaData mappingMd =
clusterState.metaData().index(indexRequest.index()).mapping(indexRequest.type());
if (mappingMd != null) {
try {
indexRequest.processRouting(mappingMd);
} catch (ElasticSearchException e) {
responses[i] =
new BulkItemResponse(
i,
indexRequest.opType().toString().toLowerCase(),
new BulkItemResponse.Failure(
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
e.getDetailedMessage()));
continue;
}
}
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
indexRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
MappingMetaData mappingMd =
clusterState.metaData().index(deleteRequest.index()).mapping(deleteRequest.type());
if (mappingMd != null
&& mappingMd.routing().required()
&& deleteRequest.routing() == null) {
// if routing is required, and no routing on the delete request, we need to broadcast
// it....
GroupShardsIterator groupShards =
clusterService
.operationRouting()
.broadcastDeleteShards(clusterState, deleteRequest.index());
for (ShardIterator shardIt : groupShards) {
List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardIt.shardId(), list);
}
list.add(new BulkItemRequest(i, request));
}
} else {
ShardId shardId =
clusterService
.operationRouting()
.deleteShards(
clusterState,
deleteRequest.index(),
deleteRequest.type(),
deleteRequest.id(),
deleteRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
}
}
}
if (requestsByShard.isEmpty()) {
listener.onResponse(new BulkResponse(responses, System.currentTimeMillis() - startTime));
return;
}
final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
final ShardId shardId = entry.getKey();
final List<BulkItemRequest> requests = entry.getValue();
BulkShardRequest bulkShardRequest =
new BulkShardRequest(
shardId.index().name(),
shardId.id(),
bulkRequest.refresh(),
requests.toArray(new BulkItemRequest[requests.size()]));
bulkShardRequest.replicationType(bulkRequest.replicationType());
bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
shardBulkAction.execute(
bulkShardRequest,
new ActionListener<BulkShardResponse>() {
@Override
public void onResponse(BulkShardResponse bulkShardResponse) {
synchronized (responses) {
for (BulkItemResponse bulkItemResponse : bulkShardResponse.responses()) {
responses[bulkItemResponse.itemId()] = bulkItemResponse;
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
@Override
public void onFailure(Throwable e) {
// create failures for all relevant requests
String message = ExceptionsHelper.detailedMessage(e);
synchronized (responses) {
for (BulkItemRequest request : requests) {
if (request.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request.request();
responses[request.id()] =
new BulkItemResponse(
request.id(),
indexRequest.opType().toString().toLowerCase(),
new BulkItemResponse.Failure(
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
message));
} else if (request.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request.request();
responses[request.id()] =
new BulkItemResponse(
request.id(),
"delete",
new BulkItemResponse.Failure(
deleteRequest.index(),
deleteRequest.type(),
deleteRequest.id(),
message));
}
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
private void finishHim() {
listener.onResponse(
new BulkResponse(responses, System.currentTimeMillis() - startTime));
}
});
}
}
