@Override
protected void runOneIteration() throws Exception {
NodesInfo action = new NodesInfo.Builder().build();
JestResult result = null;
try {
result = client.execute(action);
} catch (Exception e) {
logger.error("Error executing NodesInfo!", e);
// do not elevate the exception since that will stop the scheduled calls.
// throw new RuntimeException("Error executing NodesInfo!", e);
}
if (result != null) {
LinkedHashSet<String> httpHosts = new LinkedHashSet<String>();
JsonObject jsonMap = result.getJsonObject();
JsonObject nodes = (JsonObject) jsonMap.get("nodes");
if (nodes != null) {
for (Entry<String, JsonElement> entry : nodes.entrySet()) {
JsonObject host = (JsonObject) entry.getValue();
String http_address = host.get("http_address").getAsString();
if (null != http_address) {
String cleanHttpAddress =
"http://" + http_address.substring(6, http_address.length() - 1);
httpHosts.add(cleanHttpAddress);
}
}
}
logger.info("Discovered Http Hosts: {}", httpHosts);
client.setServers(httpHosts);
}
}
private static void createMappingIfNeeded(
JestClient jestClient, String indexName, String typeName)
throws ElasticWriterException, IOException {
synchronized (CREATE_MAPPING_LOCK) {
IndicesExists indicesExists = new IndicesExists.Builder(indexName).build();
boolean indexExists = jestClient.execute(indicesExists).isSucceeded();
if (!indexExists) {
CreateIndex createIndex = new CreateIndex.Builder(indexName).build();
jestClient.execute(createIndex);
URL url = ElasticWriter.class.getResource("/elastic-mapping.json");
String mapping = Resources.toString(url, Charsets.UTF_8);
PutMapping putMapping = new PutMapping.Builder(indexName, typeName, mapping).build();
JestResult result = jestClient.execute(putMapping);
if (!result.isSucceeded()) {
throw new ElasticWriterException(
String.format("Failed to create mapping: %s", result.getErrorMessage()));
} else {
log.info("Created mapping for index {}", indexName);
}
}
}
}
public void executeQuery(String query, int pageSize, String index, String docType, int docLimit) {
Search search =
new Search.Builder(query)
.addIndex(index)
.addType(docType)
.setParameter(Parameters.SEARCH_TYPE, SearchType.SCAN)
.setParameter(Parameters.SIZE, pageSize)
.setParameter(Parameters.SCROLL, SCROLL)
.build();
System.out.println(query + "$$$$");
try {
JestResult searchResult = client.execute(search);
// System.out.println(searchResult.getJsonString());
String scrollId = searchResult.getJsonObject().get("_scroll_id").getAsString();
int currentResultSize = 0;
int numDocs = 0;
do {
JSONArray jArrayResult = new JSONArray();
SearchScroll scrollRequest =
new SearchScroll.Builder(scrollId, SCROLL)
.setParameter(Parameters.SIZE, pageSize)
.build();
JestResult scrollResult = client.execute(scrollRequest);
scrollId = scrollResult.getJsonObject().get("_scroll_id").getAsString();
JSONObject jObj = (JSONObject) JSONSerializer.toJSON(scrollResult.getJsonString());
JSONArray jArrayHits = jObj.getJSONObject("hits").getJSONArray("hits");
for (int i = 0; i < jArrayHits.size(); i++) {
jArrayResult.add(extractTika(jArrayHits.getString(i)).toString());
}
writeToFile(jArrayResult);
// Note: Current result size will be Page Size * number of shards
currentResultSize = jArrayHits.size();
numDocs += currentResultSize;
System.out.println("num docs:" + String.valueOf(numDocs));
if (docLimit != -1 && numDocs >= docLimit) {
break;
}
} while (currentResultSize != 0);
} catch (IOException e) {
LOG.error("Error retrieving from Elasticsearch", e);
}
}
protected void removeNodeAndUpdateServers(final String hostToRemove) {
log.warn("Removing host {}", hostToRemove);
discoveredServerList.remove(hostToRemove);
if (log.isInfoEnabled()) {
log.info("Discovered server pool is now: {}", StringUtils.join(discoveredServerList, ","));
}
if (!discoveredServerList.isEmpty()) {
client.setServers(discoveredServerList);
} else {
client.setServers(bootstrapServerList);
}
}
public static void main(String[] args) {
JestClient client = ESConnectionUtil.createClient("");
Bulk bulk =
new Bulk.Builder().defaultIndex("towns").defaultType("town").addAction(getIndex()).build();
try {
client.execute(bulk);
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
protected void runOneIteration() throws Exception {
JestResult result;
try {
result = client.execute(action);
} catch (CouldNotConnectException cnce) {
// Can't connect to this node, remove it from the list
log.error("Connect exception executing NodesInfo!", cnce);
removeNodeAndUpdateServers(cnce.getHost());
return;
// do not elevate the exception since that will stop the scheduled calls.
// throw new RuntimeException("Error executing NodesInfo!", e);
} catch (Exception e) {
log.error("Error executing NodesInfo!", e);
client.setServers(bootstrapServerList);
return;
// do not elevate the exception since that will stop the scheduled calls.
// throw new RuntimeException("Error executing NodesInfo!", e);
}
if (result.isSucceeded()) {
LinkedHashSet<String> httpHosts = new LinkedHashSet<String>();
JsonObject jsonMap = result.getJsonObject();
JsonObject nodes = (JsonObject) jsonMap.get("nodes");
if (nodes != null) {
for (Entry<String, JsonElement> entry : nodes.entrySet()) {
JsonObject host = entry.getValue().getAsJsonObject();
// get as a JsonElement first as some nodes in the cluster may not have an http_address
if (host.has(PUBLISH_ADDRESS_KEY)) {
JsonElement addressElement = host.get(PUBLISH_ADDRESS_KEY);
if (!addressElement.isJsonNull()) {
String httpAddress = getHttpAddress(addressElement.getAsString());
if (httpAddress != null) httpHosts.add(httpAddress);
}
}
}
}
if (log.isDebugEnabled()) {
log.debug(
"Discovered {} HTTP hosts: {}", httpHosts.size(), StringUtils.join(httpHosts, ","));
}
discoveredServerList = httpHosts;
client.setServers(discoveredServerList);
} else {
log.warn("NodesInfo request resulted in error: {}", result.getErrorMessage());
client.setServers(bootstrapServerList);
}
}
@RequestMapping(
value = {"/", "/index"},
method = RequestMethod.GET)
public String home(Model model) {
// Two Test Dataset from datehub.io
InputStream in2005 =
servletContext.getResourceAsStream("/WEB-INF/content/ibm-publications-2005.rdf");
InputStream in2006 =
servletContext.getResourceAsStream("/WEB-INF/content/ibm-publications-2006.rdf");
try {
ArrayList<RdfModel> rms2005 = RdfProcess.processRDF(in2005);
Boolean b = rms2005.addAll(RdfProcess.processRDF(in2006));
for (int i = 0; i < rms2005.size(); i++)
jc.execute(
new Delete.Builder(String.valueOf(i))
.index("publications")
.type("publication")
.build());
RdfSearch.Indexing(jc, rms2005);
model.addAttribute("rdfModel", new RdfModel());
} catch (Exception e) {
e.printStackTrace();
}
return "index";
}
@Override
public void stop() {
LOGGER.info("Stopping Elastic reporter engine...");
client.shutdownClient();
LOGGER.info("Stopping Elastic reporter engine... DONE");
}
@Override
public void report(Reportable reportable) {
try {
String indexName = getIndexName(reportable);
String jsonObject = null;
String typeName = null;
if (reportable instanceof Metrics) {
jsonObject = super.getSource((Metrics) reportable).string();
typeName = "request";
} else if (reportable instanceof HealthStatus) {
jsonObject = super.getSource((HealthStatus) reportable).string();
typeName = "health";
}
Index index = new Index.Builder(jsonObject).index(indexName).type(typeName).build();
client.executeAsync(
index,
new JestResultHandler<JestResult>() {
public void failed(Exception ex) {}
public void completed(JestResult result) {}
});
} catch (IOException e) {
LOGGER.error("Request {} report failed", reportable, e);
}
}
public void deleteIndex(String indexName, String indexType) throws Exception {
DeleteIndex.Builder deleteBuilder = new DeleteIndex.Builder(indexName);
if (null != indexType) {
deleteBuilder.type(indexType);
}
elasticSearchClient.execute(deleteBuilder.build());
}
public void createEmptyIndex(String indexName, String indexType, Map<String, String> settings)
throws Exception {
deleteIndex(indexName, null);
CreateIndex.Builder createBuilder = new CreateIndex.Builder(indexName);
if (null != settings && !settings.isEmpty()) {
createBuilder.settings(settings);
}
elasticSearchClient.execute(createBuilder.build());
}
private JestResult execute(Action action) {
try {
JestResult result = jestClient.execute(action);
logger.debug(result.getJsonString());
return result;
} catch (Exception e) {
throw new SearchException(e);
}
}
private boolean indexExists(String index) {
Action action = new IndicesExists.Builder(index).build();
try {
JestResult result = client.execute(action);
return result.isSucceeded();
} catch (IOException e) {
throw new ConnectException(e);
}
}
public boolean delete(
String keyWork, String[] fieldName, String indexName, String indexType, boolean isFuzzy)
throws Exception {
DeleteByQuery query =
new DeleteByQuery.Builder(getQueryBuilder().createBoolQuery(keyWork, fieldName, isFuzzy))
.addIndex(indexName)
.addType(indexType)
.build();
return getJestResultConvert().validate(elasticSearchClient.execute(query), indexName);
}
public <T> T query(
String keyWord, String field, String indexName, String indexType, Class<T> type)
throws Exception {
if (!checkIndexExists(indexName)) {
return null;
}
Search.Builder search = new Search.Builder(getQueryBuilder().createIdQuery(keyWord, indexType));
search = search.addIndex(indexName).addType(indexType);
return getJestResultConvert().convert(elasticSearchClient.execute(search.build()), type);
}
public <T> List<T> prefixQuery(
String keyWork, String fieldName, String indexName, String indexType, Class<T> type)
throws Exception {
Search.Builder search =
new Search.Builder(getQueryBuilder().createFieldPrefixKeyQuery(keyWork, fieldName));
search =
search
.addIndex(indexName)
.addType(indexType)
.setParameter("from", 0)
.setParameter("size", 10);
return getJestResultConvert().convertList(elasticSearchClient.execute(search.build()), type);
}
/**
* Stop the server.
*
* @throws Exception
*/
public void stop() throws Exception {
server.stop();
if (ds != null) {
ds.close();
InitialContext ctx = new InitialContext();
ctx.unbind("java:comp/env/jdbc/ApiManagerDS");
}
if (node != null) {
if ("true".equals(System.getProperty("apiman.test.es-delete-index", "true"))) {
client.execute(new DeleteIndex.Builder("apiman_manager").build());
}
}
}
public void createIndicesForTopics(Set<String> assignedTopics) {
for (String index : indicesForTopics(assignedTopics)) {
if (!indexExists(index)) {
CreateIndex createIndex = new CreateIndex.Builder(index).build();
try {
JestResult result = client.execute(createIndex);
if (!result.isSucceeded()) {
throw new ConnectException("Could not create index:" + index);
}
} catch (IOException e) {
throw new ConnectException(e);
}
}
}
}
public void bulkPutData(List<?> data, String indexName, String indexType) throws Exception {
List<Index> actions = new ArrayList<Index>();
Index action = null;
for (Object obj : data) {
action = new Index.Builder(obj).index(indexName).type(indexType).build();
actions.add(action);
}
Bulk bulk =
new Bulk.Builder()
.defaultIndex(indexName)
.defaultType(indexType)
.addAction(actions)
.build();
elasticSearchClient.execute(bulk);
}
public <T> PageModel<T> query(
String[] keyWords,
String[] fieldName,
String indexName,
String indexType,
Class<T> type,
Integer pageNo,
Integer pageSize)
throws Exception {
Search.Builder search =
new Search.Builder(getQueryBuilder().createMatchAllQuery(keyWords, fieldName));
search = search.addIndex(indexName).addType(indexType);
search = search.setParameter("from", (pageNo - 1) * pageSize).setParameter("size", pageSize);
return getJestResultConvert()
.convert(elasticSearchClient.execute(search.build()), pageNo, pageSize, type);
}
public <T> List<T> query(
String keyWork,
Map<String, Float> fieldsMap,
List<Sort> sorts,
String indexName,
String indexType,
Class<T> type,
boolean isFuzzy)
throws Exception {
Search.Builder search =
new Search.Builder(getQueryBuilder().createMultiFieldQuery(keyWork, fieldsMap));
search =
search.addIndex(indexName).addType(indexType).setSearchType(SearchType.QUERY_THEN_FETCH);
if (sorts != null && !sorts.isEmpty()) search = search.addSort(sorts);
return getJestResultConvert().convertList(elasticSearchClient.execute(search.build()), type);
}
@Override
protected void internalWrite(Server server, Query query, ImmutableList<Result> results)
throws Exception {
for (Result result : results) {
log.debug("Query result: [{}]", result);
Map<String, Object> resultValues = result.getValues();
for (Entry<String, Object> values : resultValues.entrySet()) {
Object value = values.getValue();
if (isNumeric(value)) {
Map<String, Object> map = new HashMap<String, Object>();
map.put("serverAlias", server.getAlias());
map.put("server", server.getHost());
map.put("port", server.getPort());
map.put("objDomain", result.getObjDomain());
map.put("className", result.getClassName());
map.put("typeName", result.getTypeName());
map.put("attributeName", result.getAttributeName());
map.put("key", values.getKey());
map.put("keyAlias", result.getKeyAlias());
map.put("value", Double.parseDouble(value.toString()));
map.put("timestamp", result.getEpoch());
log.debug(
"Insert into Elastic: Index: [{}] Type: [{}] Map: [{}]",
indexName,
ELASTIC_TYPE_NAME,
map);
Index index = new Index.Builder(map).index(indexName).type(ELASTIC_TYPE_NAME).build();
JestResult addToIndex = jestClient.execute(index);
if (!addToIndex.isSucceeded()) {
throw new ElasticWriterException(
String.format(
"Unable to write entry to elastic: %s", addToIndex.getErrorMessage()));
}
} else {
log.warn(
"Unable to submit non-numeric value to Elastic: [{}] from result [{}]",
value,
result);
}
}
}
}
public <T> PageModel<T> query(
String keyWork,
Map<String, Float> fieldsMap,
List<Sort> sorts,
String indexName,
String indexType,
Class<T> type,
boolean isFuzzy,
Integer pageNo,
Integer pageSize)
throws Exception {
Search.Builder search =
new Search.Builder(getQueryBuilder().createMultiFieldQuery(keyWork, fieldsMap));
search = search.addIndex(indexName).addType(indexType);
if (sorts != null && !sorts.isEmpty()) search = search.addSort(sorts);
search = search.setParameter("from", (pageNo - 1) * pageSize).setParameter("size", pageSize);
return getJestResultConvert()
.convert(elasticSearchClient.execute(search.build()), pageNo, pageSize, type);
}
@RequestMapping(value = "/performSearch", method = RequestMethod.POST)
public String performSearch(
@ModelAttribute("RdfModel") RdfModel rdfModel, Model model, RedirectAttributes attrs) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
QueryBuilder qb = QueryBuilders.matchQuery("hasTitle", rdfModel.getHasTitle());
searchSourceBuilder.query(qb);
Search search =
new Search.Builder(searchSourceBuilder.toString()).addIndex("publications").build();
try {
SearchResult searchresult = jc.execute(search);
ArrayList<RdfModel> publications =
new ArrayList<RdfModel>(searchresult.getSourceAsObjectList(RdfModel.class));
model.addAttribute("searchResults", publications);
model.addAttribute("rdfModel", new RdfModel());
} catch (Exception e) {
e.printStackTrace();
}
return "index";
}
@Override
public void stop() throws LifecycleException {
super.stop();
jestClient.shutdownClient();
}
public void putData(Object data, String indexName, String indexType) throws Exception {
Index dataIndex = new Index.Builder(data).index(indexName).type(indexType).build();
elasticSearchClient.execute(dataIndex);
}
public void createMapping(String indexName, String indexType, Object mappingObj)
throws Exception {
PutMapping putMapping = new PutMapping.Builder(indexName, indexType, mappingObj).build();
JestResult result = elasticSearchClient.execute(putMapping);
if (result == null) throw new ElasticSearchException("Create mapping field.");
}
public boolean delete(String indexId, String indexName, String indexType) throws Exception {
Delete.Builder deleteBuilder = new Delete.Builder(indexName, indexType, indexId);
return elasticSearchClient.execute(deleteBuilder.build()).isSucceeded();
}
public boolean checkIndexExists(String indexName) throws Exception {
IndicesExists.Builder existsBuilder = new IndicesExists.Builder(indexName);
JestResult result = elasticSearchClient.execute(existsBuilder.build());
return result.isSucceeded();
}
@PreDestroy
public void shutdown() {
if (null != elasticSearchClient) elasticSearchClient.shutdownClient();
}
