@Override
public void index(List<Article> list) {
if (list.size() > 0) {
try {
BulkRequestBuilder bulkRequest = transportClient.prepareBulk();
for (Article article : list) {
XContentBuilder builder =
XContentFactory.jsonBuilder()
.startObject()
.field(TITLE, article.getTitle())
.field(SUMMARY, HtmlHelper.filterHtml(article.getSummary()))
.endObject();
bulkRequest.add(
transportClient
.prepareIndex(INDICE, TYPE, String.valueOf(article.getId()))
.setSource(builder));
}
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
logger.debug("bulkResponse.hasFailures()");
} else {
logger.debug("bulk index ok！");
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
public void run() {
Client client = null;
TransportClient transportClient = null;
try {
transportClient = new TransportClient();
client =
transportClient.addTransportAddress(new InetSocketTransportAddress("192.168.1.40", 9300));
SearchResponse response =
client
.prepareSearch("tms-allflat")
.setTypes("personal")
.setQuery(QueryBuilders.matchAllQuery())
.addAggregation(
AggregationBuilders.terms("aggs1")
.field("skill_1")
.size(20)
.order(Terms.Order.count(false)))
.execute()
.actionGet();
Terms terms = response.getAggregations().get("aggs1");
terms
.getBuckets()
.stream()
.forEach(s -> System.out.println(s.getKeyAsText() + "(" + s.getDocCount() + ")"));
} finally {
transportClient.close();
client.close();
}
}
@Bean
public Client client() {
TransportClient client = new TransportClient();
TransportAddress address = new InetSocketTransportAddress(elasticHost, elasticPort);
client.addTransportAddress(address);
return client;
}
private static TransportClient open(
final ElasticsearchReporter reporter,
final Settings settings,
final Iterable<TransportAddress> addresses) {
assert settings != null;
reporter.logNotice(format("opening shared client for %s", identify(settings)));
try {
TransportClient client = new TransportClient(settings);
client.addTransportAddresses(toArray(addresses, TransportAddress.class));
reporter.logNotice(
format(
"shared client for %s opened -> %s", identify(settings), toDefaultString(client)));
return client;
} catch (Exception failure) {
reporter.logError(
format("unable to open shared client for %s", identify(settings)), failure);
throw failure;
}
}
private static Client startClient(Path tempDir, TransportAddress... transportAddresses) {
Settings clientSettings =
Settings.settingsBuilder()
.put("name", "qa_smoke_client_" + counter.getAndIncrement())
.put(
InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING,
true) // prevents any settings to be replaced by system properties.
.put("client.transport.ignore_cluster_name", true)
.put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
.put("node.mode", "network")
.build(); // we require network here!
TransportClient.Builder transportClientBuilder =
TransportClient.builder().settings(clientSettings);
TransportClient client =
transportClientBuilder.build().addTransportAddresses(transportAddresses);
logger.info("--> Elasticsearch Java TransportClient started");
Exception clientException = null;
try {
ClusterHealthResponse health = client.admin().cluster().prepareHealth().get();
logger.info(
"--> connected to [{}] cluster which is running [{}] node(s).",
health.getClusterName(),
health.getNumberOfNodes());
} catch (Exception e) {
clientException = e;
}
assumeNoException(
"Sounds like your cluster is not running at " + clusterAddresses, clientException);
return client;
}
@Test
public void testWithSniffing() throws Exception {
TransportClient client =
new TransportClient(
ImmutableSettings.builder()
.put("client.transport.sniff", true)
.put("cluster.name", "cluster1")
.put("node.name", "transport_client_" + this.getTestName() + "_1")
.put("client.transport.nodes_sampler_interval", "1s")
.put(
TransportModule.TRANSPORT_SERVICE_TYPE_KEY,
InternalTransportService.class.getName())
.put(HEADER_SETTINGS)
.build());
try {
client.addTransportAddress(address);
InternalTransportService service =
(InternalTransportService) client.injector.getInstance(TransportService.class);
if (!service.clusterStateLatch.await(5, TimeUnit.SECONDS)) {
fail("takes way too long to get the cluster state");
}
assertThat(client.connectedNodes().size(), is(1));
assertThat(client.connectedNodes().get(0).getAddress(), is((TransportAddress) address));
} finally {
client.close();
}
}
public void testPickingUpChangesInDiscoveryNode() {
String nodeName =
internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false));
TransportClient client = (TransportClient) internalCluster().client(nodeName);
assertThat(client.connectedNodes().get(0).dataNode(), equalTo(false));
}
public void testWithSniffing() throws Exception {
try (TransportClient client =
TransportClient.builder()
.settings(
Settings.builder()
.put("client.transport.sniff", true)
.put("cluster.name", "cluster1")
.put("node.name", "transport_client_" + this.getTestName() + "_1")
.put("client.transport.nodes_sampler_interval", "1s")
.put(HEADER_SETTINGS)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
.build())
.addPlugin(InternalTransportService.TestPlugin.class)
.build()) {
client.addTransportAddress(address);
InternalTransportService service =
(InternalTransportService) client.injector.getInstance(TransportService.class);
if (!service.clusterStateLatch.await(5, TimeUnit.SECONDS)) {
fail("takes way too long to get the cluster state");
}
assertThat(client.connectedNodes().size(), is(1));
assertThat(client.connectedNodes().get(0).getAddress(), is((TransportAddress) address));
}
}
@Override
protected Client createClient(ImmutableSettings.Builder builder) {
if (_transportAddresses.isEmpty()) {
throw new IllegalStateException("There must be at least one transport address");
}
Class<?> clazz = getClass();
builder.classLoader(clazz.getClassLoader());
builder.loadFromClasspath(getConfigFileName());
TransportClient transportClient = new TransportClient(builder);
builder.put("client.transport.sniff", true);
builder.put("cluster.name", getClusterName());
for (String transportAddress : _transportAddresses) {
String[] transportAddressParts = StringUtil.split(transportAddress, StringPool.COLON);
try {
InetAddress inetAddress = InetAddress.getByName(transportAddressParts[0]);
int port = GetterUtil.getInteger(transportAddressParts[1]);
transportClient.addTransportAddress(new InetSocketTransportAddress(inetAddress, port));
} catch (Exception e) {
if (_log.isWarnEnabled()) {
_log.warn("Unable to add transport address " + transportAddress, e);
}
}
}
return transportClient;
}
protected void buildClient() throws Exception {
client = TransportClient.builder().settings(settings()).build();
Assert.hasText(clusterNodes, "[Assertion failed] clusterNodes settings missing.");
for (String clusterNode : split(clusterNodes, COMMA)) {
String hostName = substringBeforeLast(clusterNode, COLON);
String port = substringAfterLast(clusterNode, COLON);
Assert.hasText(hostName, "[Assertion failed] missing host name in 'clusterNodes'");
Assert.hasText(port, "[Assertion failed] missing port in 'clusterNodes'");
logger.info("adding transport node : " + clusterNode);
client.addTransportAddress(
new InetSocketTransportAddress(InetAddress.getByName(hostName), Integer.valueOf(port)));
}
client.connectedNodes();
}
public static void main(String[] args) {
TransportClient client = new TransportClient();
client.addTransportAddress(new InetSocketTransportAddress("localhost", 9300));
TestBulkProcessor test = new TestBulkProcessor(client);
int count = 0;
int processors = Runtime.getRuntime().availableProcessors() - 1;
while (count++ < 20) {
int count2 = 0;
while (count2++ < 3) {
test.testBulkProcessor(processors);
}
processors++;
}
client.close();
}
/**
* Builds the Elasticsearch client using the properties this connection was instantiated with
*
* @return
* @throws SQLException
*/
private Client buildClient() throws SQLException {
if (props.containsKey("test")) { // used for integration tests
return ESIntegTestCase.client();
} else
try {
Settings.Builder settingsBuilder = Settings.settingsBuilder();
for (Object key : this.props.keySet()) {
settingsBuilder.put(key, this.props.get(key));
}
Settings settings = settingsBuilder.build();
TransportClient client =
TransportClient.builder()
.settings(settings)
.build()
.addTransportAddress(
new InetSocketTransportAddress(InetAddress.getByName(host), port));
// add additional hosts if set in URL query part
if (this.props.containsKey("es.hosts"))
for (String hostPort : this.props.getProperty("es.hosts").split(",")) {
String newHost = hostPort.split(":")[0].trim();
int newPort =
(hostPort.split(":").length > 1
? Integer.parseInt(hostPort.split(":")[1])
: Utils.PORT);
client.addTransportAddress(
new InetSocketTransportAddress(InetAddress.getByName(newHost), newPort));
logger.info("Adding additional ES host: " + hostPort);
}
// check if index exists
if (index != null) {
boolean indexExists =
client
.admin()
.indices()
.exists(new IndicesExistsRequest(index))
.actionGet()
.isExists();
if (!indexExists) throw new SQLException("Index or Alias '" + index + "' does not exist");
}
return client;
} catch (UnknownHostException e) {
throw new SQLException("Unable to connect to " + host, e);
} catch (Throwable t) {
throw new SQLException("Unable to connect to database", t);
}
}
@Override
protected Client buildClient(Settings headersSettings, GenericAction[] testedActions) {
TransportClient client =
new TransportClient(
ImmutableSettings.builder()
.put("client.transport.sniff", false)
.put("node.name", "transport_client_" + this.getTestName())
.put(
TransportModule.TRANSPORT_SERVICE_TYPE_KEY,
InternalTransportService.class.getName())
.put(HEADER_SETTINGS)
.build());
client.addTransportAddress(address);
return client;
}
/*
* (non-Javadoc)
*
* @see com.datatorrent.lib.db.Connectable#isConnected()
*/
@Override
public boolean isConnected() {
if (client != null) {
return client.connectedNodes().size() != 0;
}
return false;
}
private Client createTransportClient() {
Settings settings =
ImmutableSettings.settingsBuilder()
.put("cluster.name", configuration.getClusterName())
.build();
TransportClient transportClient = new TransportClient(settings);
List<HostAddress> adresses = configuration.getHostsAddresses();
for (HostAddress address : adresses) {
transportClient.addTransportAddress(
new InetSocketTransportAddress(address.getHostname(), address.getPort()));
}
return transportClient;
}
@Override
public void notify(DelegateExecution execution) throws Exception {
HistoryService historyService = execution.getProcessEngineServices().getHistoryService();
HistoricDecisionInstance historicDecisionInstance =
historyService
.createHistoricDecisionInstanceQuery()
.includeInputs()
.includeOutputs()
.decisionDefinitionKey((String) execution.getVariableLocal("tableName"))
.processInstanceId(execution.getProcessInstanceId())
.singleResult();
// Fill a new object with stuff...
FraudScoreTableObject fraudData = new FraudScoreTableObject();
fraudData.setFraudInstanceID(historicDecisionInstance.getId());
List<HistoricDecisionInputInstance> inputs = historicDecisionInstance.getInputs();
for (HistoricDecisionInputInstance historicDecisionInputInstance : inputs) {
String inputName = historicDecisionInputInstance.getTypeName();
if (inputName.equals("paymentRejected")) {
fraudData.setPaymentRejected((Boolean) historicDecisionInputInstance.getValue());
} else if (inputName.equals("numberOfPayouts")) {
fraudData.setNumberOfPayouts((Integer) historicDecisionInputInstance.getValue());
} else if (inputName.equals("historyOfFraud")) {
fraudData.setHistoryOfFraud((Boolean) historicDecisionInputInstance.getValue());
} else if (inputName.equals("claimAmount")) {
fraudData.setCalimAmount((Long) historicDecisionInputInstance.getValue());
}
}
List<HistoricDecisionOutputInstance> outputs = historicDecisionInstance.getOutputs();
for (HistoricDecisionOutputInstance historicDecisionOutputInstance : outputs) {
String inputName = historicDecisionOutputInstance.getTypeName();
if (inputName.equals("frausScore")) {
fraudData.setFraudScore((Integer) historicDecisionOutputInstance.getValue());
}
}
ObjectMapper mapper = new ObjectMapper();
String serializedHistoricDecisionInstance =
mapper.writerWithDefaultPrettyPrinter().writeValueAsString(fraudData);
Client client =
TransportClient.builder()
.build()
.addTransportAddress(
new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300));
IndexResponse response =
client
.prepareIndex("camunda", "fraudData", historicDecisionInstance.getId())
.setSource(serializedHistoricDecisionInstance)
.get();
LOGGER.info(response.getId());
}
@Override
public List<Article> query(String queryString, int size) {
Assert.hasText(queryString, "queryString must not be empty!");
SearchResponse searchResponse =
transportClient
.prepareSearch(INDICE)
.setTypes(TYPE)
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setQuery(QueryBuilders.queryString(queryString))
.setFrom(0)
.setSize(size)
.setExplain(true)
.execute()
.actionGet();
SearchHits hits = searchResponse.getHits();
long total = hits.getTotalHits();
logger.debug("search articles result total:{}", total);
List<Article> list = new ArrayList<Article>();
Article article = null;
for (SearchHit hit : hits) {
Long id = (Long) hit.getSource().get("id");
String title = (String) hit.getSource().get(TITLE);
String summary = (String) hit.getSource().get(SUMMARY);
logger.debug("id:{},title:{},summary:{}", id, title, summary);
article = new Article();
article.setId(id);
article.setTitle(title);
article.setSummary(summary);
list.add(article);
}
return list;
}
public void testNodeVersionIsUpdated() {
TransportClient client = (TransportClient) internalCluster().client();
TransportClientNodesService nodeService = client.nodeService();
Node node =
new Node(
Settings.builder()
.put(internalCluster().getDefaultSettings())
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.put("node.name", "testNodeVersionIsUpdated")
.put("http.enabled", false)
.put(Node.NODE_DATA_SETTING.getKey(), false)
.put("cluster.name", "foobar")
.put(
InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(),
true) // make sure we get what we set :)
.build());
node.start();
try {
TransportAddress transportAddress =
node.injector().getInstance(TransportService.class).boundAddress().publishAddress();
client.addTransportAddress(transportAddress);
assertThat(
nodeService.connectedNodes().size(),
greaterThanOrEqualTo(
1)); // since we force transport clients there has to be one node started that we
// connect to.
for (DiscoveryNode discoveryNode :
nodeService.connectedNodes()) { // connected nodes have updated version
assertThat(discoveryNode.getVersion(), equalTo(Version.CURRENT));
}
for (DiscoveryNode discoveryNode : nodeService.listedNodes()) {
assertThat(discoveryNode.id(), startsWith("#transport#-"));
assertThat(
discoveryNode.getVersion(), equalTo(Version.CURRENT.minimumCompatibilityVersion()));
}
assertThat(nodeService.filteredNodes().size(), equalTo(1));
for (DiscoveryNode discoveryNode : nodeService.filteredNodes()) {
assertThat(
discoveryNode.getVersion(), equalTo(Version.CURRENT.minimumCompatibilityVersion()));
}
} finally {
node.close();
}
}
public void testThatTransportClientSettingCannotBeChanged() {
Settings baseSettings =
settingsBuilder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
try (TransportClient client = TransportClient.builder().settings(baseSettings).build()) {
Settings settings = client.injector.getInstance(Settings.class);
assertThat(Client.CLIENT_TYPE_SETTING_S.get(settings), is("transport"));
}
}
/**
* LocalElasticSearchClient.
*
* @return a {@link org.elasticsearch.client.Client} object.
* @throws java.net.UnknownHostException if any.
*/
@Profile("local")
@Bean(name = "elasticSearchClient")
public Client localElasticSearchClient() throws UnknownHostException {
return TransportClient.builder()
.build()
.addTransportAddress(
new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), 9300));
}
@Override
protected Client buildClient(Settings headersSettings, GenericAction[] testedActions) {
TransportClient client =
TransportClient.builder()
.settings(
Settings.builder()
.put("client.transport.sniff", false)
.put("cluster.name", "cluster1")
.put("node.name", "transport_client_" + this.getTestName())
.put(headersSettings)
.build())
.addPlugin(InternalTransportService.TestPlugin.class)
.build();
client.addTransportAddress(address);
return client;
}
@Override
public Client client(Node node, String clusterName, Random random) {
TransportAddress addr =
((InternalNode) node)
.injector()
.getInstance(TransportService.class)
.boundAddress()
.publishAddress();
TransportClient client =
new TransportClient(
settingsBuilder()
.put("client.transport.nodes_sampler_interval", "1s")
.put("name", "transport_client_" + node.settings().get("name"))
.put("cluster.name", clusterName)
.put("client.transport.sniff", sniff)
.build());
client.addTransportAddress(addr);
return client;
}
private void initArticleIndexMapping() {
try {
boolean isExists = ElasticsearchHelper.isExistsIndex(transportClient, INDICE);
if (!isExists) {
// create index
transportClient.admin().indices().prepareCreate(INDICE).execute().actionGet();
// crate mapping
XContentBuilder mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject(TYPE)
.startObject("_all")
.field("indexAnalyzer", "ik")
.field("searchAnalyzer", "ik")
.endObject()
.startObject("properties")
.startObject(TITLE)
.field("type", "string")
.field("indexAnalyzer", "ik")
.field("searchAnalyzer", "ik")
.endObject()
.startObject(SUMMARY)
.field("type", "string")
.field("indexAnalyzer", "ik")
.field("searchAnalyzer", "ik")
.endObject()
.endObject()
.endObject()
.endObject();
PutMappingRequest mappingRequest =
Requests.putMappingRequest(INDICE).type(TYPE).source(mapping);
transportClient.admin().indices().putMapping(mappingRequest).actionGet();
logger.debug("create index and mapping are success!");
} else {
logger.debug("Index already exists!");
}
} catch (Exception e) {
logger.error("create index and mapping are failure!", e);
}
}
@Override
public void destroy() throws Exception {
try {
logger.info("Closing elasticSearch client");
if (client != null) {
client.close();
}
} catch (final Exception e) {
logger.error("Error closing ElasticSearch client: ", e);
}
}
@Override
public void deleteAll() {
MatchAllQueryBuilder allQueryBuilder = QueryBuilders.matchAllQuery();
transportClient
.prepareDeleteByQuery(INDICE)
.setTypes(TYPE)
.setQuery(allQueryBuilder)
.execute()
.actionGet();
logger.debug("delete All ");
}
synchronized Client getClient() {
if (nodeInfo == null) {
throw new IllegalStateException("Node has not started yet");
}
if (client == null) {
TransportAddress addr = nodeInfo.getTransport().getAddress().publishAddress();
// verify that the end node setting will have network enabled.
Settings clientSettings =
settingsBuilder()
.put(externalNodeSettings)
.put("client.transport.nodes_sampler_interval", "1s")
.put("name", "transport_client_" + nodeInfo.getNode().name())
.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName)
.put("client.transport.sniff", false)
.build();
TransportClient client = TransportClient.builder().settings(clientSettings).build();
client.addTransportAddress(addr);
this.client = client;
}
return client;
}
/**
* iLabElasticSearchClient.
*
* @return a {@link org.elasticsearch.client.Client} object.
* @throws java.net.UnknownHostException if any.
*/
@Profile("iLab")
@Bean(name = "elasticSearchClient")
public Client iLabElasticSearchClient() throws UnknownHostException {
Settings settings =
Settings.settingsBuilder()
.put("cluster.name", Constants.APPLICATIONNAME)
.put("client.transport.sniff", true)
.build();
return TransportClient.builder()
.settings(settings)
.build()
.addTransportAddress(
new InetSocketTransportAddress(InetAddress.getByName("152.190.139.77"), 9300));
}
synchronized void stop() throws InterruptedException {
if (running()) {
try {
if (this.client != null) {
client.close();
}
} finally {
process.destroy();
process.waitFor();
process = null;
nodeInfo = null;
}
}
}
@PostConstruct
public void init() {
final ImmutableSettings.Builder settings = settingsBuilder();
settings.put("cluster.name", "es-events");
client = new TransportClient(settings);
LOGGER.info("Adding ES hosts:");
for (String host : config.getEsHosts().split(",")) {
LOGGER.info(host);
client.addTransportAddress(new InetSocketTransportAddress(host, config.getEsPort()));
}
LOGGER.info("^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^");
define();
}
private List<SearchRequestBuilder> makeRequestBuilders() {
List<SearchRequestBuilder> builders = new ArrayList<>();
ZonedDateTime currentDate = config.getFrom();
ZonedDateTime maxDate = up(TimeUnit.HOURS, config.getTo());
do {
builders.add(
client
.prepareSearch(
appendForEach(INDEXES_PREFIXES, ELASTIC_INDEX_DATE_FORMAT.format(currentDate)))
.setTypes(TYPES)
.setSize(config.getBucketSize())
.setPostFilter(FilterBuilders.andFilter(makeFilters(currentDate))));
currentDate = currentDate.plusDays(1);
} while (currentDate.isBefore(maxDate));
return builders;
}
