@Override
  public void index(List<Article> list) {
    if (list.size() > 0) {
      try {
        BulkRequestBuilder bulkRequest = transportClient.prepareBulk();

        for (Article article : list) {
          XContentBuilder builder =
              XContentFactory.jsonBuilder()
                  .startObject()
                  .field(TITLE, article.getTitle())
                  .field(SUMMARY, HtmlHelper.filterHtml(article.getSummary()))
                  .endObject();
          bulkRequest.add(
              transportClient
                  .prepareIndex(INDICE, TYPE, String.valueOf(article.getId()))
                  .setSource(builder));
        }

        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
        if (bulkResponse.hasFailures()) {
          logger.debug("bulkResponse.hasFailures()");
        } else {
          logger.debug("bulk index ok!");
        }
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
  }
  public void run() {
    Client client = null;
    TransportClient transportClient = null;
    try {
      transportClient = new TransportClient();
      client =
          transportClient.addTransportAddress(new InetSocketTransportAddress("192.168.1.40", 9300));

      SearchResponse response =
          client
              .prepareSearch("tms-allflat")
              .setTypes("personal")
              .setQuery(QueryBuilders.matchAllQuery())
              .addAggregation(
                  AggregationBuilders.terms("aggs1")
                      .field("skill_1")
                      .size(20)
                      .order(Terms.Order.count(false)))
              .execute()
              .actionGet();

      Terms terms = response.getAggregations().get("aggs1");
      terms
          .getBuckets()
          .stream()
          .forEach(s -> System.out.println(s.getKeyAsText() + "(" + s.getDocCount() + ")"));

    } finally {
      transportClient.close();
      client.close();
    }
  }
Beispiel #3
0
 @Bean
 public Client client() {
   TransportClient client = new TransportClient();
   TransportAddress address = new InetSocketTransportAddress(elasticHost, elasticPort);
   client.addTransportAddress(address);
   return client;
 }
    private static TransportClient open(
        final ElasticsearchReporter reporter,
        final Settings settings,
        final Iterable<TransportAddress> addresses) {
      assert settings != null;

      reporter.logNotice(format("opening shared client for %s", identify(settings)));

      try {
        TransportClient client = new TransportClient(settings);

        client.addTransportAddresses(toArray(addresses, TransportAddress.class));

        reporter.logNotice(
            format(
                "shared client for %s opened -> %s", identify(settings), toDefaultString(client)));

        return client;
      } catch (Exception failure) {
        reporter.logError(
            format("unable to open shared client for %s", identify(settings)), failure);

        throw failure;
      }
    }
  private static Client startClient(Path tempDir, TransportAddress... transportAddresses) {
    Settings clientSettings =
        Settings.settingsBuilder()
            .put("name", "qa_smoke_client_" + counter.getAndIncrement())
            .put(
                InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING,
                true) // prevents any settings to be replaced by system properties.
            .put("client.transport.ignore_cluster_name", true)
            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
            .put("node.mode", "network")
            .build(); // we require network here!

    TransportClient.Builder transportClientBuilder =
        TransportClient.builder().settings(clientSettings);
    TransportClient client =
        transportClientBuilder.build().addTransportAddresses(transportAddresses);

    logger.info("--> Elasticsearch Java TransportClient started");

    Exception clientException = null;
    try {
      ClusterHealthResponse health = client.admin().cluster().prepareHealth().get();
      logger.info(
          "--> connected to [{}] cluster which is running [{}] node(s).",
          health.getClusterName(),
          health.getNumberOfNodes());
    } catch (Exception e) {
      clientException = e;
    }

    assumeNoException(
        "Sounds like your cluster is not running at " + clusterAddresses, clientException);

    return client;
  }
  @Test
  public void testWithSniffing() throws Exception {
    TransportClient client =
        new TransportClient(
            ImmutableSettings.builder()
                .put("client.transport.sniff", true)
                .put("cluster.name", "cluster1")
                .put("node.name", "transport_client_" + this.getTestName() + "_1")
                .put("client.transport.nodes_sampler_interval", "1s")
                .put(
                    TransportModule.TRANSPORT_SERVICE_TYPE_KEY,
                    InternalTransportService.class.getName())
                .put(HEADER_SETTINGS)
                .build());
    try {
      client.addTransportAddress(address);

      InternalTransportService service =
          (InternalTransportService) client.injector.getInstance(TransportService.class);

      if (!service.clusterStateLatch.await(5, TimeUnit.SECONDS)) {
        fail("takes way too long to get the cluster state");
      }

      assertThat(client.connectedNodes().size(), is(1));
      assertThat(client.connectedNodes().get(0).getAddress(), is((TransportAddress) address));
    } finally {
      client.close();
    }
  }
  public void testPickingUpChangesInDiscoveryNode() {
    String nodeName =
        internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false));

    TransportClient client = (TransportClient) internalCluster().client(nodeName);
    assertThat(client.connectedNodes().get(0).dataNode(), equalTo(false));
  }
  public void testWithSniffing() throws Exception {
    try (TransportClient client =
        TransportClient.builder()
            .settings(
                Settings.builder()
                    .put("client.transport.sniff", true)
                    .put("cluster.name", "cluster1")
                    .put("node.name", "transport_client_" + this.getTestName() + "_1")
                    .put("client.transport.nodes_sampler_interval", "1s")
                    .put(HEADER_SETTINGS)
                    .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                    .build())
            .addPlugin(InternalTransportService.TestPlugin.class)
            .build()) {
      client.addTransportAddress(address);

      InternalTransportService service =
          (InternalTransportService) client.injector.getInstance(TransportService.class);

      if (!service.clusterStateLatch.await(5, TimeUnit.SECONDS)) {
        fail("takes way too long to get the cluster state");
      }

      assertThat(client.connectedNodes().size(), is(1));
      assertThat(client.connectedNodes().get(0).getAddress(), is((TransportAddress) address));
    }
  }
  @Override
  protected Client createClient(ImmutableSettings.Builder builder) {
    if (_transportAddresses.isEmpty()) {
      throw new IllegalStateException("There must be at least one transport address");
    }

    Class<?> clazz = getClass();

    builder.classLoader(clazz.getClassLoader());

    builder.loadFromClasspath(getConfigFileName());

    TransportClient transportClient = new TransportClient(builder);

    builder.put("client.transport.sniff", true);
    builder.put("cluster.name", getClusterName());

    for (String transportAddress : _transportAddresses) {
      String[] transportAddressParts = StringUtil.split(transportAddress, StringPool.COLON);

      try {
        InetAddress inetAddress = InetAddress.getByName(transportAddressParts[0]);

        int port = GetterUtil.getInteger(transportAddressParts[1]);

        transportClient.addTransportAddress(new InetSocketTransportAddress(inetAddress, port));
      } catch (Exception e) {
        if (_log.isWarnEnabled()) {
          _log.warn("Unable to add transport address " + transportAddress, e);
        }
      }
    }

    return transportClient;
  }
 protected void buildClient() throws Exception {
   client = TransportClient.builder().settings(settings()).build();
   Assert.hasText(clusterNodes, "[Assertion failed] clusterNodes settings missing.");
   for (String clusterNode : split(clusterNodes, COMMA)) {
     String hostName = substringBeforeLast(clusterNode, COLON);
     String port = substringAfterLast(clusterNode, COLON);
     Assert.hasText(hostName, "[Assertion failed] missing host name in 'clusterNodes'");
     Assert.hasText(port, "[Assertion failed] missing port in 'clusterNodes'");
     logger.info("adding transport node : " + clusterNode);
     client.addTransportAddress(
         new InetSocketTransportAddress(InetAddress.getByName(hostName), Integer.valueOf(port)));
   }
   client.connectedNodes();
 }
 public static void main(String[] args) {
   TransportClient client = new TransportClient();
   client.addTransportAddress(new InetSocketTransportAddress("localhost", 9300));
   TestBulkProcessor test = new TestBulkProcessor(client);
   int count = 0;
   int processors = Runtime.getRuntime().availableProcessors() - 1;
   while (count++ < 20) {
     int count2 = 0;
     while (count2++ < 3) {
       test.testBulkProcessor(processors);
     }
     processors++;
   }
   client.close();
 }
Beispiel #12
0
  /**
   * Builds the Elasticsearch client using the properties this connection was instantiated with
   *
   * @return
   * @throws SQLException
   */
  private Client buildClient() throws SQLException {
    if (props.containsKey("test")) { // used for integration tests
      return ESIntegTestCase.client();
    } else
      try {
        Settings.Builder settingsBuilder = Settings.settingsBuilder();
        for (Object key : this.props.keySet()) {
          settingsBuilder.put(key, this.props.get(key));
        }
        Settings settings = settingsBuilder.build();
        TransportClient client =
            TransportClient.builder()
                .settings(settings)
                .build()
                .addTransportAddress(
                    new InetSocketTransportAddress(InetAddress.getByName(host), port));

        // add additional hosts if set in URL query part
        if (this.props.containsKey("es.hosts"))
          for (String hostPort : this.props.getProperty("es.hosts").split(",")) {
            String newHost = hostPort.split(":")[0].trim();
            int newPort =
                (hostPort.split(":").length > 1
                    ? Integer.parseInt(hostPort.split(":")[1])
                    : Utils.PORT);
            client.addTransportAddress(
                new InetSocketTransportAddress(InetAddress.getByName(newHost), newPort));
            logger.info("Adding additional ES host: " + hostPort);
          }

        // check if index exists
        if (index != null) {
          boolean indexExists =
              client
                  .admin()
                  .indices()
                  .exists(new IndicesExistsRequest(index))
                  .actionGet()
                  .isExists();
          if (!indexExists) throw new SQLException("Index or Alias '" + index + "' does not exist");
        }
        return client;
      } catch (UnknownHostException e) {
        throw new SQLException("Unable to connect to " + host, e);
      } catch (Throwable t) {
        throw new SQLException("Unable to connect to database", t);
      }
  }
  @Override
  protected Client buildClient(Settings headersSettings, GenericAction[] testedActions) {
    TransportClient client =
        new TransportClient(
            ImmutableSettings.builder()
                .put("client.transport.sniff", false)
                .put("node.name", "transport_client_" + this.getTestName())
                .put(
                    TransportModule.TRANSPORT_SERVICE_TYPE_KEY,
                    InternalTransportService.class.getName())
                .put(HEADER_SETTINGS)
                .build());

    client.addTransportAddress(address);
    return client;
  }
 /*
  * (non-Javadoc)
  *
  * @see com.datatorrent.lib.db.Connectable#isConnected()
  */
 @Override
 public boolean isConnected() {
   if (client != null) {
     return client.connectedNodes().size() != 0;
   }
   return false;
 }
  private Client createTransportClient() {
    Settings settings =
        ImmutableSettings.settingsBuilder()
            .put("cluster.name", configuration.getClusterName())
            .build();
    TransportClient transportClient = new TransportClient(settings);

    List<HostAddress> adresses = configuration.getHostsAddresses();

    for (HostAddress address : adresses) {
      transportClient.addTransportAddress(
          new InetSocketTransportAddress(address.getHostname(), address.getPort()));
    }

    return transportClient;
  }
  @Override
  public void notify(DelegateExecution execution) throws Exception {

    HistoryService historyService = execution.getProcessEngineServices().getHistoryService();

    HistoricDecisionInstance historicDecisionInstance =
        historyService
            .createHistoricDecisionInstanceQuery()
            .includeInputs()
            .includeOutputs()
            .decisionDefinitionKey((String) execution.getVariableLocal("tableName"))
            .processInstanceId(execution.getProcessInstanceId())
            .singleResult();

    // Fill a new object with stuff...
    FraudScoreTableObject fraudData = new FraudScoreTableObject();

    fraudData.setFraudInstanceID(historicDecisionInstance.getId());

    List<HistoricDecisionInputInstance> inputs = historicDecisionInstance.getInputs();
    for (HistoricDecisionInputInstance historicDecisionInputInstance : inputs) {
      String inputName = historicDecisionInputInstance.getTypeName();
      if (inputName.equals("paymentRejected")) {
        fraudData.setPaymentRejected((Boolean) historicDecisionInputInstance.getValue());
      } else if (inputName.equals("numberOfPayouts")) {
        fraudData.setNumberOfPayouts((Integer) historicDecisionInputInstance.getValue());
      } else if (inputName.equals("historyOfFraud")) {
        fraudData.setHistoryOfFraud((Boolean) historicDecisionInputInstance.getValue());
      } else if (inputName.equals("claimAmount")) {
        fraudData.setCalimAmount((Long) historicDecisionInputInstance.getValue());
      }
    }
    List<HistoricDecisionOutputInstance> outputs = historicDecisionInstance.getOutputs();
    for (HistoricDecisionOutputInstance historicDecisionOutputInstance : outputs) {

      String inputName = historicDecisionOutputInstance.getTypeName();
      if (inputName.equals("frausScore")) {
        fraudData.setFraudScore((Integer) historicDecisionOutputInstance.getValue());
      }
    }

    ObjectMapper mapper = new ObjectMapper();

    String serializedHistoricDecisionInstance =
        mapper.writerWithDefaultPrettyPrinter().writeValueAsString(fraudData);

    Client client =
        TransportClient.builder()
            .build()
            .addTransportAddress(
                new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300));

    IndexResponse response =
        client
            .prepareIndex("camunda", "fraudData", historicDecisionInstance.getId())
            .setSource(serializedHistoricDecisionInstance)
            .get();

    LOGGER.info(response.getId());
  }
  @Override
  public List<Article> query(String queryString, int size) {
    Assert.hasText(queryString, "queryString must not be empty!");
    SearchResponse searchResponse =
        transportClient
            .prepareSearch(INDICE)
            .setTypes(TYPE)
            .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
            .setQuery(QueryBuilders.queryString(queryString))
            .setFrom(0)
            .setSize(size)
            .setExplain(true)
            .execute()
            .actionGet();

    SearchHits hits = searchResponse.getHits();
    long total = hits.getTotalHits();
    logger.debug("search articles result total:{}", total);
    List<Article> list = new ArrayList<Article>();
    Article article = null;
    for (SearchHit hit : hits) {

      Long id = (Long) hit.getSource().get("id");
      String title = (String) hit.getSource().get(TITLE);
      String summary = (String) hit.getSource().get(SUMMARY);
      logger.debug("id:{},title:{},summary:{}", id, title, summary);

      article = new Article();
      article.setId(id);
      article.setTitle(title);
      article.setSummary(summary);
      list.add(article);
    }
    return list;
  }
  public void testNodeVersionIsUpdated() {
    TransportClient client = (TransportClient) internalCluster().client();
    TransportClientNodesService nodeService = client.nodeService();
    Node node =
        new Node(
            Settings.builder()
                .put(internalCluster().getDefaultSettings())
                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                .put("node.name", "testNodeVersionIsUpdated")
                .put("http.enabled", false)
                .put(Node.NODE_DATA_SETTING.getKey(), false)
                .put("cluster.name", "foobar")
                .put(
                    InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(),
                    true) // make sure we get what we set :)
                .build());
    node.start();
    try {
      TransportAddress transportAddress =
          node.injector().getInstance(TransportService.class).boundAddress().publishAddress();
      client.addTransportAddress(transportAddress);
      assertThat(
          nodeService.connectedNodes().size(),
          greaterThanOrEqualTo(
              1)); // since we force transport clients there has to be one node started that we
                   // connect to.
      for (DiscoveryNode discoveryNode :
          nodeService.connectedNodes()) { // connected nodes have updated version
        assertThat(discoveryNode.getVersion(), equalTo(Version.CURRENT));
      }

      for (DiscoveryNode discoveryNode : nodeService.listedNodes()) {
        assertThat(discoveryNode.id(), startsWith("#transport#-"));
        assertThat(
            discoveryNode.getVersion(), equalTo(Version.CURRENT.minimumCompatibilityVersion()));
      }

      assertThat(nodeService.filteredNodes().size(), equalTo(1));
      for (DiscoveryNode discoveryNode : nodeService.filteredNodes()) {
        assertThat(
            discoveryNode.getVersion(), equalTo(Version.CURRENT.minimumCompatibilityVersion()));
      }
    } finally {
      node.close();
    }
  }
 public void testThatTransportClientSettingCannotBeChanged() {
   Settings baseSettings =
       settingsBuilder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
   try (TransportClient client = TransportClient.builder().settings(baseSettings).build()) {
     Settings settings = client.injector.getInstance(Settings.class);
     assertThat(Client.CLIENT_TYPE_SETTING_S.get(settings), is("transport"));
   }
 }
 /**
  * LocalElasticSearchClient.
  *
  * @return a {@link org.elasticsearch.client.Client} object.
  * @throws java.net.UnknownHostException if any.
  */
 @Profile("local")
 @Bean(name = "elasticSearchClient")
 public Client localElasticSearchClient() throws UnknownHostException {
   return TransportClient.builder()
       .build()
       .addTransportAddress(
           new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), 9300));
 }
  @Override
  protected Client buildClient(Settings headersSettings, GenericAction[] testedActions) {
    TransportClient client =
        TransportClient.builder()
            .settings(
                Settings.builder()
                    .put("client.transport.sniff", false)
                    .put("cluster.name", "cluster1")
                    .put("node.name", "transport_client_" + this.getTestName())
                    .put(headersSettings)
                    .build())
            .addPlugin(InternalTransportService.TestPlugin.class)
            .build();

    client.addTransportAddress(address);
    return client;
  }
Beispiel #22
0
 @Override
 public Client client(Node node, String clusterName, Random random) {
   TransportAddress addr =
       ((InternalNode) node)
           .injector()
           .getInstance(TransportService.class)
           .boundAddress()
           .publishAddress();
   TransportClient client =
       new TransportClient(
           settingsBuilder()
               .put("client.transport.nodes_sampler_interval", "1s")
               .put("name", "transport_client_" + node.settings().get("name"))
               .put("cluster.name", clusterName)
               .put("client.transport.sniff", sniff)
               .build());
   client.addTransportAddress(addr);
   return client;
 }
  private void initArticleIndexMapping() {
    try {
      boolean isExists = ElasticsearchHelper.isExistsIndex(transportClient, INDICE);
      if (!isExists) {
        // create index
        transportClient.admin().indices().prepareCreate(INDICE).execute().actionGet();
        // crate mapping
        XContentBuilder mapping =
            XContentFactory.jsonBuilder()
                .startObject()
                .startObject(TYPE)
                .startObject("_all")
                .field("indexAnalyzer", "ik")
                .field("searchAnalyzer", "ik")
                .endObject()
                .startObject("properties")
                .startObject(TITLE)
                .field("type", "string")
                .field("indexAnalyzer", "ik")
                .field("searchAnalyzer", "ik")
                .endObject()
                .startObject(SUMMARY)
                .field("type", "string")
                .field("indexAnalyzer", "ik")
                .field("searchAnalyzer", "ik")
                .endObject()
                .endObject()
                .endObject()
                .endObject();

        PutMappingRequest mappingRequest =
            Requests.putMappingRequest(INDICE).type(TYPE).source(mapping);
        transportClient.admin().indices().putMapping(mappingRequest).actionGet();

        logger.debug("create index and mapping are success!");
      } else {
        logger.debug("Index already exists!");
      }

    } catch (Exception e) {
      logger.error("create index and mapping are failure!", e);
    }
  }
 @Override
 public void destroy() throws Exception {
   try {
     logger.info("Closing elasticSearch  client");
     if (client != null) {
       client.close();
     }
   } catch (final Exception e) {
     logger.error("Error closing ElasticSearch client: ", e);
   }
 }
 @Override
 public void deleteAll() {
   MatchAllQueryBuilder allQueryBuilder = QueryBuilders.matchAllQuery();
   transportClient
       .prepareDeleteByQuery(INDICE)
       .setTypes(TYPE)
       .setQuery(allQueryBuilder)
       .execute()
       .actionGet();
   logger.debug("delete All ");
 }
  synchronized Client getClient() {
    if (nodeInfo == null) {
      throw new IllegalStateException("Node has not started yet");
    }
    if (client == null) {
      TransportAddress addr = nodeInfo.getTransport().getAddress().publishAddress();
      // verify that the end node setting will have network enabled.

      Settings clientSettings =
          settingsBuilder()
              .put(externalNodeSettings)
              .put("client.transport.nodes_sampler_interval", "1s")
              .put("name", "transport_client_" + nodeInfo.getNode().name())
              .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName)
              .put("client.transport.sniff", false)
              .build();
      TransportClient client = TransportClient.builder().settings(clientSettings).build();
      client.addTransportAddress(addr);
      this.client = client;
    }
    return client;
  }
 /**
  * iLabElasticSearchClient.
  *
  * @return a {@link org.elasticsearch.client.Client} object.
  * @throws java.net.UnknownHostException if any.
  */
 @Profile("iLab")
 @Bean(name = "elasticSearchClient")
 public Client iLabElasticSearchClient() throws UnknownHostException {
   Settings settings =
       Settings.settingsBuilder()
           .put("cluster.name", Constants.APPLICATIONNAME)
           .put("client.transport.sniff", true)
           .build();
   return TransportClient.builder()
       .settings(settings)
       .build()
       .addTransportAddress(
           new InetSocketTransportAddress(InetAddress.getByName("152.190.139.77"), 9300));
 }
 synchronized void stop() throws InterruptedException {
   if (running()) {
     try {
       if (this.client != null) {
         client.close();
       }
     } finally {
       process.destroy();
       process.waitFor();
       process = null;
       nodeInfo = null;
     }
   }
 }
  @PostConstruct
  public void init() {
    final ImmutableSettings.Builder settings = settingsBuilder();
    settings.put("cluster.name", "es-events");
    client = new TransportClient(settings);
    LOGGER.info("Adding ES hosts:");
    for (String host : config.getEsHosts().split(",")) {
      LOGGER.info(host);
      client.addTransportAddress(new InetSocketTransportAddress(host, config.getEsPort()));
    }
    LOGGER.info("^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^");

    define();
  }
 private List<SearchRequestBuilder> makeRequestBuilders() {
   List<SearchRequestBuilder> builders = new ArrayList<>();
   ZonedDateTime currentDate = config.getFrom();
   ZonedDateTime maxDate = up(TimeUnit.HOURS, config.getTo());
   do {
     builders.add(
         client
             .prepareSearch(
                 appendForEach(INDEXES_PREFIXES, ELASTIC_INDEX_DATE_FORMAT.format(currentDate)))
             .setTypes(TYPES)
             .setSize(config.getBucketSize())
             .setPostFilter(FilterBuilders.andFilter(makeFilters(currentDate))));
     currentDate = currentDate.plusDays(1);
   } while (currentDate.isBefore(maxDate));
   return builders;
 }