public static RuntimeJestClient createClient() {
String esURL = fromEnv(ES_URL, "http://localhost:9200");
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(new HttpClientConfig.Builder(esURL).multiThreaded(true).build());
return new RuntimeJestClientAdapter(factory.getObject());
}
private JestClient createJestClient(String connectionUrl) {
log.info("Create a jest elastic search client for connection url [{}]", connectionUrl);
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(
new HttpClientConfig.Builder(connectionUrl).multiThreaded(true).build());
return factory.getObject();
}
/**
* Check commandline parameters and run the indexer for all configured languages.
*
* @param args Commandline arguments.
* @throws IOException Thrown, if target directory is not accessible
*/
private void runMain(@NotNull final String... args) throws Exception {
new CmdLineParser(this.cliParams);
// parser.parseArgument(args);
parseWithHelp(this.cliParams, args);
// check, if files and directories are sane
this.cliParams.check();
// create target, if it does not exist already
Files.createDirectories(this.cliParams.dataDir.toPath());
// setup REST client
final JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(new Builder(ES_CONF.URL).multiThreaded(true).build());
this.client = factory.getObject();
// languages to index, initially empty
Collection<Language> runLanguages = Collections.emptyList();
// decide which languages to index
if (this.cliParams.onlyLang != null) {
// create an index for a single language only
LOG.info("Processing language '{}' only as requested by user.", this.cliParams.onlyLang);
final Language onlyLang = Language.getByString(this.cliParams.onlyLang);
if (onlyLang != null) {
runLanguages = Collections.singletonList(onlyLang);
} else {
LOG.error("Unknown language '{}'.", this.cliParams.onlyLang);
}
} else {
// create an index for each known language optionally skipping single ones
runLanguages = Arrays.asList(Language.values());
// optionally skip languages
if (this.cliParams.skipLang.length > 0) {
LOG.info("Skipping languages {} as requested by user.", this.cliParams.skipLang);
for (final String skipLang : this.cliParams.skipLang) {
final Language skipSrcLang = Language.getByString(skipLang);
if (skipSrcLang != null) {
runLanguages.remove(skipSrcLang);
}
}
}
}
// run index for each specified language
for (final Language lng : runLanguages) {
try {
indexByLanguage(lng);
} catch (final Exception e) {
LOG.error("Indexing failed. lang={}", lng, e);
throw e;
}
}
// close connection
this.client.shutdownClient();
}
/**
* jestClient.
*
* @return a {@link io.searchbox.client.http.JestHttpClient} object.
*/
@Bean
public JestHttpClient jestClient() {
// Construct a new Jest client according to configuration via factory
JestClientFactory factory = new JestClientFactory();
HttpClientConfig httpClientConfig =
new HttpClientConfig.Builder(getConnectionURL())
.multiThreaded(true)
.connTimeout(Constants.CONNTIMEOUT)
.readTimeout(Constants.READTIMEOUT)
.maxTotalConnection(Constants.MAXTOTALCONNECTION)
// .requestCompressionEnabled(true)
.gson(new GsonBuilder().setDateFormat(DATESTYLE).create())
.build();
factory.setHttpClientConfig(httpClientConfig);
JestHttpClient jestHttpClient = (JestHttpClient) factory.getObject();
CloseableHttpClient closableHttpClient = HttpClientBuilder.create().build();
jestHttpClient.setHttpClient(closableHttpClient);
return jestHttpClient;
}
public ScanAndScroll(
String url,
String username,
String password,
String outputFilePath,
int outputType,
String outputFile)
throws FileNotFoundException, UnsupportedEncodingException {
SSLContextBuilder builder = new SSLContextBuilder();
SSLConnectionSocketFactory sslsf = null;
try {
builder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
sslsf = new SSLConnectionSocketFactory(builder.build());
} catch (Exception e) {
LOG.error(e.getMessage());
}
System.out.println(url);
HttpClientConfig.Builder httpClientBuilder =
new HttpClientConfig.Builder(url)
.sslSocketFactory(sslsf)
.readTimeout(30000)
.multiThreaded(false);
if (username.trim() != "" && password.trim() != "") {
httpClientBuilder.defaultCredentials(username, password);
}
JestClientFactory jcf = new JestClientFactory();
jcf.setHttpClientConfig(httpClientBuilder.build());
this.client = jcf.getObject();
this.outputType = outputType;
this.outputFile = outputFile;
}
/** Stuff to do before the server is started. */
protected void preStart() throws Exception {
if (ManagerTestUtils.getTestType() == TestType.jpa) {
TestUtil.setProperty("apiman.hibernate.hbm2ddl.auto", "create-drop");
TestUtil.setProperty(
"apiman.hibernate.connection.datasource", "java:/comp/env/jdbc/ApiManagerDS");
try {
InitialContext ctx = new InitialContext();
ensureCtx(ctx, "java:/comp/env");
ensureCtx(ctx, "java:/comp/env/jdbc");
String dbOutputPath = System.getProperty("apiman.test.h2-output-dir", null);
if (dbOutputPath != null) {
ds = createFileDatasource(new File(dbOutputPath));
} else {
ds = createInMemoryDatasource();
}
ctx.bind("java:/comp/env/jdbc/ApiManagerDS", ds);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
if (ManagerTestUtils.getTestType() == TestType.es && node == null) {
System.out.println("Creating the ES node.");
File esHome = new File("target/es");
String esHomeSP = System.getProperty("apiman.test.es-home", null);
if (esHomeSP != null) {
esHome = new File(esHomeSP);
}
if (esHome.isDirectory()) {
FileUtils.deleteDirectory(esHome);
}
Builder settings = NodeBuilder.nodeBuilder().settings();
settings.put("path.home", esHome.getAbsolutePath());
settings.put("http.port", "6500-6600");
settings.put("transport.tcp.port", "6600-6700");
settings.put("discovery.zen.ping.multicast.enabled", false);
String clusterName = System.getProperty("apiman.test.es-cluster-name", ES_CLUSTER_NAME);
boolean isPersistent =
"true".equals(System.getProperty("apiman.test.es-persistence", "false"));
if (!isPersistent) {
System.out.println("Creating non-persistent ES");
settings
.put("index.store.type", "memory")
.put("gateway.type", "none")
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 1);
node =
NodeBuilder.nodeBuilder()
.client(false)
.clusterName(clusterName)
.data(true)
.local(true)
.settings(settings)
.build();
} else {
System.out.println("Creating *persistent* ES here: " + esHome);
node =
NodeBuilder.nodeBuilder()
.client(false)
.clusterName(clusterName)
.data(true)
.local(false)
.settings(settings)
.build();
}
System.out.println("Starting the ES node.");
node.start();
System.out.println("ES node was successfully started.");
// TODO parameterize this
String connectionUrl = "http://localhost:6500";
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(
new HttpClientConfig.Builder(connectionUrl).multiThreaded(true).build());
client = factory.getObject();
ES_CLIENT = client;
}
}