@Override
public void run() {
try {
URL url = buildPingUrl();
if (logger.isDebugEnabled()) {
logger.debug("Sending UDC information to {}...", url);
}
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout((int) HTTP_TIMEOUT.millis());
conn.setReadTimeout((int) HTTP_TIMEOUT.millis());
if (conn.getResponseCode() >= 300) {
throw new Exception(
String.format("%s Responded with Code %d", url.getHost(), conn.getResponseCode()));
}
if (logger.isDebugEnabled()) {
BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
String line = reader.readLine();
while (line != null) {
logger.debug(line);
line = reader.readLine();
}
reader.close();
} else {
conn.getInputStream().close();
}
successCounter.incrementAndGet();
} catch (Exception e) {
if (logger.isDebugEnabled()) {
logger.debug("Error sending UDC information", e);
}
failCounter.incrementAndGet();
}
}
public static TimeValue nodeTimeValue(Object node) {
if (node instanceof Number) {
return TimeValue.timeValueMillis(((Number) node).longValue());
}
return TimeValue.parseTimeValue(
node.toString(), null, XContentMapValues.class.getSimpleName() + ".nodeTimeValue");
}
public void testInitialSearchParamsMisc() {
SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
Version remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
TimeValue scroll = null;
if (randomBoolean()) {
scroll = TimeValue.parseTimeValue(randomPositiveTimeValue(), "test");
searchRequest.scroll(scroll);
}
int size = between(0, Integer.MAX_VALUE);
searchRequest.source().size(size);
Boolean fetchVersion = null;
if (randomBoolean()) {
fetchVersion = randomBoolean();
searchRequest.source().version(fetchVersion);
}
Map<String, String> params = initialSearchParams(searchRequest, remoteVersion);
assertThat(
params, scroll == null ? not(hasKey("scroll")) : hasEntry("scroll", scroll.toString()));
assertThat(params, hasEntry("size", Integer.toString(size)));
assertThat(
params,
fetchVersion == null || fetchVersion == true
? hasEntry("version", null)
: not(hasEntry("version", null)));
assertThat(params, hasEntry("_source", "true"));
}
public void addDeletion(SearchRequestBuilder searchRequest) {
searchRequest
.addSort("_doc", SortOrder.ASC)
.setScroll(TimeValue.timeValueMinutes(5))
.setSize(100)
// load only doc ids, not _source fields
.setFetchSource(false);
// this search is synchronous. An optimization would be to be non-blocking,
// but it requires to tracking pending requests in close().
// Same semaphore can't be reused because of potential deadlock (requires to acquire
// two locks)
SearchResponse searchResponse = searchRequest.get();
while (true) {
SearchHit[] hits = searchResponse.getHits().getHits();
for (SearchHit hit : hits) {
DeleteRequestBuilder deleteRequestBuilder =
client.prepareDelete(hit.index(), hit.type(), hit.getId());
SearchHitField routing = hit.field("_routing");
if (routing != null) {
deleteRequestBuilder.setRouting(routing.getValue());
}
add(deleteRequestBuilder.request());
}
String scrollId = searchResponse.getScrollId();
searchResponse =
client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get();
if (hits.length == 0) {
client.nativeClient().prepareClearScroll().addScrollId(scrollId).get();
break;
}
}
}
private TestResult(
String name, long avgRefreshTime, long avgQueryTime, ByteSizeValue fieldDataSizeInMemory) {
this.name = name;
this.avgRefreshTime = TimeValue.timeValueMillis(avgRefreshTime);
this.avgQueryTime = TimeValue.timeValueMillis(avgQueryTime);
this.fieldDataSizeInMemory = fieldDataSizeInMemory;
}
public void testUTCIntervalRounding() {
Rounding tzRounding = Rounding.builder(TimeValue.timeValueHours(12)).build();
DateTimeZone tz = DateTimeZone.UTC;
assertThat(
tzRounding.round(time("2009-02-03T01:01:01")),
isDate(time("2009-02-03T00:00:00.000Z"), tz));
assertThat(
tzRounding.nextRoundingValue(time("2009-02-03T00:00:00.000Z")),
isDate(time("2009-02-03T12:00:00.000Z"), tz));
assertThat(
tzRounding.round(time("2009-02-03T13:01:01")),
isDate(time("2009-02-03T12:00:00.000Z"), tz));
assertThat(
tzRounding.nextRoundingValue(time("2009-02-03T12:00:00.000Z")),
isDate(time("2009-02-04T00:00:00.000Z"), tz));
tzRounding = Rounding.builder(TimeValue.timeValueHours(48)).build();
assertThat(
tzRounding.round(time("2009-02-03T01:01:01")),
isDate(time("2009-02-03T00:00:00.000Z"), tz));
assertThat(
tzRounding.nextRoundingValue(time("2009-02-03T00:00:00.000Z")),
isDate(time("2009-02-05T00:00:00.000Z"), tz));
assertThat(
tzRounding.round(time("2009-02-05T13:01:01")),
isDate(time("2009-02-05T00:00:00.000Z"), tz));
assertThat(
tzRounding.nextRoundingValue(time("2009-02-05T00:00:00.000Z")),
isDate(time("2009-02-07T00:00:00.000Z"), tz));
}
/**
* Creates a new TranslogConfig instance
*
* @param shardId the shard ID this translog belongs to
* @param translogPath the path to use for the transaction log files
* @param indexSettings the index settings used to set internal variables
* @param durabilty the default durability setting for the translog
* @param bigArrays a bigArrays instance used for temporarily allocating write operations
* @param threadPool a {@link ThreadPool} to schedule async sync durability
*/
public TranslogConfig(
ShardId shardId,
Path translogPath,
Settings indexSettings,
Translog.Durabilty durabilty,
BigArrays bigArrays,
@Nullable ThreadPool threadPool) {
this.indexSettings = indexSettings;
this.shardId = shardId;
this.translogPath = translogPath;
this.durabilty = durabilty;
this.threadPool = threadPool;
this.bigArrays = bigArrays;
this.type =
TranslogWriter.Type.fromString(
indexSettings.get(INDEX_TRANSLOG_FS_TYPE, TranslogWriter.Type.BUFFERED.name()));
this.bufferSize =
(int)
indexSettings
.getAsBytesSize(
INDEX_TRANSLOG_BUFFER_SIZE,
IndexingMemoryController.INACTIVE_SHARD_TRANSLOG_BUFFER)
.bytes(); // Not really interesting, updated by IndexingMemoryController...
syncInterval =
indexSettings.getAsTime(INDEX_TRANSLOG_SYNC_INTERVAL, TimeValue.timeValueSeconds(5));
if (syncInterval.millis() > 0 && threadPool != null) {
syncOnEachOperation = false;
} else if (syncInterval.millis() == 0) {
syncOnEachOperation = true;
} else {
syncOnEachOperation = false;
}
}
public static void main(String[] args) throws Exception {
System.setProperty("es.logger.prefix", "");
Settings settings =
settingsBuilder()
.put("index.shard.check_index", true)
.put("gateway.type", "none")
.put("path.data", "data/data1,data/data2")
.build();
RollingRestartStressTest test =
new RollingRestartStressTest()
.settings(settings)
.numberOfNodes(4)
.numberOfShards(5)
.numberOfReplicas(1)
.initialNumberOfDocs(1000)
.textTokens(150)
.numberOfFields(10)
.cleanNodeData(false)
.indexers(5)
.indexerThrottle(TimeValue.timeValueMillis(50))
.period(TimeValue.timeValueMinutes(3));
test.run();
}
public void testClusterHealth() throws IOException {
ClusterState clusterState =
ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
int pendingTasks = randomIntBetween(0, 200);
int inFlight = randomIntBetween(0, 200);
int delayedUnassigned = randomIntBetween(0, 200);
TimeValue pendingTaskInQueueTime = TimeValue.timeValueMillis(randomIntBetween(1000, 100000));
ClusterHealthResponse clusterHealth =
new ClusterHealthResponse(
"bla",
new String[] {MetaData.ALL},
clusterState,
pendingTasks,
inFlight,
delayedUnassigned,
pendingTaskInQueueTime);
clusterHealth = maybeSerialize(clusterHealth);
assertClusterHealth(clusterHealth);
assertThat(clusterHealth.getNumberOfPendingTasks(), Matchers.equalTo(pendingTasks));
assertThat(clusterHealth.getNumberOfInFlightFetch(), Matchers.equalTo(inFlight));
assertThat(clusterHealth.getDelayedUnassignedShards(), Matchers.equalTo(delayedUnassigned));
assertThat(clusterHealth.getTaskMaxWaitingTime().millis(), is(pendingTaskInQueueTime.millis()));
assertThat(
clusterHealth.getActiveShardsPercent(),
is(allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(100.0))));
}
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
TTLFieldMapper.Builder builder = ttl();
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
EnabledAttributeMapper enabledState =
nodeBooleanValue(fieldNode)
? EnabledAttributeMapper.ENABLED
: EnabledAttributeMapper.DISABLED;
builder.enabled(enabledState);
iterator.remove();
} else if (fieldName.equals("default")) {
TimeValue ttlTimeValue = nodeTimeValue(fieldNode, null);
if (ttlTimeValue != null) {
builder.defaultTTL(ttlTimeValue.millis());
}
iterator.remove();
}
}
return builder;
}
void checkBulkAction(boolean indexShouldBeAutoCreated, BulkRequestBuilder builder) {
// bulk operation do not throw MasterNotDiscoveredException exceptions. The only test that auto
// create kicked in and failed is
// via the timeout, as bulk operation do not wait on blocks.
TimeValue timeout;
if (indexShouldBeAutoCreated) {
// we expect the bulk to fail because it will try to go to the master. Use small timeout and
// detect it has passed
timeout = new TimeValue(200);
} else {
// the request should fail very quickly - use a large timeout and make sure it didn't pass...
timeout = new TimeValue(5000);
}
builder.setTimeout(timeout);
long now = System.currentTimeMillis();
try {
builder.get();
fail("Expected ClusterBlockException");
} catch (ClusterBlockException e) {
if (indexShouldBeAutoCreated) {
// timeout is 200
assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
} else {
// timeout is 5000
assertThat(System.currentTimeMillis() - now, lessThan(timeout.millis() - 50));
}
}
}
@Test
public void testRoundingWithTimeZone() {
MutableDateTime time = new MutableDateTime(DateTimeZone.UTC);
time.setZone(DateTimeZone.forOffsetHours(-2));
time.setRounding(time.getChronology().dayOfMonth(), MutableDateTime.ROUND_FLOOR);
MutableDateTime utcTime = new MutableDateTime(DateTimeZone.UTC);
utcTime.setRounding(utcTime.getChronology().dayOfMonth(), MutableDateTime.ROUND_FLOOR);
time.setMillis(utcTimeInMillis("2009-02-03T01:01:01"));
utcTime.setMillis(utcTimeInMillis("2009-02-03T01:01:01"));
assertThat(time.toString(), equalTo("2009-02-02T00:00:00.000-02:00"));
assertThat(utcTime.toString(), equalTo("2009-02-03T00:00:00.000Z"));
// the time is on the 2nd, and utcTime is on the 3rd, but, because time already encapsulates
// time zone, the millis diff is not 24, but 22 hours
assertThat(
time.getMillis(), equalTo(utcTime.getMillis() - TimeValue.timeValueHours(22).millis()));
time.setMillis(utcTimeInMillis("2009-02-04T01:01:01"));
utcTime.setMillis(utcTimeInMillis("2009-02-04T01:01:01"));
assertThat(time.toString(), equalTo("2009-02-03T00:00:00.000-02:00"));
assertThat(utcTime.toString(), equalTo("2009-02-04T00:00:00.000Z"));
assertThat(
time.getMillis(), equalTo(utcTime.getMillis() - TimeValue.timeValueHours(22).millis()));
}
@Override
protected void doExecute(
final KnapsackPullRequest request, ActionListener<KnapsackPullResponse> listener) {
final KnapsackState state =
new KnapsackState().setMode("pull").setNodeName(nodeService.nodeName());
final KnapsackPullResponse response = new KnapsackPullResponse().setState(state);
try {
final BulkTransportClient transportClient =
ClientBuilder.builder()
.put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
.put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
.put(clientSettings(client, request))
.toBulkTransportClient();
final BulkNodeClient nodeClient =
ClientBuilder.builder()
.put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
.put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
.toBulkNodeClient(client);
state.setTimestamp(new DateTime());
response.setRunning(true);
knapsack.submit(
new Thread() {
public void run() {
performPull(request, state, transportClient, nodeClient);
}
});
listener.onResponse(response);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
listener.onFailure(e);
}
}
private void waitForOutstandingRequests(
TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
long start = System.currentTimeMillis();
do {
long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
logger.info(
"[{}] going to try and aquire [{}] in [{}]ms [{}] available to aquire right now",
name,
maxRequests,
msRemaining,
requestsOutstanding.availablePermits());
try {
requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
return;
} catch (InterruptedException ie) {
// Just keep swimming
}
} while ((System.currentTimeMillis() - start) < timeOut.getMillis());
throw new ElasticsearchTimeoutException(
"Requests were still outstanding after the timeout ["
+ timeOut
+ "] for type ["
+ name
+ "]");
}
@Test
public void testDefaultRecoverAfterTime() throws IOException {
// check that the default is not set
GatewayService service = createService(Settings.builder());
assertNull(service.recoverAfterTime());
// ensure default is set when setting expected_nodes
service = createService(Settings.builder().put("gateway.expected_nodes", 1));
assertThat(
service.recoverAfterTime(),
Matchers.equalTo(GatewayService.DEFAULT_RECOVER_AFTER_TIME_IF_EXPECTED_NODES_IS_SET));
// ensure default is set when setting expected_data_nodes
service = createService(Settings.builder().put("gateway.expected_data_nodes", 1));
assertThat(
service.recoverAfterTime(),
Matchers.equalTo(GatewayService.DEFAULT_RECOVER_AFTER_TIME_IF_EXPECTED_NODES_IS_SET));
// ensure default is set when setting expected_master_nodes
service = createService(Settings.builder().put("gateway.expected_master_nodes", 1));
assertThat(
service.recoverAfterTime(),
Matchers.equalTo(GatewayService.DEFAULT_RECOVER_AFTER_TIME_IF_EXPECTED_NODES_IS_SET));
// ensure settings override default
TimeValue timeValue = TimeValue.timeValueHours(3);
// ensure default is set when setting expected_nodes
service =
createService(
Settings.builder()
.put("gateway.expected_nodes", 1)
.put("gateway.recover_after_time", timeValue.toString()));
assertThat(service.recoverAfterTime().millis(), Matchers.equalTo(timeValue.millis()));
}
public void start() throws Exception {
for (Thread t : searcherThreads) {
t.start();
}
for (Thread t : indexThreads) {
t.start();
}
barrier1.await();
StopWatch stopWatch = new StopWatch();
stopWatch.start();
barrier2.await();
latch.await();
stopWatch.stop();
System.out.println("Done, took [" + stopWatch.totalTime() + "]");
System.out.println("Sleeping before close: " + sleepBeforeClose);
Thread.sleep(sleepBeforeClose.millis());
for (Client client : clients) {
client.close();
}
for (Node node : nodes) {
node.close();
}
System.out.println("Sleeping before exit: " + sleepBeforeClose);
Thread.sleep(sleepAfterDone.millis());
}
@Inject
public LocalGatewayMetaState(
Settings settings,
ThreadPool threadPool,
NodeEnvironment nodeEnv,
TransportNodesListGatewayMetaState nodesListGatewayMetaState,
LocalAllocateDangledIndices allocateDangledIndices,
NodeIndexDeletedAction nodeIndexDeletedAction)
throws Exception {
super(settings);
this.nodeEnv = nodeEnv;
this.threadPool = threadPool;
this.format = XContentType.fromRestContentType(settings.get("format", "smile"));
this.allocateDangledIndices = allocateDangledIndices;
this.nodeIndexDeletedAction = nodeIndexDeletedAction;
nodesListGatewayMetaState.init(this);
if (this.format == XContentType.SMILE) {
Map<String, String> params = Maps.newHashMap();
params.put("binary", "true");
formatParams = new ToXContent.MapParams(params);
Map<String, String> globalOnlyParams = Maps.newHashMap();
globalOnlyParams.put("binary", "true");
globalOnlyParams.put(MetaData.PERSISTENT_ONLY_PARAM, "true");
globalOnlyParams.put(MetaData.GLOBAL_ONLY_PARAM, "true");
globalOnlyFormatParams = new ToXContent.MapParams(globalOnlyParams);
} else {
formatParams = ToXContent.EMPTY_PARAMS;
Map<String, String> globalOnlyParams = Maps.newHashMap();
globalOnlyParams.put(MetaData.PERSISTENT_ONLY_PARAM, "true");
globalOnlyParams.put(MetaData.GLOBAL_ONLY_PARAM, "true");
globalOnlyFormatParams = new ToXContent.MapParams(globalOnlyParams);
}
this.autoImportDangled =
AutoImportDangledState.fromString(
settings.get(
"gateway.local.auto_import_dangled", AutoImportDangledState.YES.toString()));
this.danglingTimeout =
settings.getAsTime("gateway.local.dangling_timeout", TimeValue.timeValueHours(2));
logger.debug(
"using gateway.local.auto_import_dangled [{}], with gateway.local.dangling_timeout [{}]",
this.autoImportDangled,
this.danglingTimeout);
if (DiscoveryNode.masterNode(settings)) {
try {
pre019Upgrade();
long start = System.currentTimeMillis();
loadState();
logger.debug(
"took {} to load state", TimeValue.timeValueMillis(System.currentTimeMillis() - start));
} catch (Exception e) {
logger.error("failed to read local state, exiting...", e);
throw e;
}
}
}
public XContentBuilder timeValueField(
String rawFieldName, String readableFieldName, TimeValue timeValue) throws IOException {
if (humanReadable) {
field(readableFieldName, timeValue.toString());
}
field(rawFieldName, timeValue.millis());
return this;
}
public static void main(String[] args) {
final int numberOfRuns = 1;
final int numIndices = 5 * 365; // five years
final int numShards = 6;
final int numReplicas = 2;
final int numberOfNodes = 30;
final int numberOfTags = 2;
AllocationService strategy =
ElasticsearchAllocationTestCase.createAllocationService(
ImmutableSettings.EMPTY, new Random(1));
MetaData.Builder mb = MetaData.builder();
for (int i = 1; i <= numIndices; i++) {
mb.put(
IndexMetaData.builder("test_" + i)
.numberOfShards(numShards)
.numberOfReplicas(numReplicas));
}
MetaData metaData = mb.build();
RoutingTable.Builder rb = RoutingTable.builder();
for (int i = 1; i <= numIndices; i++) {
rb.addAsNew(metaData.index("test_" + i));
}
RoutingTable routingTable = rb.build();
DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
for (int i = 1; i <= numberOfNodes; i++) {
nb.put(newNode("node" + i, ImmutableMap.of("tag", "tag_" + (i % numberOfTags))));
}
ClusterState initialClusterState =
ClusterState.builder().metaData(metaData).routingTable(routingTable).nodes(nb).build();
long start = System.currentTimeMillis();
for (int i = 0; i < numberOfRuns; i++) {
logger.info("[{}] starting... ", i);
long runStart = System.currentTimeMillis();
ClusterState clusterState = initialClusterState;
while (clusterState.readOnlyRoutingNodes().hasUnassignedShards()) {
logger.info(
"[{}] remaining unassigned {}",
i,
clusterState.readOnlyRoutingNodes().unassigned().size());
RoutingAllocation.Result result =
strategy.applyStartedShards(
clusterState, clusterState.readOnlyRoutingNodes().shardsWithState(INITIALIZING));
clusterState = ClusterState.builder(clusterState).routingResult(result).build();
result = strategy.reroute(clusterState);
clusterState = ClusterState.builder(clusterState).routingResult(result).build();
}
logger.info(
"[{}] took {}", i, TimeValue.timeValueMillis(System.currentTimeMillis() - runStart));
}
long took = System.currentTimeMillis() - start;
logger.info(
"total took {}, AVG {}",
TimeValue.timeValueMillis(took),
TimeValue.timeValueMillis(took / numberOfRuns));
}
private void warnAboutSlowTaskIfNeeded(TimeValue executionTime, String source) {
if (executionTime.getMillis() > slowTaskLoggingThreshold.getMillis()) {
logger.warn(
"cluster state update task [{}] took [{}] above the warn threshold of {}",
source,
executionTime,
slowTaskLoggingThreshold);
}
}
public ProcessService(Settings settings) {
super(settings);
this.probe = ProcessProbe.getInstance();
final TimeValue refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
processStatsCache = new ProcessStatsCache(refreshInterval, probe.processStats());
this.info = probe.processInfo();
this.info.refreshInterval = refreshInterval.millis();
logger.debug("using refresh_interval [{}]", refreshInterval);
}
private void delay() {
if (poll.millis() > 0L) {
logger.info("next run waiting for {}", poll);
try {
Thread.sleep(poll.millis());
} catch (InterruptedException e) {
logger.error("Error during waiting.", e, (Object) null);
}
}
}
public void run() throws Exception {
for (Thread t : searcherThreads) {
t.start();
}
for (Thread t : writerThreads) {
t.start();
}
barrier1.await();
Refresher refresher = new Refresher();
scheduledExecutorService.scheduleWithFixedDelay(
refresher, refreshSchedule.millis(), refreshSchedule.millis(), TimeUnit.MILLISECONDS);
Flusher flusher = new Flusher();
scheduledExecutorService.scheduleWithFixedDelay(
flusher, flushSchedule.millis(), flushSchedule.millis(), TimeUnit.MILLISECONDS);
StopWatch stopWatch = new StopWatch();
stopWatch.start();
barrier2.await();
latch.await();
stopWatch.stop();
System.out.println("Summary");
System.out.println(
" -- Readers ["
+ searcherThreads.length
+ "] with ["
+ searcherIterations
+ "] iterations");
System.out.println(
" -- Writers [" + writerThreads.length + "] with [" + writerIterations + "] iterations");
System.out.println(" -- Took: " + stopWatch.totalTime());
System.out.println(
" -- Refresh [" + refresher.id + "] took: " + refresher.stopWatch.totalTime());
System.out.println(" -- Flush [" + flusher.id + "] took: " + flusher.stopWatch.totalTime());
System.out.println(" -- Store size " + store.estimateSize());
scheduledExecutorService.shutdown();
engine.refresh(new Engine.Refresh(true));
stopWatch = new StopWatch();
stopWatch.start();
Engine.Searcher searcher = engine.searcher();
TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), idGenerator.get() + 1);
stopWatch.stop();
System.out.println(
" -- Indexed ["
+ idGenerator.get()
+ "] docs, found ["
+ topDocs.totalHits
+ "] hits, took "
+ stopWatch.totalTime());
searcher.release();
}
public static void main(String[] args) throws Exception {
NodesStressTest test =
new NodesStressTest()
.numberOfNodes(2)
.indexThreads(5)
.indexIterations(10 * 1000)
.searcherThreads(5)
.searchIterations(10 * 1000)
.sleepBeforeClose(TimeValue.timeValueMinutes(10))
.sleepAfterDone(TimeValue.timeValueMinutes(10))
.build(EMPTY_SETTINGS);
test.start();
}
@Override
public void run() {
try {
startLatch.await();
for (int j = 0; j < numberOfIds; j++) {
for (int k = 0; k < numberOfUpdatesPerId; ++k) {
updateRequestsOutstanding.acquire();
UpdateRequest ur =
client()
.prepareUpdate("test", "type1", Integer.toString(j))
.setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
.setRetryOnConflict(retryOnConflict)
.setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
.setListenerThreaded(false)
.request();
client().update(ur, new UpdateListener(j));
deleteRequestsOutstanding.acquire();
DeleteRequest dr =
client()
.prepareDelete("test", "type1", Integer.toString(j))
.setListenerThreaded(false)
.setOperationThreaded(false)
.request();
client().delete(dr, new DeleteListener(j));
}
}
} catch (Throwable e) {
logger.error("Something went wrong", e);
failures.add(e);
} finally {
try {
waitForOutstandingRequests(
TimeValue.timeValueSeconds(60),
updateRequestsOutstanding,
maxUpdateRequests,
"Update");
waitForOutstandingRequests(
TimeValue.timeValueSeconds(60),
deleteRequestsOutstanding,
maxDeleteRequests,
"Delete");
} catch (ElasticsearchTimeoutException ete) {
failures.add(ete);
}
latch.countDown();
}
}
@Test
public void testRandomDocsNodeClient() throws Exception {
final NodeClient es =
new NodeClient()
.maxActionsPerBulkRequest(1000)
.flushInterval(TimeValue.timeValueSeconds(10))
.newClient(client("1"))
.newIndex("test");
try {
for (int i = 0; i < 12345; i++) {
es.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
}
es.flush();
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
es.shutdown();
assertEquals(13, es.getState().getTotalIngest().count());
if (es.hasThrowable()) {
logger.error("error", es.getThrowable());
}
assertFalse(es.hasThrowable());
}
}
@Inject
public DiskThresholdDecider(
Settings settings,
NodeSettingsService nodeSettingsService,
ClusterInfoService infoService,
Client client) {
super(settings);
String lowWatermark = settings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, "85%");
String highWatermark = settings.get(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, "90%");
if (!validWatermarkSetting(lowWatermark, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK)) {
throw new ElasticsearchParseException("unable to parse low watermark [{}]", lowWatermark);
}
if (!validWatermarkSetting(highWatermark, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK)) {
throw new ElasticsearchParseException("unable to parse high watermark [{}]", highWatermark);
}
// Watermark is expressed in terms of used data, but we need "free" data watermark
this.freeDiskThresholdLow = 100.0 - thresholdPercentageFromWatermark(lowWatermark);
this.freeDiskThresholdHigh = 100.0 - thresholdPercentageFromWatermark(highWatermark);
this.freeBytesThresholdLow =
thresholdBytesFromWatermark(lowWatermark, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK);
this.freeBytesThresholdHigh =
thresholdBytesFromWatermark(highWatermark, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK);
this.includeRelocations =
settings.getAsBoolean(CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS, true);
this.rerouteInterval =
settings.getAsTime(
CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL, TimeValue.timeValueSeconds(60));
this.enabled = settings.getAsBoolean(CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, true);
nodeSettingsService.addListener(new ApplySettings());
infoService.addListener(new DiskListener(client));
}
public void testSerializeRequest() throws IOException {
ClusterRerouteRequest req = new ClusterRerouteRequest();
req.setRetryFailed(randomBoolean());
req.dryRun(randomBoolean());
req.explain(randomBoolean());
req.add(new AllocateEmptyPrimaryAllocationCommand("foo", 1, "bar", randomBoolean()));
req.timeout(TimeValue.timeValueMillis(randomIntBetween(0, 100)));
BytesStreamOutput out = new BytesStreamOutput();
req.writeTo(out);
BytesReference bytes = out.bytes();
NetworkModule networkModule = new NetworkModule(null, Settings.EMPTY, true);
NamedWriteableRegistry namedWriteableRegistry =
new NamedWriteableRegistry(networkModule.getNamedWriteables());
StreamInput wrap =
new NamedWriteableAwareStreamInput(bytes.streamInput(), namedWriteableRegistry);
ClusterRerouteRequest deserializedReq = new ClusterRerouteRequest();
deserializedReq.readFrom(wrap);
assertEquals(req.isRetryFailed(), deserializedReq.isRetryFailed());
assertEquals(req.dryRun(), deserializedReq.dryRun());
assertEquals(req.explain(), deserializedReq.explain());
assertEquals(req.timeout(), deserializedReq.timeout());
assertEquals(
1,
deserializedReq
.getCommands()
.commands()
.size()); // allocation commands have their own tests
assertEquals(
req.getCommands().commands().size(), deserializedReq.getCommands().commands().size());
}
@SuppressWarnings({"unchecked"})
@Inject
public SimpleRiver(
RiverName riverName, RiverSettings settings, Client client, ThreadPool threadPool) {
super(riverName, settings);
this.client = client;
if (settings.settings().containsKey("simple")) {
Map<String, Object> simpleSettings = (Map<String, Object>) settings.settings().get("simple");
simpleNumber = XContentMapValues.nodeIntegerValue(simpleSettings.get("number"), 100);
fieldName = XContentMapValues.nodeStringValue(simpleSettings.get("field"), "test");
poll =
XContentMapValues.nodeTimeValue(
simpleSettings.get("poll"), TimeValue.timeValueMinutes(60));
}
logger.info(
"creating simple stream river for [{} numbers] with field [{}]", simpleNumber, fieldName);
if (settings.settings().containsKey("index")) {
Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
typeName = XContentMapValues.nodeStringValue(indexSettings.get("type"), "simple_type");
bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100);
bulkThreshold = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_threshold"), 10);
} else {
indexName = riverName.name();
typeName = "simple_type";
bulkSize = 100;
bulkThreshold = 10;
}
}
public ScrollChunk nextChunk() {
final SearchResponse search;
// make sure to return the initial hits, see
// https://github.com/Graylog2/graylog2-server/issues/2126
if (firstResponse == null) {
search =
client
.prepareSearchScroll(scrollId)
.setScroll(TimeValue.timeValueMinutes(1))
.execute()
.actionGet();
} else {
search = firstResponse;
firstResponse = null;
}
final SearchHits hits = search.getHits();
if (hits.getHits().length == 0) {
// scroll exhausted
LOG.debug("[{}] Reached end of scroll results.", queryHash, getOriginalQuery());
return null;
}
LOG.debug(
"[{}][{}] New scroll id {}, number of hits in chunk: {}",
queryHash,
chunkId,
search.getScrollId(),
hits.getHits().length);
scrollId = search.getScrollId(); // save the id for the next request.
return new ScrollChunk(hits, fields, chunkId++);
}
