@Test
public void testThreadedUpdatesToChildBreaker() throws Exception {
final int NUM_THREADS = scaledRandomIntBetween(3, 15);
final int BYTES_PER_THREAD = scaledRandomIntBetween(500, 4500);
final Thread[] threads = new Thread[NUM_THREADS];
final AtomicBoolean tripped = new AtomicBoolean(false);
final AtomicReference<Throwable> lastException = new AtomicReference<>(null);
final AtomicReference<ChildMemoryCircuitBreaker> breakerRef = new AtomicReference<>(null);
final CircuitBreakerService service =
new HierarchyCircuitBreakerService(
Settings.EMPTY, new NodeSettingsService(Settings.EMPTY)) {
@Override
public CircuitBreaker getBreaker(String name) {
return breakerRef.get();
}
@Override
public void checkParentLimit(String label) throws CircuitBreakingException {
// never trip
}
};
final BreakerSettings settings =
new BreakerSettings(CircuitBreaker.REQUEST, (BYTES_PER_THREAD * NUM_THREADS) - 1, 1.0);
final ChildMemoryCircuitBreaker breaker =
new ChildMemoryCircuitBreaker(
settings, logger, (HierarchyCircuitBreakerService) service, CircuitBreaker.REQUEST);
breakerRef.set(breaker);
for (int i = 0; i < NUM_THREADS; i++) {
threads[i] =
new Thread(
new Runnable() {
@Override
public void run() {
for (int j = 0; j < BYTES_PER_THREAD; j++) {
try {
breaker.addEstimateBytesAndMaybeBreak(1L, "test");
} catch (CircuitBreakingException e) {
if (tripped.get()) {
assertThat("tripped too many times", true, equalTo(false));
} else {
assertThat(tripped.compareAndSet(false, true), equalTo(true));
}
} catch (Throwable e2) {
lastException.set(e2);
}
}
}
});
threads[i].start();
}
for (Thread t : threads) {
t.join();
}
assertThat("no other exceptions were thrown", lastException.get(), equalTo(null));
assertThat("breaker was tripped", tripped.get(), equalTo(true));
assertThat(
"breaker was tripped at least once", breaker.getTrippedCount(), greaterThanOrEqualTo(1L));
}
@Test
public void testThreadedUpdatesToChildBreakerWithParentLimit() throws Exception {
final int NUM_THREADS = scaledRandomIntBetween(3, 15);
final int BYTES_PER_THREAD = scaledRandomIntBetween(500, 4500);
final int parentLimit = (BYTES_PER_THREAD * NUM_THREADS) - 2;
final int childLimit = parentLimit + 10;
final Thread[] threads = new Thread[NUM_THREADS];
final AtomicInteger tripped = new AtomicInteger(0);
final AtomicReference<Throwable> lastException = new AtomicReference<>(null);
final AtomicInteger parentTripped = new AtomicInteger(0);
final AtomicReference<ChildMemoryCircuitBreaker> breakerRef = new AtomicReference<>(null);
final CircuitBreakerService service =
new HierarchyCircuitBreakerService(
Settings.EMPTY, new NodeSettingsService(Settings.EMPTY)) {
@Override
public CircuitBreaker getBreaker(String name) {
return breakerRef.get();
}
@Override
public void checkParentLimit(String label) throws CircuitBreakingException {
// Parent will trip right before regular breaker would trip
if (getBreaker(CircuitBreaker.REQUEST).getUsed() > parentLimit) {
parentTripped.incrementAndGet();
logger.info("--> parent tripped");
throw new CircuitBreakingException("parent tripped");
}
}
};
final BreakerSettings settings = new BreakerSettings(CircuitBreaker.REQUEST, childLimit, 1.0);
final ChildMemoryCircuitBreaker breaker =
new ChildMemoryCircuitBreaker(
settings, logger, (HierarchyCircuitBreakerService) service, CircuitBreaker.REQUEST);
breakerRef.set(breaker);
for (int i = 0; i < NUM_THREADS; i++) {
threads[i] =
new Thread(
new Runnable() {
@Override
public void run() {
for (int j = 0; j < BYTES_PER_THREAD; j++) {
try {
breaker.addEstimateBytesAndMaybeBreak(1L, "test");
} catch (CircuitBreakingException e) {
tripped.incrementAndGet();
} catch (Throwable e2) {
lastException.set(e2);
}
}
}
});
}
logger.info(
"--> NUM_THREADS: [{}], BYTES_PER_THREAD: [{}], TOTAL_BYTES: [{}], PARENT_LIMIT: [{}], CHILD_LIMIT: [{}]",
NUM_THREADS,
BYTES_PER_THREAD,
(BYTES_PER_THREAD * NUM_THREADS),
parentLimit,
childLimit);
logger.info("--> starting threads...");
for (Thread t : threads) {
t.start();
}
for (Thread t : threads) {
t.join();
}
logger.info("--> child breaker: used: {}, limit: {}", breaker.getUsed(), breaker.getLimit());
logger.info(
"--> parent tripped: {}, total trip count: {} (expecting 1-2 for each)",
parentTripped.get(),
tripped.get());
assertThat("no other exceptions were thrown", lastException.get(), equalTo(null));
assertThat(
"breaker should be reset back to the parent limit after parent breaker trips",
breaker.getUsed(),
greaterThanOrEqualTo((long) parentLimit - NUM_THREADS));
assertThat(
"parent breaker was tripped at least once", parentTripped.get(), greaterThanOrEqualTo(1));
assertThat("total breaker was tripped at least once", tripped.get(), greaterThanOrEqualTo(1));
}
