private void runVerification(int numberOfRequests, int jobsPerRequest) {
// verify control info
CountInfo countInfo = new CountInfo(numberOfRequests, jobsPerRequest);
ConcurrentHashMap<Long, ConcurrentLinkedDeque<TraceContext.Info>> testInfoMap =
TraceContext.getTestInfoMap();
// LOGGER.info("Thread {}: {}", key, testInfoMap.get(key).toString().replace(",", "\n"));
for (ConcurrentLinkedDeque<TraceContext.Info> queue : testInfoMap.values()) {
mixedInvariants(queue, countInfo);
}
countInfo.verify();
// verify trace info
for (long rqId = 0; rqId < numberOfRequests; rqId++) {
ConcurrentLinkedDeque<Trace> traceQueue = TraceContext.getAllTraceInfoMap().get(rqId);
assertEquals(traceQueue.size(), jobsPerRequest + 1);
Set<Integer> jobIdSet = new HashSet<Integer>();
for (Trace trace : traceQueue) {
// one trace is used for request handler, it has no info recorded
if (trace.getValue().size() > 0) {
Object obj = trace.getValue().get(0); // we have recorded one entry per job in tests
String[] tmp = ((String) obj).split(SEP);
long reqId = Long.parseLong(tmp[0].trim());
assertEquals(rqId, reqId);
int jobId = Integer.parseInt(tmp[1].trim());
jobIdSet.add(jobId);
}
}
assertEquals(jobIdSet.size(), jobsPerRequest);
// show trace
LOGGER.info("Trace Tree: {}", TraceContext.getTraceInfoOfRequestId(rqId));
}
}
private void singleRequestCall(
ExecutorService jobService, final InstanceRequest request, int jobsPerRequest)
throws Exception {
TraceContext.register(request);
Future[] tasks = new Future[jobsPerRequest];
// fan out multiple threads for this request
for (int i = 0; i < jobsPerRequest; i++) {
final int taskId = i;
tasks[i] =
jobService.submit(
new TraceRunnable() {
@Override
public void runJob() {
String tid = Thread.currentThread().getId() + "";
TraceContext.log(tid, request.getRequestId() + SEP + taskId);
}
});
}
// wait for all threads to finish the job
for (int i = 0; i < jobsPerRequest; i++) {
// block waiting
tasks[i].get();
}
TraceContext.unregister(request);
}
@Override
public void before(Object target, Object[] args) {
if (isDebug) {
logger.beforeInterceptor(target, args);
}
Trace trace = traceContext.currentRawTraceObject();
if (trace == null) {
// create user include trace for standalone entry point.
trace = createUserIncludeTrace();
if (trace == null) {
return;
}
}
// check user include trace.
if (!isUserIncludeTrace(trace)) {
return;
}
// entry scope(default & disable trace).
entryUserIncludeTraceScope(trace);
// check sampled.
if (!trace.canSampled()) {
// skip
return;
}
trace.traceBlockBegin();
}
public UserIncludeMethodInterceptor(
TraceContext traceContext, MethodDescriptor methodDescriptor) {
this.traceContext = traceContext;
this.descriptor = methodDescriptor;
traceContext.cacheApi(USER_INCLUDE_METHOD_DESCRIPTOR);
}
private void deleteUserIncludeTrace(final Trace trace) {
if (isDebug) {
logger.debug("Delete user include trace={}, sampled={}", trace, trace.canSampled());
}
traceContext.removeTraceObject();
trace.close();
}
@Test
public void testSingleRequest() throws Exception {
TraceContext.reset();
ExecutorService jobService =
Executors.newFixedThreadPool(NUMBER_OF_THREADS, new NamedThreadFactory("jobService"));
InstanceRequest request = new InstanceRequest(0L, null);
request.setEnableTrace(true);
singleRequestCall(jobService, request, JOBS_PER_REQUEST);
runVerification(1, JOBS_PER_REQUEST);
}
/**
* If using the shared {@link ExecutorService} created by {@link
* Executors#newFixedThreadPool(int)} for handling requests and jobs, avoiding potential deadlock
* by making sure NUMBER_OF_REQUESTS < nThreads. (Using non-shared {@link ExecutorService} is
* no issue.)
*
* @throws Exception
*/
@Test
public void testMultipleRequests() throws Exception {
TraceContext.reset();
// shared Service
ExecutorService sharedService =
Executors.newFixedThreadPool(NUMBER_OF_THREADS, new NamedThreadFactory("sharedService"));
runVariousConditionTests(sharedService, sharedService);
// non-shared Service
ExecutorService requestService =
Executors.newFixedThreadPool(NUMBER_OF_THREADS, new NamedThreadFactory("requestService"));
ExecutorService jobService =
Executors.newFixedThreadPool(NUMBER_OF_THREADS, new NamedThreadFactory("jobService"));
runVariousConditionTests(requestService, jobService);
}
@Override
public void after(Object target, Object[] args, Object result, Throwable throwable) {
if (isDebug) {
logger.afterInterceptor(target, args);
}
final Trace trace = traceContext.currentRawTraceObject();
if (trace == null) {
return;
}
// check user include trace.
if (!isUserIncludeTrace(trace)) {
return;
}
// leave scope(default & disable trace).
if (!leaveUserIncludeTraceScope(trace)) {
logger.warn(
"Failed to leave scope of user include trace. trace={}, sampled={}",
trace,
trace.canSampled());
// delete unstable trace.
deleteUserIncludeTrace(trace);
return;
}
// check sampled.
if (!trace.canSampled()) {
if (isUserIncludeTraceDestination(trace)) {
deleteUserIncludeTrace(trace);
}
return;
}
try {
final SpanEventRecorder recorder = trace.currentSpanEventRecorder();
recorder.recordApi(descriptor);
recorder.recordServiceType(ServiceType.INTERNAL_METHOD);
recorder.recordException(throwable);
} finally {
trace.traceBlockEnd();
if (isUserIncludeTraceDestination(trace)) {
deleteUserIncludeTrace(trace);
}
}
}
private void runVariousConditionTests(ExecutorService requestService, ExecutorService jobService)
throws Exception {
int reqStep = NUMBER_OF_REQUESTS / 5;
int jobStep = JOBS_PER_REQUEST / 5;
boolean shared = false;
if (requestService == jobService) {
shared = true;
}
for (int numberOfRequests = reqStep;
numberOfRequests <= NUMBER_OF_REQUESTS;
numberOfRequests += reqStep) {
for (int jobsPerRequest = jobStep;
jobsPerRequest < JOBS_PER_REQUEST;
jobsPerRequest += jobStep) {
if (shared) {
LOGGER.info(
"Shared exec service for #request: "
+ numberOfRequests
+ " #jobsPerRequest: "
+ jobsPerRequest);
} else {
LOGGER.info(
"Non-shared exec service for #request: "
+ numberOfRequests
+ " #jobsPerRequest: "
+ jobsPerRequest);
}
if (shared && (numberOfRequests >= NUMBER_OF_THREADS)) {
LOGGER.info(
"Ignore tests for #request: " + numberOfRequests + ", since it will deadlock.");
} else {
TraceContext.reset();
multipleRequestsCall(requestService, jobService, numberOfRequests, jobsPerRequest);
runVerification(numberOfRequests, jobsPerRequest);
}
}
}
}
private Trace createUserIncludeTrace() {
final Trace trace = traceContext.newTraceObject();
if (isDebug) {
logger.debug("New user include trace {} and sampled {}", trace, trace.canSampled());
}
// add user scope.
TraceScope oldScope = trace.addScope(SCOPE_NAME);
if (oldScope != null) {
// delete corrupted trace.
logger.warn("Duplicated user include trace scope={}.", oldScope.getName());
deleteUserIncludeTrace(trace);
return null;
}
if (trace.canSampled()) {
// record root span.
final SpanRecorder recorder = trace.getSpanRecorder();
recorder.recordServiceType(ServiceType.STAND_ALONE);
recorder.recordApi(USER_INCLUDE_METHOD_DESCRIPTOR);
}
return trace;
}
@Override
public void before(Object target, Object[] args) {
if (isDebug) {
logger.beforeInterceptor(target, args);
}
final Trace trace = traceContext.currentRawTraceObject();
if (trace == null) {
return;
}
if (!validate(target)) {
return;
}
if (!trace.canSampled()) {
return;
}
SpanEventRecorder recorder = trace.traceBlockBegin();
try {
final TraceId nextId = trace.getTraceId().getNextTraceId();
recorder.recordNextSpanId(nextId.getSpanId());
recorder.recordServiceType(OK_HTTP_CLIENT);
InterceptorGroupInvocation invocation = interceptorGroup.getCurrentInvocation();
if (invocation != null) {
invocation.getOrCreateAttachment(
new AttachmentFactory() {
@Override
public Object createAttachment() {
return nextId;
}
});
}
} catch (Throwable t) {
logger.warn("Failed to BEFORE process. {}", t.getMessage(), t);
}
}
@Override
public void after(Object target, Object[] args, Object result, Throwable throwable) {
if (isDebug) {
logger.afterInterceptor(target, args);
}
final Trace trace = traceContext.currentTraceObject();
if (trace == null) {
return;
}
if (!validate(target)) {
return;
}
try {
SpanEventRecorder recorder = trace.currentSpanEventRecorder();
recorder.recordApi(methodDescriptor);
recorder.recordException(throwable);
Request request = ((UserRequestGetter) target)._$PINPOINT$_getUserRequest();
if (request != null) {
recorder.recordAttribute(AnnotationKey.HTTP_URL, request.httpUrl().toString());
recorder.recordDestinationId(request.httpUrl().host() + ":" + request.httpUrl().port());
recordRequest(trace, request, throwable);
}
// clear attachment.
InterceptorGroupInvocation invocation = interceptorGroup.getCurrentInvocation();
if (invocation != null && invocation.getAttachment() != null) {
invocation.removeAttachment();
}
} finally {
trace.traceBlockEnd();
}
}
