protected String getEventId(Event event) {
Map<String, Object> attrs = event.getAttributes();
if (attrs.containsKey("EventId")) {
return (String) attrs.get("EventId");
} else {
// FIXME parameterize to support other use cases/events, not just CC transactions
return (String) attrs.get(Constants.ATTR_TRANSACTION_ID);
}
}
private ArrayList<Long> getE2ELatenciesForType(Event event) {
String eventName = event.getEventName();
ArrayList<Long> e2eLatenciesForType = e2eLatenciesPerType.get(eventName);
if (e2eLatenciesForType == null) {
e2eLatenciesForType = new ArrayList<Long>();
e2eLatenciesPerType.put(eventName, e2eLatenciesForType);
}
return e2eLatenciesForType;
}
private EventLogEntry getEventLogEntry(Event event) {
String eventId = getEventId(event);
EventLogEntry entry = eventLogMap.get(eventId);
if (entry == null) {
entry = new EventLogEntry();
entry.eventId = eventId;
entry.eventName = event.getEventName();
eventLog.add(entry);
}
return entry;
}
private void updateLatenciesForRawEvents(Event event, long timestamp) {
// Only input latency is known for raw events - their detected time is set during
// deserialization in kafka spout
long detectedTime = event.getTimestamp();
long inputPhaseLatency = detectedTime - timestamp;
inLatencies.add(inputPhaseLatency);
ArrayList<Long> e2eLatenciesForType = getE2ELatenciesForType(event);
Long inTimestamp = timestamps.get(getEventId(event));
if (inTimestamp != null) {
Long e2elatency = timestamp - inTimestamp;
e2eLatenciesForType.add(e2elatency);
EventLogEntry logEntry = getEventLogEntry(event);
logEntry.e2eLatency = e2elatency;
logEntry.processingLatency = e2elatency;
logEntry.timestamp = inTimestamp;
logEntry.outLatency = timestamp - detectedTime;
}
}
protected void computeStats(InputStream eventStream) throws IOException {
BufferedReader reader = new BufferedReader(new InputStreamReader(eventStream));
boolean done = false;
e2eLatencies = new ArrayList<Long>();
processingLatencies = new ArrayList<Long>();
inLatencies = new ArrayList<Long>();
outLatencies = new ArrayList<Long>();
e2eLatenciesPerType = new HashMap<String, ArrayList<Long>>();
eventLog = new ArrayList<EventLogEntry>();
eventLogMap = new HashMap<>();
JsonEventDecoder decoder = new JsonEventDecoder();
do {
String line = reader.readLine();
if (line == null) {
done = true;
continue;
}
String[] eventEntry = line.split(":", 2);
long timestamp = Long.parseLong(eventEntry[0].trim());
if (timestamp < startTimestamp) {
continue;
}
if (isJSON(eventEntry[1])) {
// events emitted by speedd runtime are in JSON format (opposite to raw events which are
// csv)
Event event = decoder.fromBytes(eventEntry[1].trim().getBytes());
if (event.getAttributes().containsKey("timestamps")) {
// derived event - update latencies regarding the contributing events
updateLatencies(event, timestamp);
} else {
updateLatenciesForRawEvents(event, timestamp);
}
} else {
// input (csv) event
// input event - does not contain 'timestamps' - use to compute real rates
Event event = eventMetadata.getEventParser().fromBytes(eventEntry[1].trim().getBytes());
updateInEventMetrics(event, timestamp);
}
} while (!done);
Collections.sort(e2eLatencies);
Collections.sort(processingLatencies);
Collections.sort(inLatencies);
Collections.sort(outLatencies);
for (String type : e2eLatenciesPerType.keySet()) {
Collections.sort(e2eLatenciesPerType.get(type));
}
}
private void updateLatencies(Event event, long eventTimestamp) {
Object[] contributingEvents = (Object[]) event.getAttributes().get("transaction_ids");
if (contributingEvents.length == 0) {
return;
}
ArrayList<Long> e2eLatenciesForType = getE2ELatenciesForType(event);
EventLogEntry logEntry = getEventLogEntry(event);
logEntry.timestamp = eventTimestamp;
long latestContributingInEventTimestamp = 0;
for (int i = contributingEvents.length - 1; i >= 0; --i) {
String eventId = (String) contributingEvents[i];
if (timestamps.containsKey(eventId)) {
long inEventTimestamp = timestamps.get(eventId);
if (inEventTimestamp > latestContributingInEventTimestamp) {
latestContributingInEventTimestamp = inEventTimestamp;
}
}
}
if (latestContributingInEventTimestamp > 0) {
Long latency = eventTimestamp - latestContributingInEventTimestamp;
e2eLatencies.add(latency);
e2eLatenciesForType.add(latency);
Long outLatency = eventTimestamp - event.getTimestamp();
outLatencies.add(outLatency);
logEntry.e2eLatency = latency;
logEntry.outLatency = outLatency;
}
// update processing latencies
long internalTimestamp = event.getTimestamp();
Object[] contributingTimestamps = (Object[]) event.getAttributes().get("timestamps");
long latestContributingInternalTimestamp = 0;
long contributingTs = 0;
for (Object latestContributingTimestampObject : contributingTimestamps) {
if (latestContributingTimestampObject instanceof String) {
contributingTs = Long.parseLong((String) latestContributingTimestampObject);
} else {
contributingTs = (Long) latestContributingTimestampObject;
}
if (contributingTs > latestContributingInternalTimestamp) {
latestContributingInternalTimestamp = contributingTs;
}
}
Long internalLatency = internalTimestamp - latestContributingInternalTimestamp;
processingLatencies.add(internalLatency);
logEntry.processingLatency = internalLatency;
logEntry.inLatency = latestContributingInternalTimestamp - latestContributingInEventTimestamp;
}