protected String getEventId(Event event) { Map<String, Object> attrs = event.getAttributes(); if (attrs.containsKey("EventId")) { return (String) attrs.get("EventId"); } else { // FIXME parameterize to support other use cases/events, not just CC transactions return (String) attrs.get(Constants.ATTR_TRANSACTION_ID); } }
private ArrayList<Long> getE2ELatenciesForType(Event event) { String eventName = event.getEventName(); ArrayList<Long> e2eLatenciesForType = e2eLatenciesPerType.get(eventName); if (e2eLatenciesForType == null) { e2eLatenciesForType = new ArrayList<Long>(); e2eLatenciesPerType.put(eventName, e2eLatenciesForType); } return e2eLatenciesForType; }
private EventLogEntry getEventLogEntry(Event event) { String eventId = getEventId(event); EventLogEntry entry = eventLogMap.get(eventId); if (entry == null) { entry = new EventLogEntry(); entry.eventId = eventId; entry.eventName = event.getEventName(); eventLog.add(entry); } return entry; }
private void updateLatenciesForRawEvents(Event event, long timestamp) { // Only input latency is known for raw events - their detected time is set during // deserialization in kafka spout long detectedTime = event.getTimestamp(); long inputPhaseLatency = detectedTime - timestamp; inLatencies.add(inputPhaseLatency); ArrayList<Long> e2eLatenciesForType = getE2ELatenciesForType(event); Long inTimestamp = timestamps.get(getEventId(event)); if (inTimestamp != null) { Long e2elatency = timestamp - inTimestamp; e2eLatenciesForType.add(e2elatency); EventLogEntry logEntry = getEventLogEntry(event); logEntry.e2eLatency = e2elatency; logEntry.processingLatency = e2elatency; logEntry.timestamp = inTimestamp; logEntry.outLatency = timestamp - detectedTime; } }
protected void computeStats(InputStream eventStream) throws IOException { BufferedReader reader = new BufferedReader(new InputStreamReader(eventStream)); boolean done = false; e2eLatencies = new ArrayList<Long>(); processingLatencies = new ArrayList<Long>(); inLatencies = new ArrayList<Long>(); outLatencies = new ArrayList<Long>(); e2eLatenciesPerType = new HashMap<String, ArrayList<Long>>(); eventLog = new ArrayList<EventLogEntry>(); eventLogMap = new HashMap<>(); JsonEventDecoder decoder = new JsonEventDecoder(); do { String line = reader.readLine(); if (line == null) { done = true; continue; } String[] eventEntry = line.split(":", 2); long timestamp = Long.parseLong(eventEntry[0].trim()); if (timestamp < startTimestamp) { continue; } if (isJSON(eventEntry[1])) { // events emitted by speedd runtime are in JSON format (opposite to raw events which are // csv) Event event = decoder.fromBytes(eventEntry[1].trim().getBytes()); if (event.getAttributes().containsKey("timestamps")) { // derived event - update latencies regarding the contributing events updateLatencies(event, timestamp); } else { updateLatenciesForRawEvents(event, timestamp); } } else { // input (csv) event // input event - does not contain 'timestamps' - use to compute real rates Event event = eventMetadata.getEventParser().fromBytes(eventEntry[1].trim().getBytes()); updateInEventMetrics(event, timestamp); } } while (!done); Collections.sort(e2eLatencies); Collections.sort(processingLatencies); Collections.sort(inLatencies); Collections.sort(outLatencies); for (String type : e2eLatenciesPerType.keySet()) { Collections.sort(e2eLatenciesPerType.get(type)); } }
private void updateLatencies(Event event, long eventTimestamp) { Object[] contributingEvents = (Object[]) event.getAttributes().get("transaction_ids"); if (contributingEvents.length == 0) { return; } ArrayList<Long> e2eLatenciesForType = getE2ELatenciesForType(event); EventLogEntry logEntry = getEventLogEntry(event); logEntry.timestamp = eventTimestamp; long latestContributingInEventTimestamp = 0; for (int i = contributingEvents.length - 1; i >= 0; --i) { String eventId = (String) contributingEvents[i]; if (timestamps.containsKey(eventId)) { long inEventTimestamp = timestamps.get(eventId); if (inEventTimestamp > latestContributingInEventTimestamp) { latestContributingInEventTimestamp = inEventTimestamp; } } } if (latestContributingInEventTimestamp > 0) { Long latency = eventTimestamp - latestContributingInEventTimestamp; e2eLatencies.add(latency); e2eLatenciesForType.add(latency); Long outLatency = eventTimestamp - event.getTimestamp(); outLatencies.add(outLatency); logEntry.e2eLatency = latency; logEntry.outLatency = outLatency; } // update processing latencies long internalTimestamp = event.getTimestamp(); Object[] contributingTimestamps = (Object[]) event.getAttributes().get("timestamps"); long latestContributingInternalTimestamp = 0; long contributingTs = 0; for (Object latestContributingTimestampObject : contributingTimestamps) { if (latestContributingTimestampObject instanceof String) { contributingTs = Long.parseLong((String) latestContributingTimestampObject); } else { contributingTs = (Long) latestContributingTimestampObject; } if (contributingTs > latestContributingInternalTimestamp) { latestContributingInternalTimestamp = contributingTs; } } Long internalLatency = internalTimestamp - latestContributingInternalTimestamp; processingLatencies.add(internalLatency); logEntry.processingLatency = internalLatency; logEntry.inLatency = latestContributingInternalTimestamp - latestContributingInEventTimestamp; }