@Test
public void testAggWithSafeWindow() throws ParseException {
SimpleCollector collector = new SimpleCollector();
aggregator_1.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:00,000"), "user1", "open"),
collector); // create window
aggregator_1.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:01,000"), "user1", "rename"),
collector);
aggregator_1.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:59,000"), "user1", "delete"),
collector);
aggregator_1.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:01,000"), "user1", "open"),
collector);
aggregator_1.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:58,000"), "user2", "open"),
collector); // inside safe-window
aggregator_1.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:06,000"), "user2", "open"),
collector); //
aggregator_1.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:06:02,000"), "user2", "open"),
collector); // outside safe-window
Assert.assertEquals(4, collector.getResult().size());
Assert.assertEquals("user2", collector.getResult().get(0).f0());
Assert.assertEquals("user1", collector.getResult().get(1).f0());
}
private Map<String, String> buildAlertContext(AlertStreamEvent event) {
Map<String, String> alertContext = new HashMap<>();
alertContext.put(PublishConstants.ALERT_EMAIL_MESSAGE, event.toString());
alertContext.put(PublishConstants.ALERT_EMAIL_POLICY, event.getPolicyId());
alertContext.put(
PublishConstants.ALERT_EMAIL_TIMESTAMP,
DateTimeUtil.millisecondsToHumanDateWithSeconds(event.getCreatedTime()));
alertContext.put(PublishConstants.ALERT_EMAIL_STREAM, event.getStreamId());
alertContext.put(PublishConstants.ALERT_EMAIL_CREATOR, event.getCreatedBy());
return alertContext;
}
@Test
public void testAggWithoutSafeWindow() throws ParseException {
SimpleCollector collector = new SimpleCollector();
// yyyy-MM-dd HH:mm:ss,SSS
aggregator_0.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:00,000"), "user1", "open"),
collector);
aggregator_0.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:01,000"), "user1", "open"),
collector);
aggregator_0.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:02,000"), "user1", "open"),
collector);
aggregator_0.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:59,000"), "user2", "open"),
collector);
aggregator_0.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:01,000"), "user1", "open"),
collector);
aggregator_0.accumulate(
event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:02,000"), "user2", "open"),
collector);
Assert.assertEquals(2, collector.getResult().size());
Assert.assertEquals("user2", collector.getResult().get(0).f0());
Assert.assertEquals("user1", collector.getResult().get(1).f0());
}
/**
* TODO: refactor time series aggregator to remove dependency of business logic entity class
*
* @param entityDefinition
* @param scan
* @param groupbyFields
* @param aggregateFuncTypes
* @param aggregatedFields
* @param intervalMin
* @return
* @throws Exception
*/
@Override
public AggregateResult aggregate(
EntityDefinition entityDefinition,
Scan scan,
List<String> groupbyFields,
List<byte[]> aggregateFuncTypes,
List<String> aggregatedFields,
long startTime,
long endTime,
long intervalMin)
throws IOException {
// LOG.info("Using coprocessor instance: "+this);
checkNotNull(entityDefinition, "entityDefinition");
String serviceName = entityDefinition.getService();
LOG.info(
this.getLogHeader()
+ " time series group aggregate on service: "
+ serviceName
+ " by: "
+ groupbyFields
+ " func: "
+ AggregateFunctionType.fromBytesList(aggregateFuncTypes)
+ " fields: "
+ aggregatedFields
+ " intervalMin: "
+ intervalMin
+ " from: "
+ DateTimeUtil.millisecondsToHumanDateWithMilliseconds(startTime)
+ " to: "
+ DateTimeUtil.millisecondsToHumanDateWithMilliseconds(endTime));
if (LOG.isDebugEnabled()) LOG.debug("SCAN: " + scan.toJSON());
long _start = System.currentTimeMillis();
final TimeSeriesAggregator aggregator =
new TimeSeriesAggregator(
groupbyFields,
AggregateFunctionType.fromBytesList(aggregateFuncTypes),
aggregatedFields,
startTime,
endTime,
intervalMin);
InternalReadReport report = this.asyncStreamRead(entityDefinition, scan, aggregator);
List<GroupbyKeyValue> keyValues = aggregator.getGroupbyKeyValues();
AggregateResult result = new AggregateResult();
result.setKeyValues(keyValues);
result.setStartTimestamp(report.getStartTimestamp());
result.setStopTimestamp(report.getStopTimestamp());
long _stop = System.currentTimeMillis();
LOG.info(
String.format(
"%s: scan = %d rows, group = %d keys, startTime = %d, endTime = %d, spend = %d ms",
this.getLogHeader(),
report.getCounter(),
keyValues.size(),
report.getStartTimestamp(),
report.getStopTimestamp(),
(_stop - _start)));
return result;
}