/**
* Build TrendContent with external services top by average response time.
*
* @param period
* @param limit
* @param timeRange
* @param transactions
* @param transactionsStatistic
* @return
*/
public static TrendContext<String> topByAvgResponseTime(
Long period,
Integer limit,
TimeRange timeRange,
Iterable<ExternalService> transactions,
Iterable<ExternalServiceStatistic> transactionsStatistic) {
TrendContext<String> trendContext = new TrendContext<>(period * 1000, timeRange);
Map<Long, ExternalService> externalServiceMap =
StreamSupport.stream(transactions.spliterator(), false)
.collect(Collectors.toMap(ExternalService::getId, (t) -> t));
Map<String, List<ExternalServiceStatistic>> transactionStatisticMap =
StreamSupport.stream(transactionsStatistic.spliterator(), false)
.collect(
Collectors.groupingBy(
statistic -> {
Long serviceId = statistic.getExternalServiceId();
ExternalService transaction = externalServiceMap.get(serviceId);
return transaction.getUrl();
}));
transactionStatisticMap
.entrySet()
.stream()
.sorted(
Comparator.comparing(
new Function<Map.Entry<String, List<ExternalServiceStatistic>>, Double>() {
@Override
public Double apply(Map.Entry<String, List<ExternalServiceStatistic>> entry) {
DoubleSummaryStatistics responseSummaryStatistics =
entry
.getValue()
.stream()
.filter(statistic -> statistic.getSumResponseTime() != null)
.mapToDouble(ExternalServiceStatistic::getSumResponseTime)
.summaryStatistics();
LongSummaryStatistics pvSummaryStatistics =
entry
.getValue()
.stream()
.filter(statistic -> statistic.getPv() != null)
.mapToLong(ExternalServiceStatistic::getPv)
.summaryStatistics();
return calculateRate(
responseSummaryStatistics.getSum(), pvSummaryStatistics.getSum());
}
}))
.limit(limit)
.forEach(entry -> trendContext.addStatistics(entry.getKey(), timeRange, entry.getValue()));
return trendContext;
}
private static MatchTableImpl createEmptyTable(
VariantGraphRanking ranking, VariantGraph graph, Iterable<Token> witness) {
// -2 === ignore the start and the end vertex
return new MatchTableImpl(
StreamSupport.stream(witness.spliterator(), false).toArray(Token[]::new),
IntStream.range(0, Math.max(0, ranking.apply(graph.getEnd()) - 1)).toArray());
}
private Map<HivePartitionName, Optional<Partition>> loadPartitionsByNames(
Iterable<? extends HivePartitionName> partitionNames) throws Exception {
requireNonNull(partitionNames, "partitionNames is null");
checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty");
HivePartitionName firstPartition = Iterables.get(partitionNames, 0);
HiveTableName hiveTableName = firstPartition.getHiveTableName();
String databaseName = hiveTableName.getDatabaseName();
String tableName = hiveTableName.getTableName();
List<String> partitionsToFetch = new ArrayList<>();
for (HivePartitionName partitionName : partitionNames) {
checkArgument(
partitionName.getHiveTableName().equals(hiveTableName),
"Expected table name %s but got %s",
hiveTableName,
partitionName.getHiveTableName());
partitionsToFetch.add(partitionName.getPartitionName());
}
List<String> partitionColumnNames =
ImmutableList.copyOf(
Warehouse.makeSpecFromName(firstPartition.getPartitionName()).keySet());
try {
return retry()
.stopOn(NoSuchObjectException.class)
.stopOnIllegalExceptions()
.run(
"getPartitionsByNames",
stats
.getGetPartitionsByNames()
.wrap(
() -> {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) {
ImmutableMap.Builder<HivePartitionName, Optional<Partition>> partitions =
ImmutableMap.builder();
for (Partition partition :
client.getPartitionsByNames(
databaseName, tableName, partitionsToFetch)) {
String partitionId =
FileUtils.makePartName(
partitionColumnNames, partition.getValues(), null);
partitions.put(
HivePartitionName.partition(databaseName, tableName, partitionId),
Optional.of(partition));
}
return partitions.build();
}
}));
} catch (NoSuchObjectException e) {
// assume none of the partitions in the batch are available
return stream(partitionNames.spliterator(), false)
.collect(toMap(identity(), (name) -> Optional.empty()));
} catch (TException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
public static <T> String listToString(Iterable<T> elements, String sep) {
if (elements == null) throw new IllegalArgumentException("elements cannot be null.");
Stream<T> stream = StreamSupport.stream(elements.spliterator(), false);
return stream.map(a -> a.toString()).collect(Collectors.joining(sep));
}
/**
* Gets the quantity of the Mesos resource specified by {@code type}.
*
* @param resources Mesos resources.
* @param type Type of resource to quantify.
* @return Aggregate Mesos resource value.
*/
public static Double quantityOfMesosResource(Iterable<Resource> resources, ResourceType type) {
return StreamSupport.stream(resources.spliterator(), false)
.filter(r -> SUPPORTED_RESOURCE.apply(r))
.filter(r -> fromResource(r).equals(type))
.map(QUANTIFY_MESOS_RESOURCE)
.reduce(REDUCE_VALUES)
.orElse(0.0);
}
@RequestMapping("/gameStatus")
public ResponseEntity<GameStatus> broadcastGameStatus() {
GameStatus gameStatus = new GameStatus();
Iterable<GameStep> gameSteps = gameStepRepository.findAll();
List<GameStep> collect = StreamSupport.stream(gameSteps.spliterator(), false).collect(toList());
Map<String, List<GameStep>> gameStepByUser =
StreamSupport.stream(gameSteps.spliterator(), false)
.collect(Collectors.groupingBy(GameStep::getUserPseudo));
for (Map.Entry<String, List<GameStep>> gameStepByUserEntry : gameStepByUser.entrySet()) {
UserGameStatus userGameStatus = new UserGameStatus();
userGameStatus.setPseudo(gameStepByUserEntry.getKey());
Map<Step, List<GameStep>> gameStepByStep =
gameStepByUserEntry.getValue().stream().collect(Collectors.groupingBy(GameStep::getStep));
for (Map.Entry<Step, List<GameStep>> gameStepByStepEntry : gameStepByStep.entrySet()) {
gameStepByStepEntry
.getValue()
.stream()
.min(Comparator.comparing(GameStep::getInstant))
.ifPresent(
gameStep -> {
userGameStatus
.getStepInfos()
.add(new StepInfo(gameStep.getStep(), gameStep.getInstant()));
});
}
gameStatus.getUserGameStatus().add(userGameStatus);
}
webSocketTemplate.convertAndSend("/topic/gameStatus", gameStatus);
return new ResponseEntity<GameStatus>(gameStatus, HttpStatus.OK);
}
/**
* @param timeRange
* @param apps
* @param period in Second
* @return
*/
public Iterable<ApplicationStatistic> selectApplicationStatistics(
TimeRange timeRange, Iterable<Application> apps, Long period) {
final long periodInMillis = period * 1000;
List<Long> appIds =
StreamSupport.stream(apps.spliterator(), false)
.map(Application::getId)
.collect(Collectors.toList());
QApplicationStatistic appStatistic = QApplicationStatistic.applicationStatistic;
BooleanExpression timestampCondition =
appStatistic.timestamp.between(
toMillisSecond(timeRange.getFrom()), toMillisSecond(timeRange.getTo()));
BooleanExpression appIdCondition = appStatistic.appId.in(appIds);
BooleanExpression periodCondition = appStatistic.period.eq(periodInMillis);
BooleanExpression whereCondition = timestampCondition.and(appIdCondition).and(periodCondition);
return applicationStatisticRepository.findAll(whereCondition);
}
/**
* Ordinary constructor.
*
* @param resource the resource
* @param idTranslator the id translator
* @param blobs the blobs
* @param digest the digest uri
* @param size the size
*/
public FixityRdfContext(
final FedoraResource resource,
final IdentifierConverter<Resource, FedoraResource> idTranslator,
final Iterable<FixityResult> blobs,
final URI digest,
final long size) {
super(resource, idTranslator);
concat(
StreamSupport.stream(blobs.spliterator(), false)
.flatMap(
uncheck(
blob -> {
final org.apache.jena.graph.Node resultSubject =
createURI(subject().getURI() + "#fixity/" + now().toEpochMilli());
final List<Triple> b = new ArrayList<>();
b.add(create(subject(), HAS_FIXITY_RESULT.asNode(), resultSubject));
b.add(create(resultSubject, type.asNode(), FIXITY_TYPE.asNode()));
b.add(
create(resultSubject, type.asNode(), EVENT_OUTCOME_INFORMATION.asNode()));
blob.getStatus(size, digest)
.stream()
.map(state -> createLiteral(state.toString()))
.map(state -> create(resultSubject, HAS_FIXITY_STATE.asNode(), state))
.forEach(b::add);
b.add(
create(
resultSubject,
HAS_MESSAGE_DIGEST.asNode(),
createURI(blob.getComputedChecksum().toString())));
b.add(
create(
resultSubject,
HAS_SIZE.asNode(),
createTypedLiteral(blob.getComputedSize()).asNode()));
return b.stream();
})));
}
private static <T> ResourceBag bagFromResources(
Iterable<T> resources,
Function<T, ResourceType> typeMapper,
Function<T, Double> valueMapper) {
return new ResourceBag(
StreamSupport.stream(resources.spliterator(), false)
.collect(Collectors.groupingBy(typeMapper))
.entrySet()
.stream()
.collect(
Collectors.toMap(
Map.Entry::getKey,
group ->
group
.getValue()
.stream()
.map(valueMapper)
.reduce(REDUCE_VALUES)
.orElse(0.0))));
}
public static JavaPairRDD<GATKRead, ReferenceBases> addBases(
final ReferenceDataflowSource referenceDataflowSource, final JavaRDD<GATKRead> reads) {
SerializableFunction<GATKRead, SimpleInterval> windowFunction =
referenceDataflowSource.getReferenceWindowFunction();
JavaPairRDD<ReferenceShard, GATKRead> shardRead =
reads.mapToPair(
gatkRead -> {
ReferenceShard shard =
ReferenceShard.getShardNumberFromInterval(windowFunction.apply(gatkRead));
return new Tuple2<>(shard, gatkRead);
});
JavaPairRDD<ReferenceShard, Iterable<GATKRead>> shardiRead = shardRead.groupByKey();
return shardiRead.flatMapToPair(
in -> {
List<Tuple2<GATKRead, ReferenceBases>> out = Lists.newArrayList();
Iterable<GATKRead> iReads = in._2();
// Apply the reference window function to each read to produce a set of intervals
// representing
// the desired reference bases for each read.
final List<SimpleInterval> readWindows =
StreamSupport.stream(iReads.spliterator(), false)
.map(read -> windowFunction.apply(read))
.collect(Collectors.toList());
SimpleInterval interval = SimpleInterval.getSpanningInterval(readWindows);
ReferenceBases bases = referenceDataflowSource.getReferenceBases(null, interval);
for (GATKRead r : iReads) {
final ReferenceBases subset = bases.getSubset(windowFunction.apply(r));
out.add(new Tuple2<>(r, subset));
}
return out;
});
}
@Override
protected void compute() {
StreamSupport.stream(elements.spliterator(), true).forEach(this.elementProcess);
}
public static <T> Stream<T> asStream(Iterator<T> sourceIterator, boolean parallel) {
Iterable<T> iterable = () -> sourceIterator;
return StreamSupport.stream(iterable.spliterator(), parallel);
}
private void validateQueryRule(QueryRule queryRule, EntityType entityType) {
QueryRule.Operator operator = queryRule.getOperator();
switch (operator) {
case AND:
case NOT:
case OR:
break;
case EQUALS:
case FUZZY_MATCH:
case FUZZY_MATCH_NGRAM:
case GREATER:
case GREATER_EQUAL:
case LESS:
case LESS_EQUAL:
case LIKE:
{
Attribute attr = getQueryRuleAttribute(queryRule, entityType);
Object value = toQueryRuleValue(queryRule.getValue(), attr);
queryRule.setValue(value);
break;
}
case SEARCH:
{
Object queryRuleValue = queryRule.getValue();
if (queryRuleValue != null && !(queryRuleValue instanceof String)) {
// fix value type
queryRule.setValue(queryRuleValue.toString());
}
break;
}
case IN:
case RANGE:
{
Attribute attr = getQueryRuleAttribute(queryRule, entityType);
Object queryRuleValue = queryRule.getValue();
if (queryRuleValue != null) {
if (!(queryRuleValue instanceof Iterable<?>)) {
throw new MolgenisValidationException(
new ConstraintViolation(
format(
"Query rule with operator [%s] value is of type [%s] instead of [Iterable]",
operator, queryRuleValue.getClass().getSimpleName())));
}
// fix value types
Iterable<?> queryRuleValues = (Iterable<?>) queryRuleValue;
List<Object> values =
stream(queryRuleValues.spliterator(), false)
.map(value -> toQueryRuleValue(value, attr))
.collect(toList());
queryRule.setValue(values);
}
break;
}
case DIS_MAX:
case NESTED:
case SHOULD:
queryRule
.getNestedRules()
.forEach(nestedQueryRule -> validateQueryRule(nestedQueryRule, entityType));
break;
default:
throw new RuntimeException(format("Unknown query operator [%s]", operator.toString()));
}
}
public static <T> Stream<T> asStream(final Iterator<T> source) {
final Iterable<T> iterable = () -> source;
return StreamSupport.stream(iterable.spliterator(), false);
}
/**
* Get the top n external Services of the given services.
*
* @param statistics
* @param externalServices
* @param timeRange
* @param limit
* @return
*/
public static List<ExternalServiceVo> topByAvgResponseTime(
Iterable<ExternalServiceStatistic> statistics,
Iterable<ExternalService> externalServices,
TimeRange timeRange,
Integer limit) {
Map<Long, ExternalService> rpcTransactionMap = new HashMap<>();
StreamSupport.stream(externalServices.spliterator(), false)
.forEach(transaction -> rpcTransactionMap.put(transaction.getId(), transaction));
Map<TransactionGroup, List<ExternalServiceStatistic>> groups =
StreamSupport.stream(statistics.spliterator(), false)
.collect(
Collectors.groupingBy(
new Function<ExternalServiceStatistic, TransactionGroup>() {
@Override
public TransactionGroup apply(ExternalServiceStatistic statistic) {
Long transactionId = statistic.getExternalServiceId();
ExternalService webTransaction = rpcTransactionMap.get(transactionId);
TransactionGroup group = new TransactionGroup();
group.setAppId(webTransaction.getAppId());
group.setDisplayName(webTransaction.getUrl());
return group;
}
}));
List<ExternalServiceVo> result = new ArrayList<>();
groups.forEach(
(group, webTransactionStatistics) -> {
DoubleSummaryStatistics responseSummaryStatistics =
webTransactionStatistics
.stream()
.filter(statistic -> statistic.getSumResponseTime() != null)
.mapToDouble(ExternalServiceStatistic::getSumResponseTime)
.summaryStatistics();
DoubleSummaryStatistics maxSummaryStatistics =
webTransactionStatistics
.stream()
.filter(statistic -> statistic.getMaxResponseTime() != null)
.mapToDouble(ExternalServiceStatistic::getMaxResponseTime)
.summaryStatistics();
DoubleSummaryStatistics minSummaryStatistics =
webTransactionStatistics
.stream()
.filter(statistic -> statistic.getMinResponseTime() != null)
.mapToDouble(ExternalServiceStatistic::getMinResponseTime)
.summaryStatistics();
LongSummaryStatistics pvSummaryStatistics =
webTransactionStatistics
.stream()
.filter(statistic -> statistic.getPv() != null)
.mapToLong(ExternalServiceStatistic::getPv)
.summaryStatistics();
ExternalServiceVo transaction = new ExternalServiceVo();
transaction.setAppId(group.getAppId());
transaction.setDestination(group.getDisplayName());
transaction.setPv(pvSummaryStatistics.getSum());
transaction.setCpm(
format(
calculateRate(
pvSummaryStatistics.getSum(), timeRange.getDuration(ChronoUnit.MINUTES))));
transaction.setMaxResponseTime(format(maxSummaryStatistics.getMax()));
transaction.setMinResponseTime(format(minSummaryStatistics.getMin()));
transaction.setResponseTime(
format(
calculateRate(responseSummaryStatistics.getSum(), pvSummaryStatistics.getSum())));
result.add(transaction);
});
return result
.stream()
.sorted(Comparator.comparing(ExternalServiceVo::getResponseTime))
.limit(limit)
.collect(Collectors.toList());
}
private static String rulesToString(Iterable<Rule> rules) {
return StreamSupport.stream(rules.spliterator(), false)
.map(r -> "\n// " + r.toString() + "\n.addRule(" + r + ")")
.collect(Collectors.joining());
}
public static <T> List<T> toList(final Iterable<T> iterable) {
return StreamSupport.stream(iterable.spliterator(), false).collect(Collectors.toList());
}
/**
* Gets the quantity of resources. Caller to ensure all resources are of the same type.
*
* @param resources Resources to sum up.
* @return Aggregate resource value.
*/
public static Double quantityOf(Iterable<IResource> resources) {
return StreamSupport.stream(resources.spliterator(), false)
.map(QUANTIFY_RESOURCE)
.reduce(REDUCE_VALUES)
.orElse(0.0);
}
/**
* Gets the quantity of resource specified by {@code type}.
*
* @param resources Resources.
* @param type Type of resource to quantify.
* @return Aggregate resource value.
*/
public static Double quantityOf(Iterable<IResource> resources, ResourceType type) {
return quantityOf(
StreamSupport.stream(resources.spliterator(), false)
.filter(r -> fromResource(r).equals(type))
.collect(Collectors.toList()));
}
