/**
* 获取订单抓取开始时间
*
* @param fetchTime
* @return
*/
public Date getStartDate(Date startDate, Date fetchTime) {
if (startDate != null) {
return startDate;
}
Date start;
if (fetchTime == null) { // 没有抓取记录
// 当前时间往前推N秒
start =
DateUtils.addSeconds(
EJSDateUtils.getCurrentDate(), -ConstantJingDong.JD_FETCH_ORDER_TIME_INTERVAL);
} else {
// 将抓取订单时间往前推N秒
start = DateUtils.addSeconds(fetchTime, ConstantJingDong.JD_FETCH_ORDER_TIME_DELAY);
}
if (EJSDateUtils.isNew(start, DateUtils.addMinutes(EJSDateUtils.getCurrentDate(), -3))) {
start = DateUtils.addMinutes(EJSDateUtils.getCurrentDate(), -3);
}
return start;
}
/**
* 获取订单抓取结束时间
*
* @return
*/
public Date getEndDate(Date endDate, Date start) {
Date end = DateUtils.addMinutes(EJSDateUtils.getCurrentDate(), -3); // 默认结束时间为当前时间往前推3分钟
if (endDate != null) {
end = endDate;
}
if (DateUtils.addSeconds(start, ConstantJingDong.JD_FETCH_ORDER_TIME_INTERVAL_INCREMENT)
.compareTo(end)
< 0) {
// 开始时间相对于抓单结束时间超过30Min
end = DateUtils.addSeconds(start, ConstantJingDong.JD_FETCH_ORDER_TIME_INTERVAL_INCREMENT);
}
return end;
}
@Override
public List<Properties> buildCoords(Cluster cluster, Path buildPath) throws FalconException {
org.apache.falcon.entity.v0.feed.Cluster feedCluster =
FeedHelper.getCluster(entity, cluster.getName());
if (feedCluster == null) {
return null;
}
COORDINATORAPP coord = new COORDINATORAPP();
String coordName = getEntityName();
coord.setName(coordName);
Date endDate = feedCluster.getValidity().getEnd();
coord.setEnd(SchemaHelper.formatDateUTC(endDate));
if (feedCluster.getValidity().getEnd().before(new Date())) {
Date startDate = DateUtils.addMinutes(endDate, -1);
coord.setStart(SchemaHelper.formatDateUTC(startDate));
} else {
coord.setStart(SchemaHelper.formatDateUTC(new Date()));
}
coord.setTimezone(entity.getTimezone().getID());
TimeUnit timeUnit = entity.getFrequency().getTimeUnit();
if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
coord.setFrequency("${coord:hours(6)}");
} else {
coord.setFrequency("${coord:days(1)}");
}
Path coordPath = getBuildPath(buildPath);
Properties props = createCoordDefaultConfiguration(coordName);
WORKFLOW workflow = new WORKFLOW();
Properties wfProps =
OozieOrchestrationWorkflowBuilder.get(entity, cluster, Tag.RETENTION)
.build(cluster, coordPath);
workflow.setAppPath(getStoragePath(wfProps.getProperty(OozieEntityBuilder.ENTITY_PATH)));
props.putAll(getProperties(coordPath, coordName));
// Add the custom properties set in feed. Else, dryrun won't catch any missing props.
props.putAll(EntityUtil.getEntityProperties(entity));
workflow.setConfiguration(getConfig(props));
ACTION action = new ACTION();
action.setWorkflow(workflow);
coord.setAction(action);
Path marshalPath = marshal(cluster, coord, coordPath);
return Arrays.asList(getProperties(marshalPath, coordName));
}
/**
* Creates a signed and encoded JSON Web Token with the given jsonUserDetails, an issue time and
* an expiration time according to the timeout configured in the authentication properties file.
*/
public String createToken(String jsonUserDetails) throws JsonProcessingException {
final Date issueTime = new Date();
final Date expirationTime = DateUtils.addMinutes(issueTime, maxAuthMinutes);
// collect JWT claims
Map<Claims, Object> claimSet = new HashMap<>();
claimSet.put(Claims.SUB, jsonUserDetails);
claimSet.put(Claims.IAT, issueTime.getTime() / DateUtils.MILLIS_PER_SECOND);
claimSet.put(Claims.EXP, expirationTime.getTime() / DateUtils.MILLIS_PER_SECOND);
// convert claims to JSON and create JSON Web Token
Jwt token = JwtHelper.encode(jsonMapper.writeValueAsString(claimSet), signer);
token.verifySignature(verifier);
// return the encoded token as a string
return token.getEncoded();
}
private Date calculateDate(Date beginDate, String timeType, Integer amount) {
Date limitDate = null;
timeType = timeType == null ? "" : timeType;
amount = amount == null ? 0 : amount;
switch (timeType) {
case "M":
limitDate = DateUtils.addMinutes(beginDate, amount);
break;
case "H":
limitDate = DateUtils.addHours(beginDate, amount);
break;
case "D":
limitDate = DateUtils.addDays(beginDate, amount);
break;
default:
// limitDate = beginDate;
break;
}
return limitDate;
}
@Override
public byte[] renderUserMethodGraphForUser(long theUserPid, TimeRange theRange)
throws IOException {
ourLog.info("Rendering user method graph for user {}", theUserPid);
myBroadcastSender.requestFlushQueuedStats();
PersUser user = myDao.getUser(theUserPid);
Date start = theRange.getNoPresetFrom();
Date end = theRange.getNoPresetTo();
if (theRange.getWithPresetRange() != null) {
start =
new Date(
System.currentTimeMillis()
- (theRange.getWithPresetRange().getNumMins() * DateUtils.MILLIS_PER_MINUTE));
end = new Date();
}
HashMap<Long, List<Double>> methods = new HashMap<>();
List<Long> timestamps = new ArrayList<>();
InvocationStatsIntervalEnum nextInterval =
doWithStatsSupportFindInterval(myConfig.getConfig(), start);
Date nextDate = nextInterval.truncate(start);
Date nextDateEnd = null;
List<PersInvocationMethodUserStats> stats = myDao.getUserStatsWithinTimeRange(user, start, end);
Validate.notNull(stats);
stats = new ArrayList<>(stats);
Collections.sort(
stats,
new Comparator<PersInvocationMethodUserStats>() {
@Override
public int compare(
PersInvocationMethodUserStats theO1, PersInvocationMethodUserStats theO2) {
return theO1.getPk().getStartTime().compareTo(theO2.getPk().getStartTime());
}
});
ourLog.debug(
"Loaded {} user {} stats for range {} - {}",
new Object[] {stats.size(), theUserPid, start, end});
if (stats.size() > 0) {
ourLog.debug(
"Found stats with range {} - {}",
stats.get(0).getPk().getStartTime(),
stats.get(stats.size() - 1).getPk().getStartTime());
}
ourLog.debug("Looking for stats starting at {}", nextDate);
Iterator<PersInvocationMethodUserStats> statIter = stats.iterator();
PersInvocationMethodUserStats nextStat = null;
int timesPassed = 0;
int foundEntries = 0;
double grandTotal = 0;
while (nextDate.after(end) == false) {
double numMinutes = nextInterval.numMinutes();
nextDateEnd = DateUtils.addMinutes(nextDate, (int) nextInterval.numMinutes());
timestamps.add(nextDate.getTime());
int arrayIndex = timesPassed;
timesPassed++;
for (List<Double> next : methods.values()) {
next.add(ZERO_DOUBLE);
}
while (nextStat == null || statIter.hasNext()) {
if (nextStat == null && statIter.hasNext()) {
nextStat = statIter.next();
foundEntries++;
}
if (nextStat != null && nextStat.getPk().getStartTime().before(nextDateEnd)) {
long methodPid = nextStat.getPk().getMethod();
List<Double> newList = methods.get(methodPid);
if (newList == null) {
newList = new ArrayList<>();
for (int i = 0; i < timesPassed; i++) {
newList.add(ZERO_DOUBLE);
}
methods.put(methodPid, newList);
}
// TODO: should we have an option to include fails/faults?
double total = nextStat.getSuccessInvocationCount();
double minuteTotal = total / numMinutes;
grandTotal += minuteTotal;
newList.set(arrayIndex, newList.get(arrayIndex) + minuteTotal);
nextStat = null;
} else {
break;
}
}
nextDate = doWithStatsSupportIncrement(nextDate, nextInterval);
nextInterval = doWithStatsSupportFindInterval(myConfig.getConfig(), nextDate);
nextDate = nextInterval.truncate(nextDate);
}
ourLog.debug("Found {} entries for {} methods", new Object[] {foundEntries, methods.size()});
// Come up with a good order
final Map<Long, PersMethod> pidToMethod = new HashMap<>();
for (long nextPid : methods.keySet()) {
PersMethod method = myDao.getServiceVersionMethodByPid(nextPid);
if (method != null) {
pidToMethod.put(nextPid, method);
} else {
ourLog.debug("Discarding unknown method: {}", nextPid);
}
}
ArrayList<Long> pids = new ArrayList<>(pidToMethod.keySet());
Collections.sort(
pids,
new Comparator<Long>() {
@Override
public int compare(Long theO1, Long theO2) {
PersMethod m1 = pidToMethod.get(theO1);
PersMethod m2 = pidToMethod.get(theO2);
return MethodComparator.INSTANCE.compare(m1, m2);
}
});
RrdGraphDef graphDef = new RrdGraphDef();
graphDef.setWidth(600);
graphDef.setHeight(200);
long[] graphTimestamps = new long[timestamps.size()];
for (int i = 0; i < timestamps.size(); i++) {
graphTimestamps[i] = timestamps.get(i) / 1000;
}
graphDef.setTimeSpan(graphTimestamps);
graphDef.setVerticalLabel("Calls / Minute");
graphDef.setTextAntiAliasing(true);
int longestName = 0;
for (PersMethod next : pidToMethod.values()) {
longestName = Math.max(longestName, next.getName().length());
}
// Draw the methods
String previousServiceDesc = null;
List<Color> colours = createStackColours(pids.size());
for (int i = 0; i < pids.size(); i++) {
Long nextPid = pids.get(i);
PersMethod nextMethod = pidToMethod.get(nextPid);
List<Double> values = methods.get(nextPid);
LinearInterpolator avgPlot =
new LinearInterpolator(graphTimestamps, toDoublesFromDoubles(values));
String srcName = "inv" + i;
graphDef.datasource(srcName, avgPlot);
String methodDesc =
nextMethod.getServiceVersion().getService().getServiceId()
+ " "
+ nextMethod.getServiceVersion().getVersionId();
if (!StringUtils.equals(previousServiceDesc, methodDesc)) {
graphDef.comment(methodDesc + "\\l");
previousServiceDesc = methodDesc;
}
double sumDoubles = sumDoubles(values);
double pct = (sumDoubles / grandTotal);
if (i == 0) {
graphDef.area(
srcName, colours.get(i), " " + StringUtils.rightPad(nextMethod.getName(), longestName));
} else {
graphDef.stack(
srcName, colours.get(i), " " + StringUtils.rightPad(nextMethod.getName(), longestName));
}
graphDef.gprint(srcName, ConsolFun.AVERAGE, "Avg %5.1f ");
graphDef.gprint(srcName, ConsolFun.MIN, "Min %5.1f ");
graphDef.gprint(srcName, ConsolFun.MAX, "Max %5.1f ");
String formattedPct = new DecimalFormat("0.0#%").format(pct);
graphDef.comment("Pct: " + formattedPct + "\\l");
}
if (pids.size() == 0) {
double[] values = new double[timestamps.size()];
for (int j = 0; j < values.length; j++) {
values[j] = 0.0;
}
LinearInterpolator avgPlot = new LinearInterpolator(graphTimestamps, values);
String srcName = "inv";
graphDef.datasource(srcName, avgPlot);
graphDef.area(
srcName, Color.BLACK, StringUtils.rightPad("No activity during this range", 100));
}
return render(graphDef);
}
