Ejemplo n.º 1
0
  /**
   * 获取订单抓取开始时间
   *
   * @param fetchTime
   * @return
   */
  public Date getStartDate(Date startDate, Date fetchTime) {
    if (startDate != null) {
      return startDate;
    }

    Date start;
    if (fetchTime == null) { // 没有抓取记录
      // 当前时间往前推N秒
      start =
          DateUtils.addSeconds(
              EJSDateUtils.getCurrentDate(), -ConstantJingDong.JD_FETCH_ORDER_TIME_INTERVAL);
    } else {
      // 将抓取订单时间往前推N秒
      start = DateUtils.addSeconds(fetchTime, ConstantJingDong.JD_FETCH_ORDER_TIME_DELAY);
    }

    if (EJSDateUtils.isNew(start, DateUtils.addMinutes(EJSDateUtils.getCurrentDate(), -3))) {
      start = DateUtils.addMinutes(EJSDateUtils.getCurrentDate(), -3);
    }

    return start;
  }
Ejemplo n.º 2
0
 /**
  * 获取订单抓取结束时间
  *
  * @return
  */
 public Date getEndDate(Date endDate, Date start) {
   Date end = DateUtils.addMinutes(EJSDateUtils.getCurrentDate(), -3); // 默认结束时间为当前时间往前推3分钟
   if (endDate != null) {
     end = endDate;
   }
   if (DateUtils.addSeconds(start, ConstantJingDong.JD_FETCH_ORDER_TIME_INTERVAL_INCREMENT)
           .compareTo(end)
       < 0) {
     // 开始时间相对于抓单结束时间超过30Min
     end = DateUtils.addSeconds(start, ConstantJingDong.JD_FETCH_ORDER_TIME_INTERVAL_INCREMENT);
   }
   return end;
 }
  @Override
  public List<Properties> buildCoords(Cluster cluster, Path buildPath) throws FalconException {
    org.apache.falcon.entity.v0.feed.Cluster feedCluster =
        FeedHelper.getCluster(entity, cluster.getName());
    if (feedCluster == null) {
      return null;
    }

    COORDINATORAPP coord = new COORDINATORAPP();
    String coordName = getEntityName();
    coord.setName(coordName);
    Date endDate = feedCluster.getValidity().getEnd();
    coord.setEnd(SchemaHelper.formatDateUTC(endDate));
    if (feedCluster.getValidity().getEnd().before(new Date())) {
      Date startDate = DateUtils.addMinutes(endDate, -1);
      coord.setStart(SchemaHelper.formatDateUTC(startDate));
    } else {
      coord.setStart(SchemaHelper.formatDateUTC(new Date()));
    }
    coord.setTimezone(entity.getTimezone().getID());
    TimeUnit timeUnit = entity.getFrequency().getTimeUnit();
    if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
      coord.setFrequency("${coord:hours(6)}");
    } else {
      coord.setFrequency("${coord:days(1)}");
    }

    Path coordPath = getBuildPath(buildPath);
    Properties props = createCoordDefaultConfiguration(coordName);

    WORKFLOW workflow = new WORKFLOW();
    Properties wfProps =
        OozieOrchestrationWorkflowBuilder.get(entity, cluster, Tag.RETENTION)
            .build(cluster, coordPath);
    workflow.setAppPath(getStoragePath(wfProps.getProperty(OozieEntityBuilder.ENTITY_PATH)));
    props.putAll(getProperties(coordPath, coordName));
    // Add the custom properties set in feed. Else, dryrun won't catch any missing props.
    props.putAll(EntityUtil.getEntityProperties(entity));
    workflow.setConfiguration(getConfig(props));
    ACTION action = new ACTION();
    action.setWorkflow(workflow);

    coord.setAction(action);

    Path marshalPath = marshal(cluster, coord, coordPath);
    return Arrays.asList(getProperties(marshalPath, coordName));
  }
Ejemplo n.º 4
0
  /**
   * Creates a signed and encoded JSON Web Token with the given jsonUserDetails, an issue time and
   * an expiration time according to the timeout configured in the authentication properties file.
   */
  public String createToken(String jsonUserDetails) throws JsonProcessingException {
    final Date issueTime = new Date();
    final Date expirationTime = DateUtils.addMinutes(issueTime, maxAuthMinutes);

    // collect JWT claims
    Map<Claims, Object> claimSet = new HashMap<>();
    claimSet.put(Claims.SUB, jsonUserDetails);
    claimSet.put(Claims.IAT, issueTime.getTime() / DateUtils.MILLIS_PER_SECOND);
    claimSet.put(Claims.EXP, expirationTime.getTime() / DateUtils.MILLIS_PER_SECOND);

    // convert claims to JSON and create JSON Web Token
    Jwt token = JwtHelper.encode(jsonMapper.writeValueAsString(claimSet), signer);
    token.verifySignature(verifier);

    // return the encoded token as a string
    return token.getEncoded();
  }
Ejemplo n.º 5
0
 private Date calculateDate(Date beginDate, String timeType, Integer amount) {
   Date limitDate = null;
   timeType = timeType == null ? "" : timeType;
   amount = amount == null ? 0 : amount;
   switch (timeType) {
     case "M":
       limitDate = DateUtils.addMinutes(beginDate, amount);
       break;
     case "H":
       limitDate = DateUtils.addHours(beginDate, amount);
       break;
     case "D":
       limitDate = DateUtils.addDays(beginDate, amount);
       break;
     default:
       //			limitDate = beginDate;
       break;
   }
   return limitDate;
 }
  @Override
  public byte[] renderUserMethodGraphForUser(long theUserPid, TimeRange theRange)
      throws IOException {
    ourLog.info("Rendering user method graph for user {}", theUserPid);

    myBroadcastSender.requestFlushQueuedStats();

    PersUser user = myDao.getUser(theUserPid);

    Date start = theRange.getNoPresetFrom();
    Date end = theRange.getNoPresetTo();
    if (theRange.getWithPresetRange() != null) {
      start =
          new Date(
              System.currentTimeMillis()
                  - (theRange.getWithPresetRange().getNumMins() * DateUtils.MILLIS_PER_MINUTE));
      end = new Date();
    }

    HashMap<Long, List<Double>> methods = new HashMap<>();
    List<Long> timestamps = new ArrayList<>();

    InvocationStatsIntervalEnum nextInterval =
        doWithStatsSupportFindInterval(myConfig.getConfig(), start);
    Date nextDate = nextInterval.truncate(start);
    Date nextDateEnd = null;

    List<PersInvocationMethodUserStats> stats = myDao.getUserStatsWithinTimeRange(user, start, end);
    Validate.notNull(stats);
    stats = new ArrayList<>(stats);
    Collections.sort(
        stats,
        new Comparator<PersInvocationMethodUserStats>() {
          @Override
          public int compare(
              PersInvocationMethodUserStats theO1, PersInvocationMethodUserStats theO2) {
            return theO1.getPk().getStartTime().compareTo(theO2.getPk().getStartTime());
          }
        });
    ourLog.debug(
        "Loaded {} user {} stats for range {} - {}",
        new Object[] {stats.size(), theUserPid, start, end});
    if (stats.size() > 0) {
      ourLog.debug(
          "Found stats with range {} - {}",
          stats.get(0).getPk().getStartTime(),
          stats.get(stats.size() - 1).getPk().getStartTime());
    }
    ourLog.debug("Looking for stats starting at {}", nextDate);

    Iterator<PersInvocationMethodUserStats> statIter = stats.iterator();
    PersInvocationMethodUserStats nextStat = null;

    int timesPassed = 0;
    int foundEntries = 0;
    double grandTotal = 0;
    while (nextDate.after(end) == false) {
      double numMinutes = nextInterval.numMinutes();
      nextDateEnd = DateUtils.addMinutes(nextDate, (int) nextInterval.numMinutes());
      timestamps.add(nextDate.getTime());
      int arrayIndex = timesPassed;
      timesPassed++;

      for (List<Double> next : methods.values()) {
        next.add(ZERO_DOUBLE);
      }

      while (nextStat == null || statIter.hasNext()) {
        if (nextStat == null && statIter.hasNext()) {
          nextStat = statIter.next();
          foundEntries++;
        }
        if (nextStat != null && nextStat.getPk().getStartTime().before(nextDateEnd)) {
          long methodPid = nextStat.getPk().getMethod();
          List<Double> newList = methods.get(methodPid);
          if (newList == null) {
            newList = new ArrayList<>();
            for (int i = 0; i < timesPassed; i++) {
              newList.add(ZERO_DOUBLE);
            }
            methods.put(methodPid, newList);
          }

          // TODO: should we have an option to include fails/faults?
          double total = nextStat.getSuccessInvocationCount();
          double minuteTotal = total / numMinutes;

          grandTotal += minuteTotal;

          newList.set(arrayIndex, newList.get(arrayIndex) + minuteTotal);
          nextStat = null;
        } else {
          break;
        }
      }

      nextDate = doWithStatsSupportIncrement(nextDate, nextInterval);
      nextInterval = doWithStatsSupportFindInterval(myConfig.getConfig(), nextDate);
      nextDate = nextInterval.truncate(nextDate);
    }

    ourLog.debug("Found {} entries for {} methods", new Object[] {foundEntries, methods.size()});

    // Come up with a good order
    final Map<Long, PersMethod> pidToMethod = new HashMap<>();
    for (long nextPid : methods.keySet()) {
      PersMethod method = myDao.getServiceVersionMethodByPid(nextPid);
      if (method != null) {
        pidToMethod.put(nextPid, method);
      } else {
        ourLog.debug("Discarding unknown method: {}", nextPid);
      }
    }
    ArrayList<Long> pids = new ArrayList<>(pidToMethod.keySet());
    Collections.sort(
        pids,
        new Comparator<Long>() {
          @Override
          public int compare(Long theO1, Long theO2) {
            PersMethod m1 = pidToMethod.get(theO1);
            PersMethod m2 = pidToMethod.get(theO2);
            return MethodComparator.INSTANCE.compare(m1, m2);
          }
        });

    RrdGraphDef graphDef = new RrdGraphDef();
    graphDef.setWidth(600);
    graphDef.setHeight(200);

    long[] graphTimestamps = new long[timestamps.size()];
    for (int i = 0; i < timestamps.size(); i++) {
      graphTimestamps[i] = timestamps.get(i) / 1000;
    }

    graphDef.setTimeSpan(graphTimestamps);
    graphDef.setVerticalLabel("Calls / Minute");
    graphDef.setTextAntiAliasing(true);

    int longestName = 0;
    for (PersMethod next : pidToMethod.values()) {
      longestName = Math.max(longestName, next.getName().length());
    }

    // Draw the methods
    String previousServiceDesc = null;
    List<Color> colours = createStackColours(pids.size());
    for (int i = 0; i < pids.size(); i++) {
      Long nextPid = pids.get(i);
      PersMethod nextMethod = pidToMethod.get(nextPid);
      List<Double> values = methods.get(nextPid);

      LinearInterpolator avgPlot =
          new LinearInterpolator(graphTimestamps, toDoublesFromDoubles(values));
      String srcName = "inv" + i;
      graphDef.datasource(srcName, avgPlot);
      String methodDesc =
          nextMethod.getServiceVersion().getService().getServiceId()
              + " "
              + nextMethod.getServiceVersion().getVersionId();
      if (!StringUtils.equals(previousServiceDesc, methodDesc)) {
        graphDef.comment(methodDesc + "\\l");
        previousServiceDesc = methodDesc;
      }

      double sumDoubles = sumDoubles(values);
      double pct = (sumDoubles / grandTotal);

      if (i == 0) {
        graphDef.area(
            srcName, colours.get(i), " " + StringUtils.rightPad(nextMethod.getName(), longestName));
      } else {
        graphDef.stack(
            srcName, colours.get(i), " " + StringUtils.rightPad(nextMethod.getName(), longestName));
      }
      graphDef.gprint(srcName, ConsolFun.AVERAGE, "Avg %5.1f ");
      graphDef.gprint(srcName, ConsolFun.MIN, "Min %5.1f ");
      graphDef.gprint(srcName, ConsolFun.MAX, "Max %5.1f ");

      String formattedPct = new DecimalFormat("0.0#%").format(pct);
      graphDef.comment("Pct: " + formattedPct + "\\l");
    }

    if (pids.size() == 0) {
      double[] values = new double[timestamps.size()];
      for (int j = 0; j < values.length; j++) {
        values[j] = 0.0;
      }
      LinearInterpolator avgPlot = new LinearInterpolator(graphTimestamps, values);
      String srcName = "inv";
      graphDef.datasource(srcName, avgPlot);
      graphDef.area(
          srcName, Color.BLACK, StringUtils.rightPad("No activity during this range", 100));
    }

    return render(graphDef);
  }