コード例 #1
0
    @Override
    public void reduce(
        K key,
        Iterator<V> values,
        OutputCollector<Text, TextArrayWritable> output,
        Reporter reporter)
        throws IOException {
      // TODO Auto-generated method stub
      // organizing into csv format
      Map<String, String> newRecord = new HashMap<String, String>();
      String[] headers =
          new String[] {
            "attempt_id",
            "breakdown_count",
            "breakdown_name",
            "breakdown_type",
            "host",
            "job_id",
            "phase_count",
            "phase_end",
            "phase_name",
            "phase_stack",
            "phase_start",
            "thread_id",
            "thread_name"
          };
      for (String head : headers) {
        newRecord.put(head, "");
      }

      long start = -1, end = -1;
      long phaseCount = 0;
      String funcCount = "", statusCount = "";
      String funcList = "", statusList = "";
      while (values.hasNext()) {
        HiTuneRecord valproxy = new HiTuneRecord(values.next());
        long phaseStart = Long.parseLong(valproxy.getValue("start"));
        long phaseEnd = Long.parseLong(valproxy.getValue("start"));
        start = start == -1 ? phaseStart : Math.min(start, phaseStart);
        end = end == -1 ? phaseEnd : Math.max(end, phaseEnd);
        phaseCount++;
        funcCount =
            funcCount == ""
                ? valproxy.getValue("funCount")
                : vectorAdd(valproxy.getValue("funCount"), funcCount, SEPERATOR_COMMA);
        statusCount =
            statusCount == ""
                ? valproxy.getValue("statusCount")
                : vectorAdd(valproxy.getValue("statusCount"), statusCount, SEPERATOR_COMMA);
        newRecord.put("host", valproxy.getHost());
        newRecord.put("job_id", valproxy.getValue(AnalysisProcessorConfiguration.jobid));
        newRecord.put("phase_stack", valproxy.getValue("phase_stack"));
        newRecord.put("phase_name", valproxy.getValue("phase_name"));
        newRecord.put("attempt_id", valproxy.getValue("attempt_id"));
        newRecord.put("thread_id", valproxy.getValue("thread_id"));
        newRecord.put("thread_name", valproxy.getValue("thread_name"));
        funcList = valproxy.getValue("funList");
        statusList = valproxy.getValue("statusList");
      }

      newRecord.put("phase_start", "" + start);
      newRecord.put("phase_end", "" + end);
      newRecord.put("phase_count", "" + phaseCount);

      if (!initialized) {
        TextArrayWritable newValue =
            new TextArrayWritable(newRecord.keySet().toArray(new String[0]));
        output.collect(null, newValue);
        initialized = true;
      }

      if (!funcCount.equals("")) {
        newRecord.put("breakdown_type", "function");
        log.debug("funcList: " + funcList);
        log.debug("funcCount: " + funcCount);
        List<String> tmp = String2List(funcList, SEPERATOR_COMMA);
        List<String> counts = String2List(funcCount, SEPERATOR_COMMA);
        for (int i = 0; i < tmp.size(); i++) {
          log.debug("function:" + tmp.get(i) + " count:" + counts.get(i));
          newRecord.put("breakdown_name", tmp.get(i));
          newRecord.put("breakdown_count", "" + counts.get(i));
          String[] contents = new String[newRecord.keySet().size()];
          int j = 0;
          for (String index : newRecord.keySet()) {
            contents[j] = newRecord.get(index);
            log.debug("content: " + index + "," + contents[j]);
            j++;
          }
          TextArrayWritable newValue = new TextArrayWritable(contents);
          output.collect(null, newValue);
          contents = null;
        }
      }
      if (!statusCount.equals("")) {
        newRecord.put("breakdown_type", "state");
        log.debug("statusList: " + statusList);
        log.debug("statusCount: " + statusCount);
        List<String> tmp = String2List(statusList, SEPERATOR_COMMA);
        List<String> counts = String2List(statusCount, SEPERATOR_COMMA);
        for (int i = 0; i < tmp.size(); i++) {
          log.debug("function:" + tmp.get(i) + " count:" + counts.get(i));
          newRecord.put("breakdown_name", tmp.get(i));
          newRecord.put("breakdown_count", "" + counts.get(i));
          String[] contents = new String[newRecord.size()];
          int j = 0;
          for (String index : newRecord.keySet()) {
            contents[j] = newRecord.get(index);
            j++;
            log.debug("content: " + index + "," + contents[i]);
          }
          TextArrayWritable newValue = new TextArrayWritable(contents);
          output.collect(null, newValue);
          contents = null;
        }
      }
    }
コード例 #2
0
    @Override
    public void map(K key, V value, OutputCollector<K, V> output, Reporter reporter)
        throws IOException {
      // TODO Auto-generated method stub
      // doing the filter

      // <key,value>
      // <[AttemptID/PhaseStack/PhaseAlias],
      // [ThreadName,ThreadId,starttime,endtime,funlist,funcountlist,statelist,statecountlist,funStateMatric]>
      HiTuneRecord valproxy = new HiTuneRecord(value);
      String hostname = valproxy.getHost();
      String status = valproxy.getValue("ThreadState");
      String stack = valproxy.getValue("CallStack");
      String attemptID = valproxy.getValue("TaskID");
      log.debug(
          "hostname:"
              + hostname
              + " ThreadState:"
              + status
              + " stack:"
              + stack
              + " attemptID:"
              + attemptID);
      if (isMatched(this.nodelist, hostname)) {
        for (String s : phasealias.keySet()) {
          log.debug("phasealias:" + s);
          if (s == null || s.length() == 0) s = "";
          Pattern p = Pattern.compile(s);
          if (stack != null && stack.length() != 0) stack = stack.replace(" ", "");
          else stack = "";
          Matcher matcher = p.matcher(stack);
          if (matcher.find()) {
            try {
              log.debug("find pattern");
              K newkey = (K) key.getClass().getConstructor().newInstance();
              V newval = (V) value.getClass().getConstructor().newInstance();

              HiTuneKey newkeyproxy = new HiTuneKey(newkey);
              HiTuneRecord newvalproxy = new HiTuneRecord(newval);

              newkeyproxy.setKey(attemptID + "/" + s + "/" + phasealias.get(s));
              newkeyproxy.setDataType(new HiTuneKey(key).getDataType());
              newvalproxy.copyCommonFields(value);

              newvalproxy.add("thread_id", valproxy.getValue("ThreadID"));
              newvalproxy.add("thread_name", valproxy.getValue("ThreadName"));
              newvalproxy.add("attempt_id", attemptID);
              newvalproxy.add("phase_stack", s);
              newvalproxy.add("phase_name", phasealias.get(s));
              newvalproxy.add("start", "" + newvalproxy.getTime());
              newvalproxy.add("count", "1");
              log.debug("status:" + conf.get("status"));
              newvalproxy.add("statusList", conf.get("status"));
              newvalproxy.add("statusCount", count(status, this.statuslist));

              log.debug("funList:" + this.phases.get(s));
              newvalproxy.add("funList", List2String(this.phases.get(s), SEPERATOR_COMMA));
              newvalproxy.add("funCount", count(stack, this.phases.get(s)));
              newvalproxy.add(
                  AnalysisProcessorConfiguration.jobid,
                  conf.get(AnalysisProcessorConfiguration.jobid));

              log.debug("Key:" + newkeyproxy.toString() + " Record" + newkeyproxy.toString());
              output.collect((K) newkeyproxy.getObject(), (V) newvalproxy.getObject());
            } catch (IllegalArgumentException e) {
              // TODO Auto-generated catch block
              log.warn(e);
              e.printStackTrace();
            } catch (SecurityException e) {
              // TODO Auto-generated catch block
              log.warn(e);
              e.printStackTrace();
            } catch (InstantiationException e) {
              // TODO Auto-generated catch block
              log.warn(e);
              e.printStackTrace();
            } catch (IllegalAccessException e) {
              // TODO Auto-generated catch block
              log.warn(e);
              e.printStackTrace();
            } catch (InvocationTargetException e) {
              // TODO Auto-generated catch block
              log.warn(e);
              e.printStackTrace();
            } catch (NoSuchMethodException e) {
              // TODO Auto-generated catch block
              log.warn(e);
              e.printStackTrace();
            }
          }
        }
      }
    }