@Override
public void reduce(
K key,
Iterator<V> values,
OutputCollector<Text, TextArrayWritable> output,
Reporter reporter)
throws IOException {
// TODO Auto-generated method stub
// organizing into csv format
Map<String, String> newRecord = new HashMap<String, String>();
String[] headers =
new String[] {
"attempt_id",
"breakdown_count",
"breakdown_name",
"breakdown_type",
"host",
"job_id",
"phase_count",
"phase_end",
"phase_name",
"phase_stack",
"phase_start",
"thread_id",
"thread_name"
};
for (String head : headers) {
newRecord.put(head, "");
}
long start = -1, end = -1;
long phaseCount = 0;
String funcCount = "", statusCount = "";
String funcList = "", statusList = "";
while (values.hasNext()) {
HiTuneRecord valproxy = new HiTuneRecord(values.next());
long phaseStart = Long.parseLong(valproxy.getValue("start"));
long phaseEnd = Long.parseLong(valproxy.getValue("start"));
start = start == -1 ? phaseStart : Math.min(start, phaseStart);
end = end == -1 ? phaseEnd : Math.max(end, phaseEnd);
phaseCount++;
funcCount =
funcCount == ""
? valproxy.getValue("funCount")
: vectorAdd(valproxy.getValue("funCount"), funcCount, SEPERATOR_COMMA);
statusCount =
statusCount == ""
? valproxy.getValue("statusCount")
: vectorAdd(valproxy.getValue("statusCount"), statusCount, SEPERATOR_COMMA);
newRecord.put("host", valproxy.getHost());
newRecord.put("job_id", valproxy.getValue(AnalysisProcessorConfiguration.jobid));
newRecord.put("phase_stack", valproxy.getValue("phase_stack"));
newRecord.put("phase_name", valproxy.getValue("phase_name"));
newRecord.put("attempt_id", valproxy.getValue("attempt_id"));
newRecord.put("thread_id", valproxy.getValue("thread_id"));
newRecord.put("thread_name", valproxy.getValue("thread_name"));
funcList = valproxy.getValue("funList");
statusList = valproxy.getValue("statusList");
}
newRecord.put("phase_start", "" + start);
newRecord.put("phase_end", "" + end);
newRecord.put("phase_count", "" + phaseCount);
if (!initialized) {
TextArrayWritable newValue =
new TextArrayWritable(newRecord.keySet().toArray(new String[0]));
output.collect(null, newValue);
initialized = true;
}
if (!funcCount.equals("")) {
newRecord.put("breakdown_type", "function");
log.debug("funcList: " + funcList);
log.debug("funcCount: " + funcCount);
List<String> tmp = String2List(funcList, SEPERATOR_COMMA);
List<String> counts = String2List(funcCount, SEPERATOR_COMMA);
for (int i = 0; i < tmp.size(); i++) {
log.debug("function:" + tmp.get(i) + " count:" + counts.get(i));
newRecord.put("breakdown_name", tmp.get(i));
newRecord.put("breakdown_count", "" + counts.get(i));
String[] contents = new String[newRecord.keySet().size()];
int j = 0;
for (String index : newRecord.keySet()) {
contents[j] = newRecord.get(index);
log.debug("content: " + index + "," + contents[j]);
j++;
}
TextArrayWritable newValue = new TextArrayWritable(contents);
output.collect(null, newValue);
contents = null;
}
}
if (!statusCount.equals("")) {
newRecord.put("breakdown_type", "state");
log.debug("statusList: " + statusList);
log.debug("statusCount: " + statusCount);
List<String> tmp = String2List(statusList, SEPERATOR_COMMA);
List<String> counts = String2List(statusCount, SEPERATOR_COMMA);
for (int i = 0; i < tmp.size(); i++) {
log.debug("function:" + tmp.get(i) + " count:" + counts.get(i));
newRecord.put("breakdown_name", tmp.get(i));
newRecord.put("breakdown_count", "" + counts.get(i));
String[] contents = new String[newRecord.size()];
int j = 0;
for (String index : newRecord.keySet()) {
contents[j] = newRecord.get(index);
j++;
log.debug("content: " + index + "," + contents[i]);
}
TextArrayWritable newValue = new TextArrayWritable(contents);
output.collect(null, newValue);
contents = null;
}
}
}
@Override
public void map(K key, V value, OutputCollector<K, V> output, Reporter reporter)
throws IOException {
// TODO Auto-generated method stub
// doing the filter
// <key,value>
// <[AttemptID/PhaseStack/PhaseAlias],
// [ThreadName,ThreadId,starttime,endtime,funlist,funcountlist,statelist,statecountlist,funStateMatric]>
HiTuneRecord valproxy = new HiTuneRecord(value);
String hostname = valproxy.getHost();
String status = valproxy.getValue("ThreadState");
String stack = valproxy.getValue("CallStack");
String attemptID = valproxy.getValue("TaskID");
log.debug(
"hostname:"
+ hostname
+ " ThreadState:"
+ status
+ " stack:"
+ stack
+ " attemptID:"
+ attemptID);
if (isMatched(this.nodelist, hostname)) {
for (String s : phasealias.keySet()) {
log.debug("phasealias:" + s);
if (s == null || s.length() == 0) s = "";
Pattern p = Pattern.compile(s);
if (stack != null && stack.length() != 0) stack = stack.replace(" ", "");
else stack = "";
Matcher matcher = p.matcher(stack);
if (matcher.find()) {
try {
log.debug("find pattern");
K newkey = (K) key.getClass().getConstructor().newInstance();
V newval = (V) value.getClass().getConstructor().newInstance();
HiTuneKey newkeyproxy = new HiTuneKey(newkey);
HiTuneRecord newvalproxy = new HiTuneRecord(newval);
newkeyproxy.setKey(attemptID + "/" + s + "/" + phasealias.get(s));
newkeyproxy.setDataType(new HiTuneKey(key).getDataType());
newvalproxy.copyCommonFields(value);
newvalproxy.add("thread_id", valproxy.getValue("ThreadID"));
newvalproxy.add("thread_name", valproxy.getValue("ThreadName"));
newvalproxy.add("attempt_id", attemptID);
newvalproxy.add("phase_stack", s);
newvalproxy.add("phase_name", phasealias.get(s));
newvalproxy.add("start", "" + newvalproxy.getTime());
newvalproxy.add("count", "1");
log.debug("status:" + conf.get("status"));
newvalproxy.add("statusList", conf.get("status"));
newvalproxy.add("statusCount", count(status, this.statuslist));
log.debug("funList:" + this.phases.get(s));
newvalproxy.add("funList", List2String(this.phases.get(s), SEPERATOR_COMMA));
newvalproxy.add("funCount", count(stack, this.phases.get(s)));
newvalproxy.add(
AnalysisProcessorConfiguration.jobid,
conf.get(AnalysisProcessorConfiguration.jobid));
log.debug("Key:" + newkeyproxy.toString() + " Record" + newkeyproxy.toString());
output.collect((K) newkeyproxy.getObject(), (V) newvalproxy.getObject());
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
log.warn(e);
e.printStackTrace();
} catch (SecurityException e) {
// TODO Auto-generated catch block
log.warn(e);
e.printStackTrace();
} catch (InstantiationException e) {
// TODO Auto-generated catch block
log.warn(e);
e.printStackTrace();
} catch (IllegalAccessException e) {
// TODO Auto-generated catch block
log.warn(e);
e.printStackTrace();
} catch (InvocationTargetException e) {
// TODO Auto-generated catch block
log.warn(e);
e.printStackTrace();
} catch (NoSuchMethodException e) {
// TODO Auto-generated catch block
log.warn(e);
e.printStackTrace();
}
}
}
}
}