static String addRemovePrefix(String prefix, String object, boolean add) {
if (prefix == null) {
throw new IllegalArgumentException("prefix");
}
if (object == null) {
throw new NullPointerException("object");
}
if (object.length() == 0) {
return "";
}
DBObject bsonObject = (DBObject) JSON.parse(object);
BasicBSONObject newObject = new BasicBSONObject();
for (String key : bsonObject.keySet()) {
if (add) {
newObject.put(prefix + key, bsonObject.get(key));
} else {
if (key.startsWith(prefix)) {
newObject.put(key.substring(prefix.length()), bsonObject.get(key));
} else {
newObject.put(key, bsonObject.get(key));
}
}
}
return newObject.toString();
}
@Override
public void reduce(
final IntWritable key,
final Iterator<DoubleWritable> values,
final OutputCollector<IntWritable, BSONWritable> output,
final Reporter reporter)
throws IOException {
int count = 0;
double sum = 0;
while (values.hasNext()) {
sum += values.next().get();
count++;
}
final double avg = sum / count;
if (LOG.isDebugEnabled()) {
LOG.debug("Average 10 Year Treasury for " + key.get() + " was " + avg);
}
BasicBSONObject bsonObject = new BasicBSONObject();
bsonObject.put("count", count);
bsonObject.put("avg", avg);
bsonObject.put("sum", sum);
reduceResult.setDoc(bsonObject);
output.collect(key, reduceResult);
}
@Override
public void reduce(
final IntWritable pKey, final Iterable<DoubleWritable> pValues, final Context pContext)
throws IOException, InterruptedException {
int count = 0;
double sum = 0;
for (final DoubleWritable value : pValues) {
sum += value.get();
count++;
}
final double avg = sum / count;
if (LOG.isDebugEnabled()) {
LOG.debug("Average 10 Year Treasury for " + pKey.get() + " was " + avg);
}
BasicBSONObject output = new BasicBSONObject();
output.put("count", count);
output.put("avg", avg);
output.put("sum", sum);
reduceResult.setDoc(output);
pContext.write(pKey, reduceResult);
}
private static BasicBSONObject canonicalizeBSONObject(final BSONObject from) {
final BasicBSONObject canonicalized = new BasicBSONObject();
final TreeSet<String> keysInOrder = new TreeSet<String>(from.keySet());
for (final String key : keysInOrder) {
final Object val = from.get(key);
canonicalized.put(key, canonicalize(val));
}
return canonicalized;
}
@Override
public void initialize(final InputSplit split, final TaskAttemptContext context) {
LOG.info("COLOMBO QUERY RECORD READER: Get QUERY");
this.conf = context.getConfiguration();
BasicBSONObject bObj = new BasicBSONObject();
bObj.append("keywords", this.split.getKeywords());
bObj.append("queryString", this.split.getQueryString());
this.current = bObj;
this.seen = 0;
this.total = 1;
}
public void reduce(Text key, Iterable<PageDataValue> values, Context context)
throws IOException, InterruptedException {
Map<Date, Integer> stats = new HashMap<>();
while (values.iterator().hasNext()) {
PageDataValue tmp = values.iterator().next();
Date date;
try {
date = formatter.parse(tmp.date);
if (stats.containsKey(date)) {
stats.put(date, stats.get(date) + tmp.countViews);
// System.out.println(key.toString() + " - " + stats.get(date));
} else {
stats.put(date, tmp.countViews);
}
} catch (ParseException e) {
e.printStackTrace();
}
}
Date minDate = null, maxDate = null;
for (Date date : stats.keySet()) {
if ((minDate == null) || (date.before(minDate))) {
minDate = date;
}
if ((maxDate == null) || (date.after(maxDate))) {
maxDate = date;
}
}
// System.out.println("Min Date - " + minDate.toString());
// System.out.println("Max Date - " + maxDate.toString());
Date prev = null;
Map<Date, Double> trend = new HashMap<>();
int totalPageViews = 0;
for (Date date = minDate; date.compareTo(maxDate) <= 0; ) {
if (prev != null) {
if (stats.containsKey(prev) == false) {
if (stats.containsKey(date)) {
trend.put(date, (double) stats.get(date));
} else {
trend.put(date, 0.0);
}
} else {
if (stats.containsKey(date)) {
trend.put(date, (double) (stats.get(date) - stats.get(prev))); // Slope
// System.out.println(key.toString() + " - Slope = " + stats.get(date) + "-" +
// stats.get(prev) + " = "+ trend.get(date));
} else {
trend.put(date, (double) (0 - stats.get(prev)));
}
}
} else {
trend.put(date, 0.0);
}
if (stats.containsKey(date)) {
totalPageViews += stats.get(date);
}
prev = date;
calendar.setTime(date);
calendar.add(Calendar.DATE, 1);
date = calendar.getTime();
}
for (Entry<Date, Double> entry : trend.entrySet()) {
BasicBSONObject query = new BasicBSONObject();
query.append("_id", key.toString());
BSONObject boVal =
BasicDBObjectBuilder.start()
.add("date", formatter.format(entry.getKey()))
.add("viewCount", stats.containsKey(entry.getKey()) ? stats.get(entry.getKey()) : 0)
.add("trend", entry.getValue() * (Math.log(1 + totalPageViews)))
.get();
// System.out.println(key.toString() + " - viewCount -" + (stats.containsKey(entry.getKey()) ?
// stats.get(entry.getKey()):0));
// System.out.println(key.toString() + " - trend -" + entry.getValue() * (Math.log(1 +
// totalPageViews)));
BasicBSONObject pageTitleData =
new BasicBSONObject(formatterOut.format(entry.getKey()), boVal);
BasicBSONObject pageTitleDataUpdate = new BasicBSONObject("$push", pageTitleData);
reduceResult.setQuery(query);
reduceResult.setModifiers(pageTitleDataUpdate);
reduceResult.setUpsert(true);
context.write(null, reduceResult);
}
}
/**
* 这里获取全部数据
*
* @param field
* @return
*/
public Release fill(Fields field) {
Release tb = null;
if (null != field) {
tb = new Release();
tb.setId(field.getString("_id"));
if (null != field.get(Meta.STATUS)) tb.setStatus(field.getInt(Meta.STATUS));
if (null != field.get(Meta.TID)) tb.setTid(field.getString(Meta.TID));
if (null != field.get(Meta.NAME)) tb.setName(field.getString(Meta.NAME));
if (null != field.get(Meta.OWNER)) tb.setOwner(field.getString(Meta.OWNER));
if (null != field.get(Meta.VERSION)) tb.setVersion(field.getString(Meta.VERSION));
if (null != field.get(Meta.URL)) tb.setUrl(field.getString(Meta.URL));
if (null != field.get(Meta.CRTAT)) tb.setCrtAt(field.getLong(Meta.CRTAT));
if (null != field.get(Meta.UPDAT)) tb.setUpdAt(field.getLong(Meta.UPDAT));
if (null != field.get(Meta.LEVEL)) tb.setLevel(field.getInt(Meta.LEVEL));
if (null != field.get(Meta.COPYRIGHT)) tb.setCopyright(field.getInt(Meta.COPYRIGHT));
if (null != field.get(Meta.OPERATOR)) tb.setOperator(field.getString(Meta.OPERATOR));
if (null != field.get(Meta.RELEASEEXPLAIN))
tb.setReleaseExplain(field.getString(Meta.RELEASEEXPLAIN));
if (null != field.get(Meta.UPDATEEXPLAIN))
tb.setUpdateExplain(field.getString(Meta.UPDATEEXPLAIN));
List<Server> listCover = new ArrayList<Server>();
if (null != field.get(Meta.SERVER)) {
BasicBSONList listC = (BasicBSONList) field.get(Meta.SERVER);
for (int j = 0; listC.size() > j; j++) {
BasicBSONObject bd = (BasicBSONObject) listC.get(j);
Server s = new Server();
s.setAddr(bd.getString(Meta.ADDR));
s.setContent(bd.getString(Meta.CONTENT));
s.setDownAt(bd.getLong(Meta.DOWNAT));
s.setId(bd.getString(Meta.ID));
s.setName(bd.getString(Meta.NAME));
s.setNo(bd.getString(Meta.NO));
s.setReleaseAt(bd.getLong(Meta.RELEASEAT));
s.setReleaseOwner(bd.getString(Meta.RELEASEOWNER));
s.setStauts(bd.getInt(Meta.STATUS));
s.setSummary(bd.getString(Meta.SUMMARY));
listCover.add(s);
}
tb.setServer(listCover);
}
}
return tb;
}