static String addRemovePrefix(String prefix, String object, boolean add) {
    if (prefix == null) {
      throw new IllegalArgumentException("prefix");
    }
    if (object == null) {
      throw new NullPointerException("object");
    }
    if (object.length() == 0) {
      return "";
    }
    DBObject bsonObject = (DBObject) JSON.parse(object);

    BasicBSONObject newObject = new BasicBSONObject();
    for (String key : bsonObject.keySet()) {
      if (add) {
        newObject.put(prefix + key, bsonObject.get(key));
      } else {
        if (key.startsWith(prefix)) {
          newObject.put(key.substring(prefix.length()), bsonObject.get(key));
        } else {
          newObject.put(key, bsonObject.get(key));
        }
      }
    }
    return newObject.toString();
  }
Esempio n. 2
0
  @Override
  public void reduce(
      final IntWritable key,
      final Iterator<DoubleWritable> values,
      final OutputCollector<IntWritable, BSONWritable> output,
      final Reporter reporter)
      throws IOException {
    int count = 0;
    double sum = 0;
    while (values.hasNext()) {
      sum += values.next().get();
      count++;
    }

    final double avg = sum / count;

    if (LOG.isDebugEnabled()) {
      LOG.debug("Average 10 Year Treasury for " + key.get() + " was " + avg);
    }

    BasicBSONObject bsonObject = new BasicBSONObject();
    bsonObject.put("count", count);
    bsonObject.put("avg", avg);
    bsonObject.put("sum", sum);
    reduceResult.setDoc(bsonObject);
    output.collect(key, reduceResult);
  }
Esempio n. 3
0
  @Override
  public void reduce(
      final IntWritable pKey, final Iterable<DoubleWritable> pValues, final Context pContext)
      throws IOException, InterruptedException {

    int count = 0;
    double sum = 0;
    for (final DoubleWritable value : pValues) {
      sum += value.get();
      count++;
    }

    final double avg = sum / count;

    if (LOG.isDebugEnabled()) {
      LOG.debug("Average 10 Year Treasury for " + pKey.get() + " was " + avg);
    }

    BasicBSONObject output = new BasicBSONObject();
    output.put("count", count);
    output.put("avg", avg);
    output.put("sum", sum);
    reduceResult.setDoc(output);
    pContext.write(pKey, reduceResult);
  }
Esempio n. 4
0
 private static BasicBSONObject canonicalizeBSONObject(final BSONObject from) {
   final BasicBSONObject canonicalized = new BasicBSONObject();
   final TreeSet<String> keysInOrder = new TreeSet<String>(from.keySet());
   for (final String key : keysInOrder) {
     final Object val = from.get(key);
     canonicalized.put(key, canonicalize(val));
   }
   return canonicalized;
 }
 @Override
 public void initialize(final InputSplit split, final TaskAttemptContext context) {
   LOG.info("COLOMBO QUERY RECORD READER: Get QUERY");
   this.conf = context.getConfiguration();
   BasicBSONObject bObj = new BasicBSONObject();
   bObj.append("keywords", this.split.getKeywords());
   bObj.append("queryString", this.split.getQueryString());
   this.current = bObj;
   this.seen = 0;
   this.total = 1;
 }
Esempio n. 6
0
  public void reduce(Text key, Iterable<PageDataValue> values, Context context)
      throws IOException, InterruptedException {

    Map<Date, Integer> stats = new HashMap<>();
    while (values.iterator().hasNext()) {
      PageDataValue tmp = values.iterator().next();
      Date date;
      try {
        date = formatter.parse(tmp.date);
        if (stats.containsKey(date)) {
          stats.put(date, stats.get(date) + tmp.countViews);

          // System.out.println(key.toString() + " - " + stats.get(date));
        } else {
          stats.put(date, tmp.countViews);
        }
      } catch (ParseException e) {
        e.printStackTrace();
      }
    }

    Date minDate = null, maxDate = null;
    for (Date date : stats.keySet()) {
      if ((minDate == null) || (date.before(minDate))) {
        minDate = date;
      }

      if ((maxDate == null) || (date.after(maxDate))) {
        maxDate = date;
      }
    }

    // System.out.println("Min Date - " + minDate.toString());
    // System.out.println("Max Date - " + maxDate.toString());

    Date prev = null;
    Map<Date, Double> trend = new HashMap<>();
    int totalPageViews = 0;
    for (Date date = minDate; date.compareTo(maxDate) <= 0; ) {

      if (prev != null) {
        if (stats.containsKey(prev) == false) {
          if (stats.containsKey(date)) {
            trend.put(date, (double) stats.get(date));
          } else {
            trend.put(date, 0.0);
          }
        } else {
          if (stats.containsKey(date)) {
            trend.put(date, (double) (stats.get(date) - stats.get(prev))); // Slope
            // System.out.println(key.toString() + " - Slope =  " + stats.get(date) + "-" +
            // stats.get(prev) + " = "+ trend.get(date));
          } else {
            trend.put(date, (double) (0 - stats.get(prev)));
          }
        }
      } else {
        trend.put(date, 0.0);
      }

      if (stats.containsKey(date)) {
        totalPageViews += stats.get(date);
      }

      prev = date;

      calendar.setTime(date);
      calendar.add(Calendar.DATE, 1);

      date = calendar.getTime();
    }

    for (Entry<Date, Double> entry : trend.entrySet()) {

      BasicBSONObject query = new BasicBSONObject();
      query.append("_id", key.toString());

      BSONObject boVal =
          BasicDBObjectBuilder.start()
              .add("date", formatter.format(entry.getKey()))
              .add("viewCount", stats.containsKey(entry.getKey()) ? stats.get(entry.getKey()) : 0)
              .add("trend", entry.getValue() * (Math.log(1 + totalPageViews)))
              .get();

      // System.out.println(key.toString() + " - viewCount -" + (stats.containsKey(entry.getKey()) ?
      // stats.get(entry.getKey()):0));
      // System.out.println(key.toString() + " - trend -" + entry.getValue() * (Math.log(1 +
      // totalPageViews)));

      BasicBSONObject pageTitleData =
          new BasicBSONObject(formatterOut.format(entry.getKey()), boVal);
      BasicBSONObject pageTitleDataUpdate = new BasicBSONObject("$push", pageTitleData);

      reduceResult.setQuery(query);
      reduceResult.setModifiers(pageTitleDataUpdate);
      reduceResult.setUpsert(true);
      context.write(null, reduceResult);
    }
  }
Esempio n. 7
0
 /**
  * 这里获取全部数据
  *
  * @param field
  * @return
  */
 public Release fill(Fields field) {
   Release tb = null;
   if (null != field) {
     tb = new Release();
     tb.setId(field.getString("_id"));
     if (null != field.get(Meta.STATUS)) tb.setStatus(field.getInt(Meta.STATUS));
     if (null != field.get(Meta.TID)) tb.setTid(field.getString(Meta.TID));
     if (null != field.get(Meta.NAME)) tb.setName(field.getString(Meta.NAME));
     if (null != field.get(Meta.OWNER)) tb.setOwner(field.getString(Meta.OWNER));
     if (null != field.get(Meta.VERSION)) tb.setVersion(field.getString(Meta.VERSION));
     if (null != field.get(Meta.URL)) tb.setUrl(field.getString(Meta.URL));
     if (null != field.get(Meta.CRTAT)) tb.setCrtAt(field.getLong(Meta.CRTAT));
     if (null != field.get(Meta.UPDAT)) tb.setUpdAt(field.getLong(Meta.UPDAT));
     if (null != field.get(Meta.LEVEL)) tb.setLevel(field.getInt(Meta.LEVEL));
     if (null != field.get(Meta.COPYRIGHT)) tb.setCopyright(field.getInt(Meta.COPYRIGHT));
     if (null != field.get(Meta.OPERATOR)) tb.setOperator(field.getString(Meta.OPERATOR));
     if (null != field.get(Meta.RELEASEEXPLAIN))
       tb.setReleaseExplain(field.getString(Meta.RELEASEEXPLAIN));
     if (null != field.get(Meta.UPDATEEXPLAIN))
       tb.setUpdateExplain(field.getString(Meta.UPDATEEXPLAIN));
     List<Server> listCover = new ArrayList<Server>();
     if (null != field.get(Meta.SERVER)) {
       BasicBSONList listC = (BasicBSONList) field.get(Meta.SERVER);
       for (int j = 0; listC.size() > j; j++) {
         BasicBSONObject bd = (BasicBSONObject) listC.get(j);
         Server s = new Server();
         s.setAddr(bd.getString(Meta.ADDR));
         s.setContent(bd.getString(Meta.CONTENT));
         s.setDownAt(bd.getLong(Meta.DOWNAT));
         s.setId(bd.getString(Meta.ID));
         s.setName(bd.getString(Meta.NAME));
         s.setNo(bd.getString(Meta.NO));
         s.setReleaseAt(bd.getLong(Meta.RELEASEAT));
         s.setReleaseOwner(bd.getString(Meta.RELEASEOWNER));
         s.setStauts(bd.getInt(Meta.STATUS));
         s.setSummary(bd.getString(Meta.SUMMARY));
         listCover.add(s);
       }
       tb.setServer(listCover);
     }
   }
   return tb;
 }