public void reduce(Record key, Iterator<Record> values, TaskContext context) throws IOException {
   long c = 0;
   while (values.hasNext()) {
     Record val = values.next();
     c += val.getBigint(0);
   }
   count.set(0, c);
   context.write(key, count);
 }
 public void reduce(Record key, Iterator<Record> values, TaskContext context) throws IOException {
   long count = 0;
   while (values.hasNext()) {
     Record val = values.next();
     count += val.getBigint(0);
   }
   result.set(0, key.get(0));
   result.set(1, count);
   context.write(result);
 }
示例#3
0
  public void map(long recordNum, Record record, TaskContext context) throws IOException {
    key.set("user_id", record.getString(0));
    long type = record.getBigint(3);
    long date = record.getBigint(5);

    //    	  user_item last 1 day buy times
    if (type == 4 && date == 30) {
      value.set("ui_l1_buy", 1);
    } else {
      value.set("ui_l1_buy", 0);
    }

    //    	  user_item last 3 days click times
    if (type == 1 && (date >= 28 && date <= 30)) {
      value.set("ui_l3_clk", 1);
    } else {
      value.set("ui_l3_clk", 0);
    }

    //    	  user_item last 5 days action times
    if (date >= 26 && date <= 30) {
      value.set("ui_l5_act", 1);
    } else {
      value.set("ui_l5_act", 0);
    }

    context.write(key, value);
  }
示例#4
0
    /** Writes all keys and values directly to output. */
    @Override
    public void reduce(Record key, Iterator<Record> values, TaskContext context)
        throws IOException {
      result.set(0, key.get(0));

      while (values.hasNext()) {
        Record val = values.next();
        result.set(1, val.get(0));
        context.write(result);
      }
    }
 @Override
 public void reduce(Record key, Iterator<Record> values, TaskContext context) throws IOException {
   HashMap<Long, ArrayList<String>> item_date = new HashMap<Long, ArrayList<String>>();
   while (values.hasNext()) {
     Record val = values.next();
     String[] item_info = val.get(0).toString().split(":");
     long item_id = Long.parseLong(item_info[0]);
     String date_time = item_info[1];
     if (!item_date.containsKey(item_id)) {
       ArrayList<String> date_list = new ArrayList<String>();
       date_list.add(date_time);
       item_date.put(item_id, date_list);
     } else {
       ArrayList<String> date_list = item_date.get(item_id);
       date_list.add(date_time);
       item_date.put(item_id, date_list);
     }
   }
   // load item_ids into list
   Long[] item_id_list = new Long[item_date.size()];
   Iterator<Long> item_id_iter = item_date.keySet().iterator();
   int count = 0;
   while (item_id_iter.hasNext()) {
     item_id_list[count] = item_id_iter.next();
     count++;
   }
   // calculate time gap
   for (int ii = 0; ii < item_id_list.length; ii++) {
     for (int jj = ii + 1; jj < item_id_list.length; jj++) {
       long item_id_ii = item_id_list[ii];
       long item_id_jj = item_id_list[jj];
       ArrayList<String> date_list_ii = item_date.get(item_id_ii);
       ArrayList<String> date_list_jj = item_date.get(item_id_jj);
       long min_gap = 100000000l;
       Iterator<String> date_iter_ii = date_list_ii.iterator();
       while (date_iter_ii.hasNext()) {
         String date_ii = date_iter_ii.next();
         Iterator<String> date_iter_jj = date_list_jj.iterator();
         while (date_iter_jj.hasNext()) {
           String date_jj = date_iter_jj.next();
           long time_gap = get_time_gap(date_ii, date_jj);
           if (time_gap < min_gap) {
             min_gap = time_gap;
           }
         }
       }
       result.set(0, item_id_ii);
       result.set(1, item_id_jj);
       result.set(2, min_gap);
       context.write(result);
     }
   }
 }
示例#6
0
  public void map(long recordNum, Record record, TaskContext context) throws IOException {

    if (record.getString(4).equals("cart")) return;

    key.set("session", record.getBigint(0));
    key.set("user_id", record.getString(1));

    value.set("item_id", record.getBigint(2).toString());
    value.set("bundle", record.getString(3));
    value.set("action", record.getString(4));
    value.set("vtime", record.getString(5));

    context.write(key, value);
  }
 private Record getOdpsRecord(
     TableTunnel.UploadSession uploadSession, int idx, ColumnGenerator... extraCols)
     throws ParseException {
   Record record = uploadSession.newRecord();
   record.setBigint("id", (long) idx);
   record.setString("msg", getMsgPrefix() + idx);
   int colNum = 0;
   for (ColumnGenerator generator : extraCols) {
     String field = forIdx(colNum++);
     String fieldValue = generator.getExportText(idx);
     switch (generator.getOdpsType()) {
       case STRING:
         record.setString(field, fieldValue);
         break;
       case BIGINT:
         record.setBigint(field, Long.parseLong(fieldValue));
         break;
       case DATETIME:
         String dateFormat = generator.getDateFormat();
         record.setDatetime(field, new SimpleDateFormat(dateFormat).parse(fieldValue));
         break;
       case DOUBLE:
         record.setDouble(field, Double.parseDouble(fieldValue));
         break;
       case DECIMAL:
         record.setDecimal(field, new BigDecimal(fieldValue));
         break;
       default:
         throw new RuntimeException("Unknown column type: " + generator.getOdpsType());
     }
   }
   return record;
 }
  @Override
  public void map(long recordNum, Record record, TaskContext context) throws IOException {
    key.set("user_id", record.getString(0));
    key.set("brand_id", record.getString(1));

    value.set("type", record.getString(2));
    value.set("visit_datetime", record.getString(3));

    context.write(key, value); // hashmap ?
  }
示例#9
0
 // transfer_num:double,max_transfer_num:double,min_transfer_num:double,
 // comment_num:double,max_comment_num:double,min_comment_num:double,
 // praise_num:double,max_praise_num:double,min_praise_num:double</
 public void map(long recordNum, Record record, TaskContext context) throws IOException {
   String uid = record.getString(0);
   String mid = record.getString(1);
   mapkey.set(0, uid);
   mapkey.set(1, mid);
   // double max_transfer_num = 0,min_transfer_num=0;
   // double max_comment_num = 0,min_comment_num=0;
   // double max_praise_num = 0,min_praise_num=0;
   double transfer_num = record.get(3).toString().matches("1") ? 1 : 0;
   double comment_num = record.get(4).toString().matches("1") ? 1 : 0;
   double praise_num = record.get(5).toString().matches("1") ? 1 : 0;
   mapvalue.set(0, transfer_num);
   // mapvalue.set(1,max_transfer_num);
   // mapvalue.set(2,min_transfer_num);
   mapvalue.set(1, comment_num);
   // mapvalue.set(4,max_comment_num);
   // mapvalue.set(5,min_comment_num);
   mapvalue.set(2, praise_num);
   // mapvalue.set(7,max_praise_num);
   // mapvalue.set(8,min_praise_num);
   context.write(mapkey, mapvalue);
 }
示例#10
0
 @Override
 public void map(long recordNum, Record record, TaskContext context) throws IOException {
   key.set(new Object[] {(Long) record.get(0)});
   value.set(new Object[] {(Long) record.get(1)});
   context.write(key, value);
 }
示例#11
0
  public void reduce(Record key, Iterator<Record> values, TaskContext context) throws IOException {

    Map<String, Integer> frqCountMap = new HashMap<String, Integer>();
    String itemBought = "";
    List<Object[]> rList = new ArrayList<Object[]>();
    while (values.hasNext()) {
      Record val = values.next();
      String itemTemp = val.getString("item");
      if (val.getString("action").equals("alipay")) {
        itemBought = itemTemp;
        continue;
      }

      if (frqCountMap.containsKey(itemTemp)) {
        frqCountMap.put(itemTemp, frqCountMap.get(itemTemp));
      } else {
        frqCountMap.put(itemTemp, 1);
      }

      rList.add(val.toArray().clone());
    }

    int sessionLen = rList.size();
    int itemNum = frqCountMap.keySet().size();

    if (sessionLen < 4 || itemNum < 2) return;

    Collections.sort(
        rList,
        new Comparator<Object[]>() {
          public int compare(Object[] r1, Object[] r2) {
            return r2[2].toString().compareTo(r1[2].toString());
          }
        });

    if (frqCountMap.containsKey(itemBought) == false) {
      output.set(0, key.getBigint("session"));
      output.set(1, key.getString("user_id"));

      output.set(2, 0.99);
      output.set(3, 0.99);
      context.write(output);
      return;
    }

    output.set(0, key.getBigint("session"));
    output.set(1, key.getString("user_id"));

    int i = 0;
    while (i < sessionLen) {
      if (rList.get(i)[0].toString().equals(itemBought)) {
        output.set(2, i / (double) sessionLen);
        break;
      }
      i++;
    }

    List<Map.Entry<String, Integer>> frqList =
        new ArrayList<Map.Entry<String, Integer>>(frqCountMap.entrySet());

    Collections.sort(
        frqList,
        new Comparator<Map.Entry<String, Integer>>() {
          public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) {
            return o2.getValue().compareTo(o1.getValue());
          }
        });

    i = 0;
    while (i < itemNum) {
      if (rList.get(i)[0].toString().equals(itemBought)) {
        output.set(3, i / (double) itemNum);
        break;
      }
      i++;
    }

    context.write(output);
  }
示例#12
0
 public void map(long recordNum, Record record, TaskContext context) throws IOException {
   String w = record.getString(0);
   word.setString(0, w);
   context.write(word, one);
 }
示例#13
0
 public void setup(TaskContext context) throws IOException {
   word = context.createMapOutputKeyRecord();
   one = context.createMapOutputValueRecord();
   one.setBigint(0, 1L);
 }