public void reduce(Record key, Iterator<Record> values, TaskContext context) throws IOException {
long c = 0;
while (values.hasNext()) {
Record val = values.next();
c += val.getBigint(0);
}
count.set(0, c);
context.write(key, count);
}
public void reduce(Record key, Iterator<Record> values, TaskContext context) throws IOException {
long count = 0;
while (values.hasNext()) {
Record val = values.next();
count += val.getBigint(0);
}
result.set(0, key.get(0));
result.set(1, count);
context.write(result);
}
public void map(long recordNum, Record record, TaskContext context) throws IOException {
key.set("user_id", record.getString(0));
long type = record.getBigint(3);
long date = record.getBigint(5);
// user_item last 1 day buy times
if (type == 4 && date == 30) {
value.set("ui_l1_buy", 1);
} else {
value.set("ui_l1_buy", 0);
}
// user_item last 3 days click times
if (type == 1 && (date >= 28 && date <= 30)) {
value.set("ui_l3_clk", 1);
} else {
value.set("ui_l3_clk", 0);
}
// user_item last 5 days action times
if (date >= 26 && date <= 30) {
value.set("ui_l5_act", 1);
} else {
value.set("ui_l5_act", 0);
}
context.write(key, value);
}
/** Writes all keys and values directly to output. */
@Override
public void reduce(Record key, Iterator<Record> values, TaskContext context)
throws IOException {
result.set(0, key.get(0));
while (values.hasNext()) {
Record val = values.next();
result.set(1, val.get(0));
context.write(result);
}
}
@Override
public void reduce(Record key, Iterator<Record> values, TaskContext context) throws IOException {
HashMap<Long, ArrayList<String>> item_date = new HashMap<Long, ArrayList<String>>();
while (values.hasNext()) {
Record val = values.next();
String[] item_info = val.get(0).toString().split(":");
long item_id = Long.parseLong(item_info[0]);
String date_time = item_info[1];
if (!item_date.containsKey(item_id)) {
ArrayList<String> date_list = new ArrayList<String>();
date_list.add(date_time);
item_date.put(item_id, date_list);
} else {
ArrayList<String> date_list = item_date.get(item_id);
date_list.add(date_time);
item_date.put(item_id, date_list);
}
}
// load item_ids into list
Long[] item_id_list = new Long[item_date.size()];
Iterator<Long> item_id_iter = item_date.keySet().iterator();
int count = 0;
while (item_id_iter.hasNext()) {
item_id_list[count] = item_id_iter.next();
count++;
}
// calculate time gap
for (int ii = 0; ii < item_id_list.length; ii++) {
for (int jj = ii + 1; jj < item_id_list.length; jj++) {
long item_id_ii = item_id_list[ii];
long item_id_jj = item_id_list[jj];
ArrayList<String> date_list_ii = item_date.get(item_id_ii);
ArrayList<String> date_list_jj = item_date.get(item_id_jj);
long min_gap = 100000000l;
Iterator<String> date_iter_ii = date_list_ii.iterator();
while (date_iter_ii.hasNext()) {
String date_ii = date_iter_ii.next();
Iterator<String> date_iter_jj = date_list_jj.iterator();
while (date_iter_jj.hasNext()) {
String date_jj = date_iter_jj.next();
long time_gap = get_time_gap(date_ii, date_jj);
if (time_gap < min_gap) {
min_gap = time_gap;
}
}
}
result.set(0, item_id_ii);
result.set(1, item_id_jj);
result.set(2, min_gap);
context.write(result);
}
}
}
public void map(long recordNum, Record record, TaskContext context) throws IOException {
if (record.getString(4).equals("cart")) return;
key.set("session", record.getBigint(0));
key.set("user_id", record.getString(1));
value.set("item_id", record.getBigint(2).toString());
value.set("bundle", record.getString(3));
value.set("action", record.getString(4));
value.set("vtime", record.getString(5));
context.write(key, value);
}
private Record getOdpsRecord(
TableTunnel.UploadSession uploadSession, int idx, ColumnGenerator... extraCols)
throws ParseException {
Record record = uploadSession.newRecord();
record.setBigint("id", (long) idx);
record.setString("msg", getMsgPrefix() + idx);
int colNum = 0;
for (ColumnGenerator generator : extraCols) {
String field = forIdx(colNum++);
String fieldValue = generator.getExportText(idx);
switch (generator.getOdpsType()) {
case STRING:
record.setString(field, fieldValue);
break;
case BIGINT:
record.setBigint(field, Long.parseLong(fieldValue));
break;
case DATETIME:
String dateFormat = generator.getDateFormat();
record.setDatetime(field, new SimpleDateFormat(dateFormat).parse(fieldValue));
break;
case DOUBLE:
record.setDouble(field, Double.parseDouble(fieldValue));
break;
case DECIMAL:
record.setDecimal(field, new BigDecimal(fieldValue));
break;
default:
throw new RuntimeException("Unknown column type: " + generator.getOdpsType());
}
}
return record;
}
@Override
public void map(long recordNum, Record record, TaskContext context) throws IOException {
key.set("user_id", record.getString(0));
key.set("brand_id", record.getString(1));
value.set("type", record.getString(2));
value.set("visit_datetime", record.getString(3));
context.write(key, value); // hashmap ?
}
// transfer_num:double,max_transfer_num:double,min_transfer_num:double,
// comment_num:double,max_comment_num:double,min_comment_num:double,
// praise_num:double,max_praise_num:double,min_praise_num:double</
public void map(long recordNum, Record record, TaskContext context) throws IOException {
String uid = record.getString(0);
String mid = record.getString(1);
mapkey.set(0, uid);
mapkey.set(1, mid);
// double max_transfer_num = 0,min_transfer_num=0;
// double max_comment_num = 0,min_comment_num=0;
// double max_praise_num = 0,min_praise_num=0;
double transfer_num = record.get(3).toString().matches("1") ? 1 : 0;
double comment_num = record.get(4).toString().matches("1") ? 1 : 0;
double praise_num = record.get(5).toString().matches("1") ? 1 : 0;
mapvalue.set(0, transfer_num);
// mapvalue.set(1,max_transfer_num);
// mapvalue.set(2,min_transfer_num);
mapvalue.set(1, comment_num);
// mapvalue.set(4,max_comment_num);
// mapvalue.set(5,min_comment_num);
mapvalue.set(2, praise_num);
// mapvalue.set(7,max_praise_num);
// mapvalue.set(8,min_praise_num);
context.write(mapkey, mapvalue);
}
@Override
public void map(long recordNum, Record record, TaskContext context) throws IOException {
key.set(new Object[] {(Long) record.get(0)});
value.set(new Object[] {(Long) record.get(1)});
context.write(key, value);
}
public void reduce(Record key, Iterator<Record> values, TaskContext context) throws IOException {
Map<String, Integer> frqCountMap = new HashMap<String, Integer>();
String itemBought = "";
List<Object[]> rList = new ArrayList<Object[]>();
while (values.hasNext()) {
Record val = values.next();
String itemTemp = val.getString("item");
if (val.getString("action").equals("alipay")) {
itemBought = itemTemp;
continue;
}
if (frqCountMap.containsKey(itemTemp)) {
frqCountMap.put(itemTemp, frqCountMap.get(itemTemp));
} else {
frqCountMap.put(itemTemp, 1);
}
rList.add(val.toArray().clone());
}
int sessionLen = rList.size();
int itemNum = frqCountMap.keySet().size();
if (sessionLen < 4 || itemNum < 2) return;
Collections.sort(
rList,
new Comparator<Object[]>() {
public int compare(Object[] r1, Object[] r2) {
return r2[2].toString().compareTo(r1[2].toString());
}
});
if (frqCountMap.containsKey(itemBought) == false) {
output.set(0, key.getBigint("session"));
output.set(1, key.getString("user_id"));
output.set(2, 0.99);
output.set(3, 0.99);
context.write(output);
return;
}
output.set(0, key.getBigint("session"));
output.set(1, key.getString("user_id"));
int i = 0;
while (i < sessionLen) {
if (rList.get(i)[0].toString().equals(itemBought)) {
output.set(2, i / (double) sessionLen);
break;
}
i++;
}
List<Map.Entry<String, Integer>> frqList =
new ArrayList<Map.Entry<String, Integer>>(frqCountMap.entrySet());
Collections.sort(
frqList,
new Comparator<Map.Entry<String, Integer>>() {
public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) {
return o2.getValue().compareTo(o1.getValue());
}
});
i = 0;
while (i < itemNum) {
if (rList.get(i)[0].toString().equals(itemBought)) {
output.set(3, i / (double) itemNum);
break;
}
i++;
}
context.write(output);
}
public void map(long recordNum, Record record, TaskContext context) throws IOException {
String w = record.getString(0);
word.setString(0, w);
context.write(word, one);
}
public void setup(TaskContext context) throws IOException {
word = context.createMapOutputKeyRecord();
one = context.createMapOutputValueRecord();
one.setBigint(0, 1L);
}