public static void main(String[] args) throws IOException {
Text text = new Text("\u0041");
ObjectWritable writable = new ObjectWritable(text);
System.out.println(StringUtils.byteToHexString(serialize(writable)));
// 00196f72672e6170616368652e6861646f6f702e696f2e5465787400196f72672e6170616368652e6861646f6f702e696f2e546578740141
// (a)0019 6f72672e6170616368652e6861646f6f702e696f2e54657874， (b)0019
// 6f72672e6170616368652e6861646f6f702e696f2e54657874，(c)0141
/*
(1)序列化 ObjectWritable 的声明部分
UTF8.writeString(out, declaredClass.getName()); ==>
0019 6f72672e6170616368652e6861646f6f702e696f2e54657874(第一部分是一个short数值，为该对象class名字的字符串长度，org.apache.hadoop.io.Text，25位=0x0019)
(2)序列化 Writable 接口对象的实现类
if (Writable.class.isAssignableFrom(declaredClass)) { // Writable接口实现类
UTF8.writeString(out, instance.getClass().getName());
((Writable)instance).write(out);
} ==>
0019 6f72672e6170616368652e6861646f6f702e696f2e54657874
0141(可变长Text的序列化值，0x01长度，0x41数值内容)
*/
ObjectWritable srcWritable = new ObjectWritable(Integer.TYPE, 188);
ObjectWritable destWritable = new ObjectWritable();
cloneInto(srcWritable, destWritable);
System.out.println(serializeToHexString(srcWritable)); // 0003696e74000000bc
System.out.println((Integer) destWritable.get()); // 188
}
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
final boolean logDebug = LOG.isDebugEnabled();
long startTime = 0;
if (logDebug) {
startTime = System.currentTimeMillis();
}
ObjectWritable value = null;
try {
value =
(ObjectWritable)
client.call(
new Invocation(method, args), getAddress(), protocol, ticket, rpcTimeout);
} catch (RemoteException re) {
throw re;
} catch (ConnectException ce) {
needCheckDnsUpdate = true;
throw ce;
} catch (NoRouteToHostException nrhe) {
needCheckDnsUpdate = true;
throw nrhe;
} catch (PortUnreachableException pue) {
needCheckDnsUpdate = true;
throw pue;
} catch (UnknownHostException uhe) {
needCheckDnsUpdate = true;
throw uhe;
}
if (logDebug) {
long callTime = System.currentTimeMillis() - startTime;
LOG.debug("Call: " + method.getName() + " " + callTime);
}
return value.get();
}
@Override
public Object deserialize(Writable w) throws SerDeException {
ObjectWritable obj = (ObjectWritable) w;
Packet packet = (Packet) obj.get();
for (int i = 0; i < numColumns; i++) {
String columName = columnNames.get(i);
Object value = packet.get(columName);
row.set(i, value);
}
return row;
}
@Override
public void map(Text key, ObjectWritable value, Context context)
throws IOException, InterruptedException {
System.out.println("map");
// key == site_url
// //하나의 터치데이터 묶음
// String str = value.toString();
//
// //키값 추출 (url, device)
// int chk=0;
// int i;
// for(i=0;i<str.length();i++){
// if(str.charAt(i) == ','){
// chk++;
// if(chk == 2)
// break;
// }
// }
//
// System.out.println(" - map : " + str);
//
// PageData page = new PageData();
// System.out.println(key.toString() + " direct");
PathData ptd = null;
if (value.getDeclaredClass() == PathData.class) {
// System.out.println("pagedata");
ptd = (PathData) value.get();
System.out.println("key = " + key.toString());
System.out.println("url = " + ptd.site_url);
}
// else
// System.out.println("not pagedata");
// PageData pd = (PageData) value.get();
// if(pd == null){
// System.out.println("null?!");
// }
//
// System.out.println(pd.country + " " + pd.date);
// PageData pp = new PageData();
// System.out.println("map");
// reduce로 넘기기
context.write(new Text(ptd.site_url), new ObjectWritable(ptd));
// output.collect(value, new Text ("v" + value.toString()));
}
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
long startTime = 0;
if (LOG.isDebugEnabled()) {
startTime = Time.now();
}
ObjectWritable value =
(ObjectWritable)
client.call(RPC.RpcKind.RPC_WRITABLE, new Invocation(method, args), remoteId);
if (LOG.isDebugEnabled()) {
long callTime = Time.now() - startTime;
LOG.debug("Call: " + method.getName() + " " + callTime);
}
return value.get();
}
@Override
public void map(LongWritable key, ObjectWritable obj, Context context)
throws IOException, InterruptedException {
if (setupFailedException != null) {
throw new IOException(setupFailedException);
}
DoubleWritable classI = new DoubleWritable();
DoubleWritable value = new DoubleWritable();
lastTime = System.currentTimeMillis();
Instance instance;
instance = (Instance) obj.get();
// remove all hosts whose DF is below the threshold
if (hostsWithMinimumDF != null) {
instance.setAcceptableIndices(hostsWithMinimumDF.keySet());
}
// loop through training instances
for (Instance trainingInstance : trainingInstances) {
try {
float jaccardValue = jaccardValue(trainingInstance, instance);
int trainingClassId = classIndex.getIndexPosition(trainingInstance.getClassLabel());
classI.set(trainingClassId);
value.set(jaccardValue);
// store it in an array with the classIndex
array.set(new DoubleWritable[] {classI, value});
// and hand it to the reducer
context.write(new Text(instance.getId()), array);
} catch (Exception e) {
e.printStackTrace();
LOG.error("map failed with exception");
throw new IOException(e);
}
}
// count the number of instances per class
// context.write(new Text(Util.INSTANCES_PER_CLASS_PATH + " " +
// instance.getClassLabel()), ONE);
counter++;
long timeTaken = System.currentTimeMillis() - lastTime;
totalTime += timeTaken;
if ((counter % 10) == 0) {
// print out some performance stuff
LOG.info(
"instance "
+ counter
+ " duration: "
+ ((double) timeTaken / 1000)
+ " s - avg : "
+ ((double) (totalTime / counter) / 1000)
+ " s"
+ " num_values: "
+ instance.getNumValues());
}
double duration = ((double) timeTaken / 1000);
if (duration > REPORT_SLOW_INSTANCE_THRESHOLD) {
LOG.info(
"Mapped a particularly SLOW INSTANCE. classLabel: "
+ instance.getClassLabel()
+ ", "
+ "duration: "
+ duration
+ " s ("
+ duration / 60
+ " min),"
+ " num_values: "
+ instance.getNumValues());
}
}