public static void main(String[] args) throws IOException {
Text text = new Text("\u0041");
ObjectWritable writable = new ObjectWritable(text);
System.out.println(StringUtils.byteToHexString(serialize(writable)));
// 00196f72672e6170616368652e6861646f6f702e696f2e5465787400196f72672e6170616368652e6861646f6f702e696f2e546578740141
// (a)0019 6f72672e6170616368652e6861646f6f702e696f2e54657874， (b)0019
// 6f72672e6170616368652e6861646f6f702e696f2e54657874，(c)0141
/*
(1)序列化 ObjectWritable 的声明部分
UTF8.writeString(out, declaredClass.getName()); ==>
0019 6f72672e6170616368652e6861646f6f702e696f2e54657874(第一部分是一个short数值，为该对象class名字的字符串长度，org.apache.hadoop.io.Text，25位=0x0019)
(2)序列化 Writable 接口对象的实现类
if (Writable.class.isAssignableFrom(declaredClass)) { // Writable接口实现类
UTF8.writeString(out, instance.getClass().getName());
((Writable)instance).write(out);
} ==>
0019 6f72672e6170616368652e6861646f6f702e696f2e54657874
0141(可变长Text的序列化值，0x01长度，0x41数值内容)
*/
ObjectWritable srcWritable = new ObjectWritable(Integer.TYPE, 188);
ObjectWritable destWritable = new ObjectWritable();
cloneInto(srcWritable, destWritable);
System.out.println(serializeToHexString(srcWritable)); // 0003696e74000000bc
System.out.println((Integer) destWritable.get()); // 188
}
public void write(DataOutput out) throws IOException {
ObjectWritable.writeStringCached(out, methodName);
out.writeInt(parameterClasses.length);
for (int i = 0; i < parameterClasses.length; i++) {
ObjectWritable.writeObject(out, parameters[i], parameterClasses[i], conf);
}
}
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
final boolean logDebug = LOG.isDebugEnabled();
long startTime = 0;
if (logDebug) {
startTime = System.currentTimeMillis();
}
ObjectWritable value = null;
try {
value =
(ObjectWritable)
client.call(
new Invocation(method, args), getAddress(), protocol, ticket, rpcTimeout);
} catch (RemoteException re) {
throw re;
} catch (ConnectException ce) {
needCheckDnsUpdate = true;
throw ce;
} catch (NoRouteToHostException nrhe) {
needCheckDnsUpdate = true;
throw nrhe;
} catch (PortUnreachableException pue) {
needCheckDnsUpdate = true;
throw pue;
} catch (UnknownHostException uhe) {
needCheckDnsUpdate = true;
throw uhe;
}
if (logDebug) {
long callTime = System.currentTimeMillis() - startTime;
LOG.debug("Call: " + method.getName() + " " + callTime);
}
return value.get();
}
public void readFields(DataInput in) throws IOException {
methodName = UTF8.readString(in);
parameters = new Object[in.readInt()];
parameterClasses = new Class[parameters.length];
ObjectWritable objectWritable = new ObjectWritable();
for (int i = 0; i < parameters.length; i++) {
parameters[i] = ObjectWritable.readObject(in, objectWritable, this.conf);
parameterClasses[i] = objectWritable.getDeclaredClass();
}
}
@Override
public Object deserialize(Writable w) throws SerDeException {
ObjectWritable obj = (ObjectWritable) w;
Packet packet = (Packet) obj.get();
for (int i = 0; i < numColumns; i++) {
String columName = columnNames.get(i);
Object value = packet.get(columName);
row.set(i, value);
}
return row;
}
@Override
public void map(Text key, ObjectWritable value, Context context)
throws IOException, InterruptedException {
System.out.println("map");
// key == site_url
// //하나의 터치데이터 묶음
// String str = value.toString();
//
// //키값 추출 (url, device)
// int chk=0;
// int i;
// for(i=0;i<str.length();i++){
// if(str.charAt(i) == ','){
// chk++;
// if(chk == 2)
// break;
// }
// }
//
// System.out.println(" - map : " + str);
//
// PageData page = new PageData();
// System.out.println(key.toString() + " direct");
PathData ptd = null;
if (value.getDeclaredClass() == PathData.class) {
// System.out.println("pagedata");
ptd = (PathData) value.get();
System.out.println("key = " + key.toString());
System.out.println("url = " + ptd.site_url);
}
// else
// System.out.println("not pagedata");
// PageData pd = (PageData) value.get();
// if(pd == null){
// System.out.println("null?!");
// }
//
// System.out.println(pd.country + " " + pd.date);
// PageData pp = new PageData();
// System.out.println("map");
// reduce로 넘기기
context.write(new Text(ptd.site_url), new ObjectWritable(ptd));
// output.collect(value, new Text ("v" + value.toString()));
}
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
long startTime = 0;
if (LOG.isDebugEnabled()) {
startTime = Time.now();
}
ObjectWritable value =
(ObjectWritable)
client.call(RPC.RpcKind.RPC_WRITABLE, new Invocation(method, args), remoteId);
if (LOG.isDebugEnabled()) {
long callTime = Time.now() - startTime;
LOG.debug("Call: " + method.getName() + " " + callTime);
}
return value.get();
}
@SuppressWarnings("deprecation")
public void readFields(DataInput in) throws IOException {
rpcVersion = in.readLong();
declaringClassProtocolName = UTF8.readString(in);
methodName = UTF8.readString(in);
clientVersion = in.readLong();
clientMethodsHash = in.readInt();
parameters = new Object[in.readInt()];
parameterClasses = new Class[parameters.length];
ObjectWritable objectWritable = new ObjectWritable();
for (int i = 0; i < parameters.length; i++) {
parameters[i] = ObjectWritable.readObject(in, objectWritable, this.conf);
parameterClasses[i] = objectWritable.getDeclaredClass();
}
}
@SuppressWarnings("deprecation")
public void write(DataOutput out) throws IOException {
out.writeLong(rpcVersion);
UTF8.writeString(out, declaringClassProtocolName);
UTF8.writeString(out, methodName);
out.writeLong(clientVersion);
out.writeInt(clientMethodsHash);
out.writeInt(parameterClasses.length);
for (int i = 0; i < parameterClasses.length; i++) {
ObjectWritable.writeObject(out, parameters[i], parameterClasses[i], conf, true);
}
}
@Override
public void map(LongWritable key, ObjectWritable obj, Context context)
throws IOException, InterruptedException {
if (setupFailedException != null) {
throw new IOException(setupFailedException);
}
DoubleWritable classI = new DoubleWritable();
DoubleWritable value = new DoubleWritable();
lastTime = System.currentTimeMillis();
Instance instance;
instance = (Instance) obj.get();
// remove all hosts whose DF is below the threshold
if (hostsWithMinimumDF != null) {
instance.setAcceptableIndices(hostsWithMinimumDF.keySet());
}
// loop through training instances
for (Instance trainingInstance : trainingInstances) {
try {
float jaccardValue = jaccardValue(trainingInstance, instance);
int trainingClassId = classIndex.getIndexPosition(trainingInstance.getClassLabel());
classI.set(trainingClassId);
value.set(jaccardValue);
// store it in an array with the classIndex
array.set(new DoubleWritable[] {classI, value});
// and hand it to the reducer
context.write(new Text(instance.getId()), array);
} catch (Exception e) {
e.printStackTrace();
LOG.error("map failed with exception");
throw new IOException(e);
}
}
// count the number of instances per class
// context.write(new Text(Util.INSTANCES_PER_CLASS_PATH + " " +
// instance.getClassLabel()), ONE);
counter++;
long timeTaken = System.currentTimeMillis() - lastTime;
totalTime += timeTaken;
if ((counter % 10) == 0) {
// print out some performance stuff
LOG.info(
"instance "
+ counter
+ " duration: "
+ ((double) timeTaken / 1000)
+ " s - avg : "
+ ((double) (totalTime / counter) / 1000)
+ " s"
+ " num_values: "
+ instance.getNumValues());
}
double duration = ((double) timeTaken / 1000);
if (duration > REPORT_SLOW_INSTANCE_THRESHOLD) {
LOG.info(
"Mapped a particularly SLOW INSTANCE. classLabel: "
+ instance.getClassLabel()
+ ", "
+ "duration: "
+ duration
+ " s ("
+ duration / 60
+ " min),"
+ " num_values: "
+ instance.getNumValues());
}
}
/** Read a {@link Writable}, {@link String}, primitive type, or an array of the preceding. */
@SuppressWarnings("unchecked")
public static Object readObject(DataInput in, ObjectWritable objectWritable, Configuration conf)
throws IOException {
String className = UTF8.readString(in);
Class<?> declaredClass = PRIMITIVE_NAMES.get(className);
if (declaredClass == null) {
declaredClass = loadClass(conf, className);
}
Object instance;
if (declaredClass.isPrimitive()) { // primitive types
if (declaredClass == Boolean.TYPE) { // boolean
instance = Boolean.valueOf(in.readBoolean());
} else if (declaredClass == Character.TYPE) { // char
instance = Character.valueOf(in.readChar());
} else if (declaredClass == Byte.TYPE) { // byte
instance = Byte.valueOf(in.readByte());
} else if (declaredClass == Short.TYPE) { // short
instance = Short.valueOf(in.readShort());
} else if (declaredClass == Integer.TYPE) { // int
instance = Integer.valueOf(in.readInt());
} else if (declaredClass == Long.TYPE) { // long
instance = Long.valueOf(in.readLong());
} else if (declaredClass == Float.TYPE) { // float
instance = Float.valueOf(in.readFloat());
} else if (declaredClass == Double.TYPE) { // double
instance = Double.valueOf(in.readDouble());
} else if (declaredClass == Void.TYPE) { // void
instance = null;
} else {
throw new IllegalArgumentException("Not a primitive: " + declaredClass);
}
} else if (declaredClass.isArray()) { // array
int length = in.readInt();
instance = Array.newInstance(declaredClass.getComponentType(), length);
for (int i = 0; i < length; i++) {
Array.set(instance, i, readObject(in, conf));
}
} else if (declaredClass == String.class) { // String
instance = UTF8.readString(in);
} else if (declaredClass.isEnum()) { // enum
instance = Enum.valueOf((Class<? extends Enum>) declaredClass, UTF8.readString(in));
} else { // Writable
Class instanceClass = null;
String str = UTF8.readString(in);
instanceClass = loadClass(conf, str);
Writable writable = WritableFactories.newInstance(instanceClass, conf);
writable.readFields(in);
instance = writable;
if (instanceClass == NullInstance.class) { // null
declaredClass = ((NullInstance) instance).declaredClass;
instance = null;
}
}
if (objectWritable != null) { // store values
objectWritable.declaredClass = declaredClass;
objectWritable.instance = instance;
}
return instance;
}
