public static void main(String[] args) throws IOException {

    Text text = new Text("\u0041");
    ObjectWritable writable = new ObjectWritable(text);
    System.out.println(StringUtils.byteToHexString(serialize(writable)));
    // 00196f72672e6170616368652e6861646f6f702e696f2e5465787400196f72672e6170616368652e6861646f6f702e696f2e546578740141
    // (a)0019 6f72672e6170616368652e6861646f6f702e696f2e54657874, (b)0019
    // 6f72672e6170616368652e6861646f6f702e696f2e54657874,(c)0141
    /*
           (1)序列化  ObjectWritable 的声明部分
              UTF8.writeString(out, declaredClass.getName());  ==>

        0019 6f72672e6170616368652e6861646f6f702e696f2e54657874(第一部分是一个short数值,为该对象class名字的字符串长度,org.apache.hadoop.io.Text,25位=0x0019)
     (2)序列化 Writable 接口对象的实现类
        if (Writable.class.isAssignableFrom(declaredClass)) { // Writable接口实现类
                 UTF8.writeString(out, instance.getClass().getName());
                 ((Writable)instance).write(out);
              }                                                ==>

              0019 6f72672e6170616368652e6861646f6f702e696f2e54657874
              0141(可变长Text的序列化值,0x01长度,0x41数值内容)
    */

    ObjectWritable srcWritable = new ObjectWritable(Integer.TYPE, 188);
    ObjectWritable destWritable = new ObjectWritable();
    cloneInto(srcWritable, destWritable);
    System.out.println(serializeToHexString(srcWritable)); // 0003696e74000000bc
    System.out.println((Integer) destWritable.get()); // 188
  }
Beispiel #2
0
 public void write(DataOutput out) throws IOException {
   ObjectWritable.writeStringCached(out, methodName);
   out.writeInt(parameterClasses.length);
   for (int i = 0; i < parameterClasses.length; i++) {
     ObjectWritable.writeObject(out, parameters[i], parameterClasses[i], conf);
   }
 }
Beispiel #3
0
    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
      final boolean logDebug = LOG.isDebugEnabled();
      long startTime = 0;
      if (logDebug) {
        startTime = System.currentTimeMillis();
      }

      ObjectWritable value = null;
      try {
        value =
            (ObjectWritable)
                client.call(
                    new Invocation(method, args), getAddress(), protocol, ticket, rpcTimeout);
      } catch (RemoteException re) {
        throw re;
      } catch (ConnectException ce) {
        needCheckDnsUpdate = true;
        throw ce;
      } catch (NoRouteToHostException nrhe) {
        needCheckDnsUpdate = true;
        throw nrhe;
      } catch (PortUnreachableException pue) {
        needCheckDnsUpdate = true;
        throw pue;
      } catch (UnknownHostException uhe) {
        needCheckDnsUpdate = true;
        throw uhe;
      }
      if (logDebug) {
        long callTime = System.currentTimeMillis() - startTime;
        LOG.debug("Call: " + method.getName() + " " + callTime);
      }
      return value.get();
    }
Beispiel #4
0
 public void readFields(DataInput in) throws IOException {
   methodName = UTF8.readString(in);
   parameters = new Object[in.readInt()];
   parameterClasses = new Class[parameters.length];
   ObjectWritable objectWritable = new ObjectWritable();
   for (int i = 0; i < parameters.length; i++) {
     parameters[i] = ObjectWritable.readObject(in, objectWritable, this.conf);
     parameterClasses[i] = objectWritable.getDeclaredClass();
   }
 }
Beispiel #5
0
  @Override
  public Object deserialize(Writable w) throws SerDeException {
    ObjectWritable obj = (ObjectWritable) w;
    Packet packet = (Packet) obj.get();

    for (int i = 0; i < numColumns; i++) {
      String columName = columnNames.get(i);
      Object value = packet.get(columName);
      row.set(i, value);
    }
    return row;
  }
  @Override
  public void map(Text key, ObjectWritable value, Context context)
      throws IOException, InterruptedException {

    System.out.println("map");

    // key == site_url

    //		//하나의 터치데이터 묶음
    //		String str = value.toString();
    //
    //		//키값 추출 (url, device)
    //		int chk=0;
    //		int i;
    //		for(i=0;i<str.length();i++){
    //			if(str.charAt(i) == ','){
    //				chk++;
    //				if(chk == 2)
    //					break;
    //			}
    //		}
    //
    //		System.out.println("		-	map : " + str);
    //
    //		PageData page = new PageData();

    //		System.out.println(key.toString() + " direct");

    PathData ptd = null;
    if (value.getDeclaredClass() == PathData.class) {
      //			System.out.println("pagedata");
      ptd = (PathData) value.get();
      System.out.println("key = " + key.toString());
      System.out.println("url = " + ptd.site_url);
    }
    //		else
    //			System.out.println("not pagedata");

    //		PageData pd = (PageData) value.get();
    //		if(pd == null){
    //			System.out.println("null?!");
    //		}
    //
    //		System.out.println(pd.country + "  " + pd.date);

    //	PageData pp = new PageData();
    // System.out.println("map");
    // reduce로 넘기기
    context.write(new Text(ptd.site_url), new ObjectWritable(ptd));

    // output.collect(value, new Text ("v" + value.toString()));

  }
    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
      long startTime = 0;
      if (LOG.isDebugEnabled()) {
        startTime = Time.now();
      }

      ObjectWritable value =
          (ObjectWritable)
              client.call(RPC.RpcKind.RPC_WRITABLE, new Invocation(method, args), remoteId);
      if (LOG.isDebugEnabled()) {
        long callTime = Time.now() - startTime;
        LOG.debug("Call: " + method.getName() + " " + callTime);
      }
      return value.get();
    }
 @SuppressWarnings("deprecation")
 public void readFields(DataInput in) throws IOException {
   rpcVersion = in.readLong();
   declaringClassProtocolName = UTF8.readString(in);
   methodName = UTF8.readString(in);
   clientVersion = in.readLong();
   clientMethodsHash = in.readInt();
   parameters = new Object[in.readInt()];
   parameterClasses = new Class[parameters.length];
   ObjectWritable objectWritable = new ObjectWritable();
   for (int i = 0; i < parameters.length; i++) {
     parameters[i] = ObjectWritable.readObject(in, objectWritable, this.conf);
     parameterClasses[i] = objectWritable.getDeclaredClass();
   }
 }
 @SuppressWarnings("deprecation")
 public void write(DataOutput out) throws IOException {
   out.writeLong(rpcVersion);
   UTF8.writeString(out, declaringClassProtocolName);
   UTF8.writeString(out, methodName);
   out.writeLong(clientVersion);
   out.writeInt(clientMethodsHash);
   out.writeInt(parameterClasses.length);
   for (int i = 0; i < parameterClasses.length; i++) {
     ObjectWritable.writeObject(out, parameters[i], parameterClasses[i], conf, true);
   }
 }
  @Override
  public void map(LongWritable key, ObjectWritable obj, Context context)
      throws IOException, InterruptedException {

    if (setupFailedException != null) {
      throw new IOException(setupFailedException);
    }

    DoubleWritable classI = new DoubleWritable();
    DoubleWritable value = new DoubleWritable();

    lastTime = System.currentTimeMillis();

    Instance instance;

    instance = (Instance) obj.get();

    // remove all hosts whose DF is below the threshold
    if (hostsWithMinimumDF != null) {
      instance.setAcceptableIndices(hostsWithMinimumDF.keySet());
    }

    // loop through training instances
    for (Instance trainingInstance : trainingInstances) {
      try {
        float jaccardValue = jaccardValue(trainingInstance, instance);

        int trainingClassId = classIndex.getIndexPosition(trainingInstance.getClassLabel());

        classI.set(trainingClassId);
        value.set(jaccardValue);
        // store it in an array with the classIndex
        array.set(new DoubleWritable[] {classI, value});

        // and hand it to the reducer
        context.write(new Text(instance.getId()), array);
      } catch (Exception e) {
        e.printStackTrace();
        LOG.error("map failed with exception");
        throw new IOException(e);
      }
    }

    // count the number of instances per class
    // context.write(new Text(Util.INSTANCES_PER_CLASS_PATH + " " +
    // instance.getClassLabel()), ONE);

    counter++;

    long timeTaken = System.currentTimeMillis() - lastTime;
    totalTime += timeTaken;

    if ((counter % 10) == 0) {
      // print out some performance stuff
      LOG.info(
          "instance "
              + counter
              + " duration: "
              + ((double) timeTaken / 1000)
              + " s - avg : "
              + ((double) (totalTime / counter) / 1000)
              + " s"
              + " num_values: "
              + instance.getNumValues());
    }

    double duration = ((double) timeTaken / 1000);
    if (duration > REPORT_SLOW_INSTANCE_THRESHOLD) {
      LOG.info(
          "Mapped a particularly SLOW INSTANCE. classLabel: "
              + instance.getClassLabel()
              + ", "
              + "duration: "
              + duration
              + " s ("
              + duration / 60
              + " min),"
              + " num_values: "
              + instance.getNumValues());
    }
  }
Beispiel #11
0
  /** Read a {@link Writable}, {@link String}, primitive type, or an array of the preceding. */
  @SuppressWarnings("unchecked")
  public static Object readObject(DataInput in, ObjectWritable objectWritable, Configuration conf)
      throws IOException {
    String className = UTF8.readString(in);
    Class<?> declaredClass = PRIMITIVE_NAMES.get(className);
    if (declaredClass == null) {
      declaredClass = loadClass(conf, className);
    }

    Object instance;

    if (declaredClass.isPrimitive()) { // primitive types

      if (declaredClass == Boolean.TYPE) { // boolean
        instance = Boolean.valueOf(in.readBoolean());
      } else if (declaredClass == Character.TYPE) { // char
        instance = Character.valueOf(in.readChar());
      } else if (declaredClass == Byte.TYPE) { // byte
        instance = Byte.valueOf(in.readByte());
      } else if (declaredClass == Short.TYPE) { // short
        instance = Short.valueOf(in.readShort());
      } else if (declaredClass == Integer.TYPE) { // int
        instance = Integer.valueOf(in.readInt());
      } else if (declaredClass == Long.TYPE) { // long
        instance = Long.valueOf(in.readLong());
      } else if (declaredClass == Float.TYPE) { // float
        instance = Float.valueOf(in.readFloat());
      } else if (declaredClass == Double.TYPE) { // double
        instance = Double.valueOf(in.readDouble());
      } else if (declaredClass == Void.TYPE) { // void
        instance = null;
      } else {
        throw new IllegalArgumentException("Not a primitive: " + declaredClass);
      }

    } else if (declaredClass.isArray()) { // array
      int length = in.readInt();
      instance = Array.newInstance(declaredClass.getComponentType(), length);
      for (int i = 0; i < length; i++) {
        Array.set(instance, i, readObject(in, conf));
      }

    } else if (declaredClass == String.class) { // String
      instance = UTF8.readString(in);
    } else if (declaredClass.isEnum()) { // enum
      instance = Enum.valueOf((Class<? extends Enum>) declaredClass, UTF8.readString(in));
    } else { // Writable
      Class instanceClass = null;
      String str = UTF8.readString(in);
      instanceClass = loadClass(conf, str);

      Writable writable = WritableFactories.newInstance(instanceClass, conf);
      writable.readFields(in);
      instance = writable;

      if (instanceClass == NullInstance.class) { // null
        declaredClass = ((NullInstance) instance).declaredClass;
        instance = null;
      }
    }

    if (objectWritable != null) { // store values
      objectWritable.declaredClass = declaredClass;
      objectWritable.instance = instance;
    }

    return instance;
  }