Example #1
0
 /**
  * Get the protocol name. If the protocol class has a ProtocolAnnotation, then get the protocol
  * name from the annotation; otherwise the class name is the protocol name.
  */
 public static String getProtocolName(Class<?> protocol) {
   if (protocol == null) {
     return null;
   }
   ProtocolInfo anno = protocol.getAnnotation(ProtocolInfo.class);
   return (anno == null) ? protocol.getName() : anno.protocolName();
 }
    /**
     * Construct an RPC server.
     *
     * @param protocolClass - the protocol being registered can be null for compatibility with old
     *     usage (see below for details)
     * @param protocolImpl the protocol impl that will be called
     * @param conf the configuration to use
     * @param bindAddress the address to bind on to listen for connection
     * @param port the port to listen for connections on
     * @param numHandlers the number of method handler threads to run
     * @param verbose whether each call should be logged
     */
    public Server(
        Class<?> protocolClass,
        Object protocolImpl,
        Configuration conf,
        String bindAddress,
        int port,
        int numHandlers,
        int numReaders,
        int queueSizePerHandler,
        boolean verbose,
        SecretManager<? extends TokenIdentifier> secretManager,
        String portRangeConfig)
        throws IOException {
      super(
          bindAddress,
          port,
          null,
          numHandlers,
          numReaders,
          queueSizePerHandler,
          conf,
          classNameBase(protocolImpl.getClass().getName()),
          secretManager,
          portRangeConfig);

      this.verbose = verbose;

      Class<?>[] protocols;
      if (protocolClass == null) { // derive protocol from impl
        /*
         * In order to remain compatible with the old usage where a single
         * target protocolImpl is suppled for all protocol interfaces, and
         * the protocolImpl is derived from the protocolClass(es)
         * we register all interfaces extended by the protocolImpl
         */
        protocols = RPC.getProtocolInterfaces(protocolImpl.getClass());

      } else {
        if (!protocolClass.isAssignableFrom(protocolImpl.getClass())) {
          throw new IOException(
              "protocolClass "
                  + protocolClass
                  + " is not implemented by protocolImpl which is of class "
                  + protocolImpl.getClass());
        }
        // register protocol class and its super interfaces
        registerProtocolAndImpl(RPC.RpcKind.RPC_WRITABLE, protocolClass, protocolImpl);
        protocols = RPC.getProtocolInterfaces(protocolClass);
      }
      for (Class<?> p : protocols) {
        if (!p.equals(VersionedProtocol.class)) {
          registerProtocolAndImpl(RPC.RpcKind.RPC_WRITABLE, p, protocolImpl);
        }
      }
    }
Example #3
0
  /** Write a {@link Writable}, {@link String}, primitive type, or an array of the preceding. */
  public static void writeObject(
      DataOutput out, Object instance, Class declaredClass, Configuration conf) throws IOException {

    if (instance == null) { // null
      instance = new NullInstance(declaredClass, conf);
      declaredClass = Writable.class;
    }

    UTF8.writeString(out, declaredClass.getName()); // always write declared

    if (declaredClass.isArray()) { // array
      int length = Array.getLength(instance);
      out.writeInt(length);
      for (int i = 0; i < length; i++) {
        writeObject(out, Array.get(instance, i), declaredClass.getComponentType(), conf);
      }

    } else if (declaredClass == String.class) { // String
      UTF8.writeString(out, (String) instance);

    } else if (declaredClass.isPrimitive()) { // primitive type

      if (declaredClass == Boolean.TYPE) { // boolean
        out.writeBoolean(((Boolean) instance).booleanValue());
      } else if (declaredClass == Character.TYPE) { // char
        out.writeChar(((Character) instance).charValue());
      } else if (declaredClass == Byte.TYPE) { // byte
        out.writeByte(((Byte) instance).byteValue());
      } else if (declaredClass == Short.TYPE) { // short
        out.writeShort(((Short) instance).shortValue());
      } else if (declaredClass == Integer.TYPE) { // int
        out.writeInt(((Integer) instance).intValue());
      } else if (declaredClass == Long.TYPE) { // long
        out.writeLong(((Long) instance).longValue());
      } else if (declaredClass == Float.TYPE) { // float
        out.writeFloat(((Float) instance).floatValue());
      } else if (declaredClass == Double.TYPE) { // double
        out.writeDouble(((Double) instance).doubleValue());
      } else if (declaredClass == Void.TYPE) { // void
      } else {
        throw new IllegalArgumentException("Not a primitive: " + declaredClass);
      }
    } else if (declaredClass.isEnum()) { // enum
      UTF8.writeString(out, ((Enum) instance).name());
    } else if (Writable.class.isAssignableFrom(declaredClass)) { // Writable
      UTF8.writeString(out, instance.getClass().getName());
      ((Writable) instance).write(out);

    } else {
      throw new IOException("Can't write: " + instance + " as " + declaredClass);
    }
  }
Example #4
0
 private static void initialize_evaluator() throws Exception {
   if (Config.bsp_mode)
     Evaluator.evaluator = (Evaluator) Class.forName("org.apache.mrql.BSPEvaluator").newInstance();
   else if (Config.spark_mode)
     Evaluator.evaluator =
         (Evaluator) Class.forName("org.apache.mrql.SparkEvaluator").newInstance();
   else if (Config.flink_mode)
     Evaluator.evaluator =
         (Evaluator) Class.forName("org.apache.mrql.FlinkEvaluator").newInstance();
   else // when Config.map_reduce_mode but also the default
   Evaluator.evaluator =
         (Evaluator) Class.forName("org.apache.mrql.MapReduceEvaluator").newInstance();
 }
Example #5
0
  /**
   * Get all superInterfaces that extend VersionedProtocol
   *
   * @param childInterfaces
   * @return the super interfaces that extend VersionedProtocol
   */
  static Class<?>[] getSuperInterfaces(Class<?>[] childInterfaces) {
    List<Class<?>> allInterfaces = new ArrayList<Class<?>>();

    for (Class<?> childInterface : childInterfaces) {
      if (VersionedProtocol.class.isAssignableFrom(childInterface)) {
        allInterfaces.add(childInterface);
        allInterfaces.addAll(Arrays.asList(getSuperInterfaces(childInterface.getInterfaces())));
      } else {
        LOG.warn(
            "Interface "
                + childInterface
                + " ignored because it does not extend VersionedProtocol");
      }
    }
    return allInterfaces.toArray(new Class[allInterfaces.size()]);
  }
Example #6
0
    public Writable call(Class<?> protocol, Writable param, long receivedTime) throws IOException {
      try {
        Invocation call = (Invocation) param;
        if (verbose) log("Call: " + call);

        Method method = protocol.getMethod(call.getMethodName(), call.getParameterClasses());
        method.setAccessible(true);

        int qTime = (int) (System.currentTimeMillis() - receivedTime);
        long startNanoTime = System.nanoTime();
        Object value = method.invoke(instance, call.getParameters());
        long processingMicroTime = (System.nanoTime() - startNanoTime) / 1000;
        if (LOG.isDebugEnabled()) {
          LOG.debug(
              "Served: "
                  + call.getMethodName()
                  + " queueTime (millisec)= "
                  + qTime
                  + " procesingTime (microsec)= "
                  + processingMicroTime);
        }
        rpcMetrics.rpcQueueTime.inc(qTime);
        rpcMetrics.rpcProcessingTime.inc(processingMicroTime);

        MetricsTimeVaryingRate m =
            (MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName());
        if (m == null) {
          try {
            m = new MetricsTimeVaryingRate(call.getMethodName(), rpcMetrics.registry);
          } catch (IllegalArgumentException iae) {
            // the metrics has been registered; re-fetch the handle
            LOG.debug("Error register " + call.getMethodName(), iae);
            m = (MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName());
          }
        }
        // record call time in microseconds
        m.inc(processingMicroTime);

        if (verbose) log("Return: " + value);

        return new ObjectWritable(method.getReturnType(), value);

      } catch (InvocationTargetException e) {
        Throwable target = e.getTargetException();
        if (target instanceof IOException) {
          throw (IOException) target;
        } else {
          IOException ioe = new IOException(target.toString());
          ioe.setStackTrace(target.getStackTrace());
          throw ioe;
        }
      } catch (Throwable e) {
        if (!(e instanceof IOException)) {
          LOG.error("Unexpected throwable object ", e);
        }
        IOException ioe = new IOException(e.toString());
        ioe.setStackTrace(e.getStackTrace());
        throw ioe;
      }
    }
  /**
   * Construct a client-side proxy object that implements the named protocol, talking to a server at
   * the named address.
   *
   * @param <T>
   */
  @Override
  @SuppressWarnings("unchecked")
  public <T> ProtocolProxy<T> getProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      UserGroupInformation ticket,
      Configuration conf,
      SocketFactory factory,
      int rpcTimeout,
      RetryPolicy connectionRetryPolicy)
      throws IOException {

    if (connectionRetryPolicy != null) {
      throw new UnsupportedOperationException(
          "Not supported: connectionRetryPolicy=" + connectionRetryPolicy);
    }

    T proxy =
        (T)
            Proxy.newProxyInstance(
                protocol.getClassLoader(),
                new Class[] {protocol},
                new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
    return new ProtocolProxy<T>(protocol, proxy, true);
  }
Example #8
0
    // Register  protocol and its impl for rpc calls
    void registerProtocolAndImpl(RpcKind rpcKind, Class<?> protocolClass, Object protocolImpl) {
      String protocolName = RPC.getProtocolName(protocolClass);
      long version;

      try {
        version = RPC.getProtocolVersion(protocolClass);
      } catch (Exception ex) {
        LOG.warn("Protocol " + protocolClass + " NOT registered as cannot get protocol version ");
        return;
      }

      getProtocolImplMap(rpcKind)
          .put(
              new ProtoNameVer(protocolName, version),
              new ProtoClassProtoImpl(protocolClass, protocolImpl));
      LOG.debug(
          "RpcKind = "
              + rpcKind
              + " Protocol Name = "
              + protocolName
              + " version="
              + version
              + " ProtocolImpl="
              + protocolImpl.getClass().getName()
              + " protocolClass="
              + protocolClass.getName());
    }
Example #9
0
 /**
  * Find and load the class with given name <tt>className</tt> by first finding it in the specified
  * <tt>conf</tt>. If the specified <tt>conf</tt> is null, try load it directly.
  */
 public static Class<?> loadClass(Configuration conf, String className) {
   Class<?> declaredClass = null;
   try {
     if (conf != null) declaredClass = conf.getClassByName(className);
     else declaredClass = Class.forName(className);
   } catch (ClassNotFoundException e) {
     throw new RuntimeException("readObject can't find class " + className, e);
   }
   return declaredClass;
 }
Example #10
0
 // return the RpcEngine configured to handle a protocol
 static synchronized RpcEngine getProtocolEngine(Class<?> protocol, Configuration conf) {
   RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
   if (engine == null) {
     Class<?> impl =
         conf.getClass(ENGINE_PROP + "." + protocol.getName(), WritableRpcEngine.class);
     engine = (RpcEngine) ReflectionUtils.newInstance(impl, conf);
     PROTOCOL_ENGINES.put(protocol, engine);
   }
   return engine;
 }
Example #11
0
 /**
  * Get the protocol version from protocol class. If the protocol class has a ProtocolAnnotation,
  * then get the protocol name from the annotation; otherwise the class name is the protocol name.
  */
 public static long getProtocolVersion(Class<?> protocol) {
   if (protocol == null) {
     throw new IllegalArgumentException("Null protocol");
   }
   long version;
   ProtocolInfo anno = protocol.getAnnotation(ProtocolInfo.class);
   if (anno != null) {
     version = anno.protocolVersion();
     if (version != -1) return version;
   }
   try {
     Field versionField = protocol.getField("versionID");
     versionField.setAccessible(true);
     return versionField.getLong(protocol);
   } catch (NoSuchFieldException ex) {
     throw new RuntimeException(ex);
   } catch (IllegalAccessException ex) {
     throw new RuntimeException(ex);
   }
 }
Example #12
0
  /**
   * Construct a client-side proxy that implements the named protocol, talking to a server at the
   * named address.
   *
   * @param protocol protocol
   * @param clientVersion client's version
   * @param addr server address
   * @param ticket security ticket
   * @param conf configuration
   * @param factory socket factory
   * @param rpcTimeout max time for each rpc; 0 means no timeout
   * @return the proxy
   * @throws IOException if any error occurs
   */
  @SuppressWarnings("unchecked")
  public static <T extends VersionedProtocol> ProtocolProxy<T> getProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      UserGroupInformation ticket,
      Configuration conf,
      SocketFactory factory,
      int rpcTimeout)
      throws IOException {
    T proxy =
        (T)
            Proxy.newProxyInstance(
                protocol.getClassLoader(),
                new Class[] {protocol},
                new Invoker(addr, ticket, conf, factory, rpcTimeout, protocol));
    String protocolName = protocol.getName();

    try {
      ProtocolSignature serverInfo =
          proxy.getProtocolSignature(
              protocolName, clientVersion, ProtocolSignature.getFingerprint(protocol.getMethods()));
      return new ProtocolProxy<T>(protocol, proxy, serverInfo.getMethods());
    } catch (RemoteException re) {
      IOException ioe = re.unwrapRemoteException(IOException.class);
      if (ioe.getMessage()
          .startsWith(IOException.class.getName() + ": " + NoSuchMethodException.class.getName())) {
        // Method getProtocolSignature not supported
        long serverVersion = proxy.getProtocolVersion(protocol.getName(), clientVersion);
        if (serverVersion == clientVersion) {
          return new ProtocolProxy<T>(protocol, proxy, null);
        }
        throw new VersionMismatch(protocolName, clientVersion, serverVersion, proxy);
      }
      throw re;
    }
  }
Example #13
0
 public void write(DataOutput out) throws IOException {
   UTF8.writeString(out, declaredClass.getName());
 }
Example #14
0
  /** Read a {@link Writable}, {@link String}, primitive type, or an array of the preceding. */
  @SuppressWarnings("unchecked")
  public static Object readObject(DataInput in, ObjectWritable objectWritable, Configuration conf)
      throws IOException {
    String className = UTF8.readString(in);
    Class<?> declaredClass = PRIMITIVE_NAMES.get(className);
    if (declaredClass == null) {
      declaredClass = loadClass(conf, className);
    }

    Object instance;

    if (declaredClass.isPrimitive()) { // primitive types

      if (declaredClass == Boolean.TYPE) { // boolean
        instance = Boolean.valueOf(in.readBoolean());
      } else if (declaredClass == Character.TYPE) { // char
        instance = Character.valueOf(in.readChar());
      } else if (declaredClass == Byte.TYPE) { // byte
        instance = Byte.valueOf(in.readByte());
      } else if (declaredClass == Short.TYPE) { // short
        instance = Short.valueOf(in.readShort());
      } else if (declaredClass == Integer.TYPE) { // int
        instance = Integer.valueOf(in.readInt());
      } else if (declaredClass == Long.TYPE) { // long
        instance = Long.valueOf(in.readLong());
      } else if (declaredClass == Float.TYPE) { // float
        instance = Float.valueOf(in.readFloat());
      } else if (declaredClass == Double.TYPE) { // double
        instance = Double.valueOf(in.readDouble());
      } else if (declaredClass == Void.TYPE) { // void
        instance = null;
      } else {
        throw new IllegalArgumentException("Not a primitive: " + declaredClass);
      }

    } else if (declaredClass.isArray()) { // array
      int length = in.readInt();
      instance = Array.newInstance(declaredClass.getComponentType(), length);
      for (int i = 0; i < length; i++) {
        Array.set(instance, i, readObject(in, conf));
      }

    } else if (declaredClass == String.class) { // String
      instance = UTF8.readString(in);
    } else if (declaredClass.isEnum()) { // enum
      instance = Enum.valueOf((Class<? extends Enum>) declaredClass, UTF8.readString(in));
    } else { // Writable
      Class instanceClass = null;
      String str = UTF8.readString(in);
      instanceClass = loadClass(conf, str);

      Writable writable = WritableFactories.newInstance(instanceClass, conf);
      writable.readFields(in);
      instance = writable;

      if (instanceClass == NullInstance.class) { // null
        declaredClass = ((NullInstance) instance).declaredClass;
        instance = null;
      }
    }

    if (objectWritable != null) { // store values
      objectWritable.declaredClass = declaredClass;
      objectWritable.instance = instance;
    }

    return instance;
  }
Example #15
0
  // Information needed to get a single file:
  // BASE_PATH, FILE_ID, TIMESTAMP_START, TIMESTAMP_STOP, SOURCE, FILESYSTEM
  private static Vector<Path> getFile(
      FileSystem fs, Hashtable<String, String> config, dbutil db_util) throws Exception {
    Long latestVersion = latestVersion(config, db_util);

    try {
      config.put("timestamp_start", config.get("timestamp_start"));
      config.put("timestamp_real", latestVersion.toString());
      config.put("timestamp_stop", latestVersion.toString());
    } catch (Exception E) {
      logger.error("Tryign to get file that is impossible to generate: " + getFullPath(config));
      return null;
    }
    if (Integer.parseInt(config.get("timestamp_start"))
        > Integer.parseInt(config.get("timestamp_stop"))) {
      return null;
    }
    logger.debug(
        "Getting DB for timestamp "
            + config.get("timestamp_start")
            + " to "
            + config.get("timestamp_stop"));

    String final_result = getFullPath(config);

    String temp_path_base =
        config.get("local_temp_path")
            + "_"
            + config.get("task_id")
            + "_"
            + config.get("run_id")
            + "/";
    Path newPath = new Path(final_result + "*");
    Vector<Path> ret_path = new Vector<Path>();
    String lockName = lock(final_result.replaceAll("/", "_"));
    if (fs.globStatus(newPath).length != 0) {
      ret_path.add(newPath);
      unlock(lockName);
      config.put("full_file_name", final_result);
      return ret_path;
    } else {
      if (!config.get("source").equals("local")) {
        config.put("temp_path_base", temp_path_base);

        config.put("timestamp_start", config.get("timestamp_start"));
        config.put("timestamp_real", latestVersion.toString());
        config.put("timestamp_stop", latestVersion.toString());

        Class<?> sourceClass =
            Class.forName("org.gestore.plugin.source." + config.get("source") + "Source");
        Method process_data = sourceClass.getMethod("process", Hashtable.class, FileSystem.class);
        Object processor = sourceClass.newInstance();
        Object retVal;
        try {
          retVal = process_data.invoke(processor, config, fs);
        } catch (InvocationTargetException E) {
          Throwable exception = E.getTargetException();
          logger.error("Unable to call method in child class: " + exception.toString());
          exception.printStackTrace(System.out);
          unlock(lockName);
          return null;
        }
        FileStatus[] files = (FileStatus[]) retVal;
        if (files == null) {
          logger.error("Error getting files, no files returned");
          return null;
        }

        for (FileStatus file : files) {
          Path cur_file = file.getPath();
          Path cur_local_path = new Path(temp_path_base + config.get("file_id"));
          String suffix = getSuffix(config.get("file_id"), cur_file.getName());
          cur_local_path = cur_local_path.suffix(suffix);
          Path res_path = new Path(new String(final_result + suffix));
          logger.debug("Moving file" + cur_file.toString() + " to " + res_path.toString());
          if (config.get("copy").equals("true")) {
            fs.moveFromLocalFile(cur_file, res_path);
          } else {
            fs.rename(cur_file, res_path);
          }
        }

        config.put("full_file_name", final_result);
      }
    }
    unlock(lockName);
    return ret_path;
  }
Example #16
0
 /**
  * Set a protocol to use a non-default RpcEngine.
  *
  * @param conf configuration to use
  * @param protocol the protocol interface
  * @param engine the RpcEngine impl
  */
 public static void setProtocolEngine(Configuration conf, Class<?> protocol, Class<?> engine) {
   conf.setClass(ENGINE_PROP + "." + protocol.getName(), engine, RpcEngine.class);
 }
Example #17
0
 /**
  * Get all interfaces that the given protocol implements or extends which are assignable from
  * VersionedProtocol.
  */
 static Class<?>[] getProtocolInterfaces(Class<?> protocol) {
   Class<?>[] interfaces = protocol.getInterfaces();
   return getSuperInterfaces(interfaces);
 }