예제 #1
0
  /**
   * Stop the proxy. Proxy must either implement {@link Closeable} or must have associated {@link
   * RpcInvocationHandler}.
   *
   * @param proxy the RPC proxy object to be stopped
   * @throws HadoopIllegalArgumentException if the proxy does not implement {@link Closeable}
   *     interface or does not have closeable {@link InvocationHandler}
   */
  public static void stopProxy(Object proxy) {
    if (proxy == null) {
      throw new HadoopIllegalArgumentException("Cannot close proxy since it is null");
    }
    try {
      if (proxy instanceof Closeable) {
        ((Closeable) proxy).close();
        return;
      } else {
        InvocationHandler handler = Proxy.getInvocationHandler(proxy);
        if (handler instanceof Closeable) {
          ((Closeable) handler).close();
          return;
        }
      }
    } catch (IOException e) {
      LOG.error("Closing proxy or invocation handler caused exception", e);
    } catch (IllegalArgumentException e) {
      LOG.error("RPC.stopProxy called on non proxy: class=" + proxy.getClass().getName(), e);
    }

    // If you see this error on a mock object in a unit test you're
    // developing, make sure to use MockitoUtil.mockProtocol() to
    // create your mock.
    throw new HadoopIllegalArgumentException(
        "Cannot close proxy - is not Closeable or "
            + "does not provide closeable invocation handler "
            + proxy.getClass());
  }
예제 #2
0
    // Register  protocol and its impl for rpc calls
    void registerProtocolAndImpl(RpcKind rpcKind, Class<?> protocolClass, Object protocolImpl) {
      String protocolName = RPC.getProtocolName(protocolClass);
      long version;

      try {
        version = RPC.getProtocolVersion(protocolClass);
      } catch (Exception ex) {
        LOG.warn("Protocol " + protocolClass + " NOT registered as cannot get protocol version ");
        return;
      }

      getProtocolImplMap(rpcKind)
          .put(
              new ProtoNameVer(protocolName, version),
              new ProtoClassProtoImpl(protocolClass, protocolImpl));
      LOG.debug(
          "RpcKind = "
              + rpcKind
              + " Protocol Name = "
              + protocolName
              + " version="
              + version
              + " ProtocolImpl="
              + protocolImpl.getClass().getName()
              + " protocolClass="
              + protocolClass.getName());
    }
예제 #3
0
파일: RPC.java 프로젝트: baeeq/hadoop-20
    public Writable call(Class<?> protocol, Writable param, long receivedTime) throws IOException {
      try {
        Invocation call = (Invocation) param;
        if (verbose) log("Call: " + call);

        Method method = protocol.getMethod(call.getMethodName(), call.getParameterClasses());
        method.setAccessible(true);

        int qTime = (int) (System.currentTimeMillis() - receivedTime);
        long startNanoTime = System.nanoTime();
        Object value = method.invoke(instance, call.getParameters());
        long processingMicroTime = (System.nanoTime() - startNanoTime) / 1000;
        if (LOG.isDebugEnabled()) {
          LOG.debug(
              "Served: "
                  + call.getMethodName()
                  + " queueTime (millisec)= "
                  + qTime
                  + " procesingTime (microsec)= "
                  + processingMicroTime);
        }
        rpcMetrics.rpcQueueTime.inc(qTime);
        rpcMetrics.rpcProcessingTime.inc(processingMicroTime);

        MetricsTimeVaryingRate m =
            (MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName());
        if (m == null) {
          try {
            m = new MetricsTimeVaryingRate(call.getMethodName(), rpcMetrics.registry);
          } catch (IllegalArgumentException iae) {
            // the metrics has been registered; re-fetch the handle
            LOG.debug("Error register " + call.getMethodName(), iae);
            m = (MetricsTimeVaryingRate) rpcMetrics.registry.get(call.getMethodName());
          }
        }
        // record call time in microseconds
        m.inc(processingMicroTime);

        if (verbose) log("Return: " + value);

        return new ObjectWritable(method.getReturnType(), value);

      } catch (InvocationTargetException e) {
        Throwable target = e.getTargetException();
        if (target instanceof IOException) {
          throw (IOException) target;
        } else {
          IOException ioe = new IOException(target.toString());
          ioe.setStackTrace(target.getStackTrace());
          throw ioe;
        }
      } catch (Throwable e) {
        if (!(e instanceof IOException)) {
          LOG.error("Unexpected throwable object ", e);
        }
        IOException ioe = new IOException(e.toString());
        ioe.setStackTrace(e.getStackTrace());
        throw ioe;
      }
    }
예제 #4
0
파일: RPC.java 프로젝트: baeeq/hadoop-20
    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
      final boolean logDebug = LOG.isDebugEnabled();
      long startTime = 0;
      if (logDebug) {
        startTime = System.currentTimeMillis();
      }

      ObjectWritable value = null;
      try {
        value =
            (ObjectWritable)
                client.call(
                    new Invocation(method, args), getAddress(), protocol, ticket, rpcTimeout);
      } catch (RemoteException re) {
        throw re;
      } catch (ConnectException ce) {
        needCheckDnsUpdate = true;
        throw ce;
      } catch (NoRouteToHostException nrhe) {
        needCheckDnsUpdate = true;
        throw nrhe;
      } catch (PortUnreachableException pue) {
        needCheckDnsUpdate = true;
        throw pue;
      } catch (UnknownHostException uhe) {
        needCheckDnsUpdate = true;
        throw uhe;
      }
      if (logDebug) {
        long callTime = System.currentTimeMillis() - startTime;
        LOG.debug("Call: " + method.getName() + " " + callTime);
      }
      return value.get();
    }
예제 #5
0
    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
      long startTime = 0;
      if (LOG.isDebugEnabled()) {
        startTime = Time.now();
      }

      ObjectWritable value =
          (ObjectWritable)
              client.call(RPC.RpcKind.RPC_WRITABLE, new Invocation(method, args), remoteId);
      if (LOG.isDebugEnabled()) {
        long callTime = Time.now() - startTime;
        LOG.debug("Call: " + method.getName() + " " + callTime);
      }
      return value.get();
    }
예제 #6
0
파일: RPC.java 프로젝트: baeeq/hadoop-20
  /**
   * Get a proxy connection to a remote server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @param rpcTimeout timeout for each RPC
   * @param timeout time in milliseconds before giving up
   * @return the proxy
   * @throws IOException if the far end through a RemoteException
   */
  static <T extends VersionedProtocol> ProtocolProxy<T> waitForProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      long timeout,
      int rpcTimeout)
      throws IOException {
    long startTime = System.currentTimeMillis();
    UserGroupInformation ugi = null;
    try {
      ugi = UserGroupInformation.login(conf);
    } catch (LoginException le) {
      throw new RuntimeException("Couldn't login!");
    }
    IOException ioe;
    while (true) {
      try {
        return getProtocolProxy(
            protocol,
            clientVersion,
            addr,
            ugi,
            conf,
            NetUtils.getDefaultSocketFactory(conf),
            rpcTimeout);
      } catch (ConnectException se) { // namenode has not been started
        LOG.info("Server at " + addr + " not available yet, Zzzzz...");
        ioe = se;
      } catch (SocketTimeoutException te) { // namenode is busy
        LOG.info("Problem connecting to server: " + addr);
        ioe = te;
      }
      // check if timed out
      if (System.currentTimeMillis() - timeout >= startTime) {
        throw ioe;
      }

      // wait for retry
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
        // IGNORE
      }
    }
  }
예제 #7
0
  /**
   * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods
   * that are supported by the server
   *
   * @param protocol protocol class
   * @param clientVersion client version
   * @param addr remote address
   * @param conf configuration to use
   * @param rpcTimeout timeout for each RPC
   * @param timeout time in milliseconds before giving up
   * @return the proxy
   * @throws IOException if the far end through a RemoteException
   */
  public static <T> ProtocolProxy<T> waitForProtocolProxy(
      Class<T> protocol,
      long clientVersion,
      InetSocketAddress addr,
      Configuration conf,
      int rpcTimeout,
      RetryPolicy connectionRetryPolicy,
      long timeout)
      throws IOException {
    long startTime = Time.now();
    IOException ioe;
    while (true) {
      try {
        return getProtocolProxy(
            protocol,
            clientVersion,
            addr,
            UserGroupInformation.getCurrentUser(),
            conf,
            NetUtils.getDefaultSocketFactory(conf),
            rpcTimeout,
            connectionRetryPolicy);
      } catch (ConnectException se) { // namenode has not been started
        LOG.info("Server at " + addr + " not available yet, Zzzzz...");
        ioe = se;
      } catch (SocketTimeoutException te) { // namenode is busy
        LOG.info("Problem connecting to server: " + addr);
        ioe = te;
      } catch (NoRouteToHostException nrthe) { // perhaps a VIP is failing over
        LOG.info("No route to host for server: " + addr);
        ioe = nrthe;
      }
      // check if timed out
      if (Time.now() - timeout >= startTime) {
        throw ioe;
      }

      // wait for retry
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
        // IGNORE
      }
    }
  }
예제 #8
0
 VerProtocolImpl getHighestSupportedProtocol(RpcKind rpcKind, String protocolName) {
   Long highestVersion = 0L;
   ProtoClassProtoImpl highest = null;
   if (LOG.isDebugEnabled()) {
     LOG.debug("Size of protoMap for " + rpcKind + " =" + getProtocolImplMap(rpcKind).size());
   }
   for (Map.Entry<ProtoNameVer, ProtoClassProtoImpl> pv :
       getProtocolImplMap(rpcKind).entrySet()) {
     if (pv.getKey().protocol.equals(protocolName)) {
       if ((highest == null) || (pv.getKey().version > highestVersion)) {
         highest = pv.getValue();
         highestVersion = pv.getKey().version;
       }
     }
   }
   if (highest == null) {
     return null;
   }
   return new VerProtocolImpl(highestVersion, highest);
 }
예제 #9
0
  /**
   * Get all superInterfaces that extend VersionedProtocol
   *
   * @param childInterfaces
   * @return the super interfaces that extend VersionedProtocol
   */
  static Class<?>[] getSuperInterfaces(Class<?>[] childInterfaces) {
    List<Class<?>> allInterfaces = new ArrayList<Class<?>>();

    for (Class<?> childInterface : childInterfaces) {
      if (VersionedProtocol.class.isAssignableFrom(childInterface)) {
        allInterfaces.add(childInterface);
        allInterfaces.addAll(Arrays.asList(getSuperInterfaces(childInterface.getInterfaces())));
      } else {
        LOG.warn(
            "Interface "
                + childInterface
                + " ignored because it does not extend VersionedProtocol");
      }
    }
    return allInterfaces.toArray(new Class[allInterfaces.size()]);
  }
예제 #10
0
파일: RPC.java 프로젝트: baeeq/hadoop-20
 private synchronized InetSocketAddress getAddress() {
   if (needCheckDnsUpdate
       && address != null
       && address.getHostName() != null
       && System.currentTimeMillis() - this.timeLastDnsCheck > MIN_DNS_CHECK_INTERVAL_MSEC) {
     try {
       InetSocketAddress newAddr = NetUtils.resolveAddress(address);
       if (newAddr != null) {
         LOG.info("DNS change: " + newAddr);
         address = newAddr;
       }
     } finally {
       this.timeLastDnsCheck = System.currentTimeMillis();
     }
   }
   needCheckDnsUpdate = false;
   return address;
 }
  public void testBinary() throws IOException, InterruptedException {
    Configuration conf = new Configuration();
    Job job = new Job(conf);

    Path outdir = new Path(System.getProperty("test.build.data", "/tmp"), "outseq");
    Random r = new Random();
    long seed = r.nextLong();
    r.setSeed(seed);

    FileOutputFormat.setOutputPath(job, outdir);

    SequenceFileAsBinaryOutputFormat.setSequenceFileOutputKeyClass(job, IntWritable.class);
    SequenceFileAsBinaryOutputFormat.setSequenceFileOutputValueClass(job, DoubleWritable.class);

    SequenceFileAsBinaryOutputFormat.setCompressOutput(job, true);
    SequenceFileAsBinaryOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);

    BytesWritable bkey = new BytesWritable();
    BytesWritable bval = new BytesWritable();

    TaskAttemptContext context =
        MapReduceTestUtil.createDummyMapTaskAttemptContext(job.getConfiguration());
    OutputFormat<BytesWritable, BytesWritable> outputFormat =
        new SequenceFileAsBinaryOutputFormat();
    OutputCommitter committer = outputFormat.getOutputCommitter(context);
    committer.setupJob(job);
    RecordWriter<BytesWritable, BytesWritable> writer = outputFormat.getRecordWriter(context);

    IntWritable iwritable = new IntWritable();
    DoubleWritable dwritable = new DoubleWritable();
    DataOutputBuffer outbuf = new DataOutputBuffer();
    LOG.info("Creating data by SequenceFileAsBinaryOutputFormat");
    try {
      for (int i = 0; i < RECORDS; ++i) {
        iwritable = new IntWritable(r.nextInt());
        iwritable.write(outbuf);
        bkey.set(outbuf.getData(), 0, outbuf.getLength());
        outbuf.reset();
        dwritable = new DoubleWritable(r.nextDouble());
        dwritable.write(outbuf);
        bval.set(outbuf.getData(), 0, outbuf.getLength());
        outbuf.reset();
        writer.write(bkey, bval);
      }
    } finally {
      writer.close(context);
    }
    committer.commitTask(context);
    committer.commitJob(job);

    InputFormat<IntWritable, DoubleWritable> iformat =
        new SequenceFileInputFormat<IntWritable, DoubleWritable>();
    int count = 0;
    r.setSeed(seed);
    SequenceFileInputFormat.setInputPaths(job, outdir);
    LOG.info("Reading data by SequenceFileInputFormat");
    for (InputSplit split : iformat.getSplits(job)) {
      RecordReader<IntWritable, DoubleWritable> reader = iformat.createRecordReader(split, context);
      MapContext<IntWritable, DoubleWritable, BytesWritable, BytesWritable> mcontext =
          new MapContextImpl<IntWritable, DoubleWritable, BytesWritable, BytesWritable>(
              job.getConfiguration(),
              context.getTaskAttemptID(),
              reader,
              null,
              null,
              MapReduceTestUtil.createDummyReporter(),
              split);
      reader.initialize(split, mcontext);
      try {
        int sourceInt;
        double sourceDouble;
        while (reader.nextKeyValue()) {
          sourceInt = r.nextInt();
          sourceDouble = r.nextDouble();
          iwritable = reader.getCurrentKey();
          dwritable = reader.getCurrentValue();
          assertEquals(
              "Keys don't match: " + "*" + iwritable.get() + ":" + sourceInt + "*",
              sourceInt,
              iwritable.get());
          assertTrue(
              "Vals don't match: " + "*" + dwritable.get() + ":" + sourceDouble + "*",
              Double.compare(dwritable.get(), sourceDouble) == 0);
          ++count;
        }
      } finally {
        reader.close();
      }
    }
    assertEquals("Some records not found", RECORDS, count);
  }
예제 #12
0
파일: RPC.java 프로젝트: baeeq/hadoop-20
 private static void log(String value) {
   if (value != null && value.length() > 55) value = value.substring(0, 55) + "...";
   LOG.info(value);
 }