@Override
protected void serviceStop() throws Exception {
if (this.rmClient != null) {
RPC.stopProxy(this.rmClient);
}
super.serviceStop();
}
private void doHealthChecks() throws InterruptedException {
while (shouldRun) {
HAServiceStatus status = null;
boolean healthy = false;
try {
status = proxy.getServiceStatus();
proxy.monitorHealth();
healthy = true;
} catch (HealthCheckFailedException e) {
LOG.warn("Service health check failed for " + targetToMonitor + ": " + e.getMessage());
enterState(State.SERVICE_UNHEALTHY);
} catch (Throwable t) {
LOG.warn(
"Transport-level exception trying to monitor health of "
+ targetToMonitor
+ ": "
+ t.getLocalizedMessage());
RPC.stopProxy(proxy);
proxy = null;
enterState(State.SERVICE_NOT_RESPONDING);
Thread.sleep(sleepAfterDisconnectMillis);
return;
}
if (status != null) {
setLastServiceStatus(status);
}
if (healthy) {
enterState(State.SERVICE_HEALTHY);
}
Thread.sleep(checkIntervalMillis);
}
}
@Test
public void testProxyAddress() throws IOException {
Server server =
new RPC.Builder(conf)
.setProtocol(TestProtocol.class)
.setInstance(new TestImpl())
.setBindAddress(ADDRESS)
.setPort(0)
.build();
TestProtocol proxy = null;
try {
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
// create a client
proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
assertEquals(addr, RPC.getServerAddress(proxy));
} finally {
server.stop();
if (proxy != null) {
RPC.stopProxy(proxy);
}
}
}
private void doRPCs(Configuration conf, boolean expectFailure) throws Exception {
SecurityUtil.setPolicy(new ConfiguredPolicy(conf, new TestPolicyProvider()));
Server server = RPC.getServer(new TestImpl(), ADDRESS, 0, 5, true, conf);
TestProtocol proxy = null;
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
try {
proxy = (TestProtocol) RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
proxy.ping();
if (expectFailure) {
fail("Expect RPC.getProxy to fail with AuthorizationException!");
}
} catch (RemoteException e) {
if (expectFailure) {
assertTrue(e.unwrapRemoteException() instanceof AuthorizationException);
} else {
throw e;
}
} finally {
server.stop();
if (proxy != null) {
RPC.stopProxy(proxy);
}
}
}
@Test
public void testConnectionPing() throws Exception {
Configuration conf = new Configuration();
int pingInterval = 50;
conf.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true);
conf.setInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, pingInterval);
final Server server =
new RPC.Builder(conf)
.setProtocol(TestProtocol.class)
.setInstance(new TestImpl())
.setBindAddress(ADDRESS)
.setPort(0)
.setNumHandlers(5)
.setVerbose(true)
.build();
server.start();
final TestProtocol proxy =
RPC.getProxy(TestProtocol.class, TestProtocol.versionID, server.getListenerAddress(), conf);
try {
// this call will throw exception if server couldn't decode the ping
proxy.sleep(pingInterval * 4);
} finally {
if (proxy != null) RPC.stopProxy(proxy);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
BizProtocol proxy =
RPC.getProxy(BizProtocol.class, 10010, new InetSocketAddress("localhost", 8888), conf);
String result = proxy.hello("world");
System.out.println(result);
RPC.stopProxy(proxy);
}
@Test
public void testStopProxy() throws IOException {
StoppedProtocol proxy =
RPC.getProxy(StoppedProtocol.class, StoppedProtocol.versionID, null, conf);
StoppedInvocationHandler invocationHandler =
(StoppedInvocationHandler) Proxy.getInvocationHandler(proxy);
assertEquals(invocationHandler.getCloseCalled(), 0);
RPC.stopProxy(proxy);
assertEquals(invocationHandler.getCloseCalled(), 1);
}
public void shutdown() {
if (executor != null) {
executor.shutdownNow();
}
if (taskReporter != null) {
taskReporter.shutdown();
}
if (umbilical != null) {
RPC.stopProxy(umbilical);
}
}
@Test
public void testSlowRpc() throws IOException {
System.out.println("Testing Slow RPC");
// create a server with two handlers
Server server =
new RPC.Builder(conf)
.setProtocol(TestProtocol.class)
.setInstance(new TestImpl())
.setBindAddress(ADDRESS)
.setPort(0)
.setNumHandlers(2)
.setVerbose(false)
.build();
TestProtocol proxy = null;
try {
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
// create a client
proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
SlowRPC slowrpc = new SlowRPC(proxy);
Thread thread = new Thread(slowrpc, "SlowRPC");
thread.start(); // send a slow RPC, which won't return until two fast pings
assertTrue("Slow RPC should not have finished1.", !slowrpc.isDone());
proxy.slowPing(false); // first fast ping
// verify that the first RPC is still stuck
assertTrue("Slow RPC should not have finished2.", !slowrpc.isDone());
proxy.slowPing(false); // second fast ping
// Now the slow ping should be able to be executed
while (!slowrpc.isDone()) {
System.out.println("Waiting for slow RPC to get done.");
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
} finally {
server.stop();
if (proxy != null) {
RPC.stopProxy(proxy);
}
System.out.println("Down slow rpc testing");
}
}
@Override
protected void serviceStop() throws Exception {
if (this.rmClient != null) {
RPC.stopProxy(this.rmClient);
}
if (historyServiceEnabled) {
historyClient.stop();
}
if (timelineServiceEnabled) {
timelineClient.stop();
}
super.serviceStop();
}
/** Read the block length from one of the datanodes. */
private long readBlockLength(LocatedBlock locatedblock) throws IOException {
assert locatedblock != null : "LocatedBlock cannot be null";
int replicaNotFoundCount = locatedblock.getLocations().length;
for (DatanodeInfo datanode : locatedblock.getLocations()) {
ClientDatanodeProtocol cdp = null;
try {
cdp =
DFSUtil.createClientDatanodeProtocolProxy(
datanode, dfsClient.conf, dfsClient.getConf().socketTimeout, locatedblock);
final long n = cdp.getReplicaVisibleLength(locatedblock.getBlock());
if (n >= 0) {
return n;
}
} catch (IOException ioe) {
if (ioe instanceof RemoteException
&& (((RemoteException) ioe).unwrapRemoteException()
instanceof ReplicaNotFoundException)) {
// special case : replica might not be on the DN, treat as 0 length
replicaNotFoundCount--;
}
if (DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug(
"Failed to getReplicaVisibleLength from datanode "
+ datanode
+ " for block "
+ locatedblock.getBlock(),
ioe);
}
} finally {
if (cdp != null) {
RPC.stopProxy(cdp);
}
}
}
// Namenode told us about these locations, but none know about the replica
// means that we hit the race between pipeline creation start and end.
// we require all 3 because some other exception could have happened
// on a DN that has it. we want to report that error
if (replicaNotFoundCount == 0) {
return 0;
}
throw new IOException("Cannot obtain block length for " + locatedblock);
}
private void doRPCs(Configuration conf, boolean expectFailure) throws IOException {
Server server =
new RPC.Builder(conf)
.setProtocol(TestProtocol.class)
.setInstance(new TestImpl())
.setBindAddress(ADDRESS)
.setPort(0)
.setNumHandlers(5)
.setVerbose(true)
.build();
server.refreshServiceAcl(conf, new TestPolicyProvider());
TestProtocol proxy = null;
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
try {
proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
proxy.ping();
if (expectFailure) {
fail("Expect RPC.getProxy to fail with AuthorizationException!");
}
} catch (RemoteException e) {
if (expectFailure) {
assertTrue(e.unwrapRemoteException() instanceof AuthorizationException);
} else {
throw e;
}
} finally {
server.stop();
if (proxy != null) {
RPC.stopProxy(proxy);
}
MetricsRecordBuilder rb = getMetrics(server.rpcMetrics.name());
if (expectFailure) {
assertCounter("RpcAuthorizationFailures", 1, rb);
} else {
assertCounter("RpcAuthorizationSuccesses", 1, rb);
}
// since we don't have authentication turned ON, we should see
// 0 for the authentication successes and 0 for failure
assertCounter("RpcAuthenticationFailures", 0, rb);
assertCounter("RpcAuthenticationSuccesses", 0, rb);
}
}
@Test
public void testWrappedStopProxy() throws IOException {
StoppedProtocol wrappedProxy =
RPC.getProxy(StoppedProtocol.class, StoppedProtocol.versionID, null, conf);
StoppedInvocationHandler invocationHandler =
(StoppedInvocationHandler) Proxy.getInvocationHandler(wrappedProxy);
StoppedProtocol proxy =
(StoppedProtocol)
RetryProxy.create(StoppedProtocol.class, wrappedProxy, RetryPolicies.RETRY_FOREVER);
assertEquals(invocationHandler.getCloseCalled(), 0);
RPC.stopProxy(proxy);
assertEquals(invocationHandler.getCloseCalled(), 1);
}
/**
* Test to verify that InterDatanode RPC timesout as expected when the server DN does not respond.
*/
@Test(expected = SocketTimeoutException.class)
public void testInterDNProtocolTimeout() throws Throwable {
final Server server = new TestServer(1, true);
server.start();
final InetSocketAddress addr = NetUtils.getConnectAddress(server);
DatanodeID fakeDnId =
new DatanodeID("localhost", "localhost", "fake-storage", addr.getPort(), 0, addr.getPort());
DatanodeInfo dInfo = new DatanodeInfo(fakeDnId);
InterDatanodeProtocol proxy = null;
try {
proxy = DataNode.createInterDataNodeProtocolProxy(dInfo, conf, 500);
proxy.initReplicaRecovery(new RecoveringBlock(new ExtendedBlock("bpid", 1), null, 100));
fail("Expected SocketTimeoutException exception, but did not get.");
} finally {
if (proxy != null) {
RPC.stopProxy(proxy);
}
server.stop();
}
}
@Test
public void testRpcMetrics() throws Exception {
Configuration configuration = new Configuration();
final int interval = 1;
configuration.setBoolean(CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE, true);
configuration.set(CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval);
final Server server =
new RPC.Builder(configuration)
.setProtocol(TestProtocol.class)
.setInstance(new TestImpl())
.setBindAddress(ADDRESS)
.setPort(0)
.setNumHandlers(5)
.setVerbose(true)
.build();
server.start();
final TestProtocol proxy =
RPC.getProxy(
TestProtocol.class, TestProtocol.versionID, server.getListenerAddress(), configuration);
try {
for (int i = 0; i < 1000; i++) {
proxy.ping();
proxy.echo("" + i);
}
MetricsRecordBuilder rpcMetrics = getMetrics(server.getRpcMetrics().name());
assertTrue(
"Expected non-zero rpc queue time", getLongCounter("RpcQueueTimeNumOps", rpcMetrics) > 0);
assertTrue(
"Expected non-zero rpc processing time",
getLongCounter("RpcProcessingTimeNumOps", rpcMetrics) > 0);
MetricsAsserts.assertQuantileGauges("RpcQueueTime" + interval + "s", rpcMetrics);
MetricsAsserts.assertQuantileGauges("RpcProcessingTime" + interval + "s", rpcMetrics);
} finally {
if (proxy != null) {
RPC.stopProxy(proxy);
}
server.stop();
}
}
@Override
public void close() {
RPC.stopProxy(rpcProxy);
}
@Test
public void testErrorMsgForInsecureClient() throws IOException {
Configuration serverConf = new Configuration(conf);
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, serverConf);
UserGroupInformation.setConfiguration(serverConf);
final Server server =
new RPC.Builder(serverConf)
.setProtocol(TestProtocol.class)
.setInstance(new TestImpl())
.setBindAddress(ADDRESS)
.setPort(0)
.setNumHandlers(5)
.setVerbose(true)
.build();
server.start();
UserGroupInformation.setConfiguration(conf);
boolean succeeded = false;
final InetSocketAddress addr = NetUtils.getConnectAddress(server);
TestProtocol proxy = null;
try {
proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
proxy.echo("");
} catch (RemoteException e) {
LOG.info("LOGGING MESSAGE: " + e.getLocalizedMessage());
assertTrue(e.unwrapRemoteException() instanceof AccessControlException);
succeeded = true;
} finally {
server.stop();
if (proxy != null) {
RPC.stopProxy(proxy);
}
}
assertTrue(succeeded);
conf.setInt(CommonConfigurationKeys.IPC_SERVER_RPC_READ_THREADS_KEY, 2);
UserGroupInformation.setConfiguration(serverConf);
final Server multiServer =
new RPC.Builder(serverConf)
.setProtocol(TestProtocol.class)
.setInstance(new TestImpl())
.setBindAddress(ADDRESS)
.setPort(0)
.setNumHandlers(5)
.setVerbose(true)
.build();
multiServer.start();
succeeded = false;
final InetSocketAddress mulitServerAddr = NetUtils.getConnectAddress(multiServer);
proxy = null;
try {
UserGroupInformation.setConfiguration(conf);
proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, mulitServerAddr, conf);
proxy.echo("");
} catch (RemoteException e) {
LOG.info("LOGGING MESSAGE: " + e.getLocalizedMessage());
assertTrue(e.unwrapRemoteException() instanceof AccessControlException);
succeeded = true;
} finally {
multiServer.stop();
if (proxy != null) {
RPC.stopProxy(proxy);
}
}
assertTrue(succeeded);
}
public void testCalls() throws Exception {
Server server = RPC.getServer(new TestImpl(), ADDRESS, 0, conf);
TestProtocol proxy = null;
try {
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
proxy = (TestProtocol) RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
proxy.ping();
String stringResult = proxy.echo("foo");
assertEquals(stringResult, "foo");
stringResult = proxy.echo((String) null);
assertEquals(stringResult, null);
String[] stringResults = proxy.echo(new String[] {"foo", "bar"});
assertTrue(Arrays.equals(stringResults, new String[] {"foo", "bar"}));
stringResults = proxy.echo((String[]) null);
assertTrue(Arrays.equals(stringResults, null));
UTF8 utf8Result = (UTF8) proxy.echo(new UTF8("hello world"));
assertEquals(utf8Result, new UTF8("hello world"));
utf8Result = (UTF8) proxy.echo((UTF8) null);
assertEquals(utf8Result, null);
int intResult = proxy.add(1, 2);
assertEquals(intResult, 3);
intResult = proxy.add(new int[] {1, 2});
assertEquals(intResult, 3);
boolean caught = false;
try {
proxy.error();
} catch (IOException e) {
LOG.debug("Caught " + e);
caught = true;
}
assertTrue(caught);
proxy.testServerGet();
// create multiple threads and make them do large data transfers
System.out.println("Starting multi-threaded RPC test...");
server.setSocketSendBufSize(1024);
Thread threadId[] = new Thread[numThreads];
for (int i = 0; i < numThreads; i++) {
Transactions trans = new Transactions(proxy, datasize);
threadId[i] = new Thread(trans, "TransactionThread-" + i);
threadId[i].start();
}
// wait for all transactions to get over
System.out.println("Waiting for all threads to finish RPCs...");
for (int i = 0; i < numThreads; i++) {
try {
threadId[i].join();
} catch (InterruptedException e) {
i--; // retry
}
}
// try some multi-calls
Method echo = TestProtocol.class.getMethod("echo", new Class[] {String.class});
String[] strings =
(String[])
RPC.call(
echo, new String[][] {{"a"}, {"b"}}, new InetSocketAddress[] {addr, addr}, conf);
assertTrue(Arrays.equals(strings, new String[] {"a", "b"}));
Method ping = TestProtocol.class.getMethod("ping", new Class[] {});
Object[] voids =
(Object[])
RPC.call(ping, new Object[][] {{}, {}}, new InetSocketAddress[] {addr, addr}, conf);
assertEquals(voids, null);
} finally {
server.stop();
if (proxy != null) RPC.stopProxy(proxy);
}
}
/** Test that the mockProtocol helper returns mock proxies that can be stopped without error. */
@Test
public void testStopMockObject() throws IOException {
RPC.stopProxy(MockitoUtil.mockProtocol(TestProtocol.class));
}
/**
* Test stopping a non-registered proxy
*
* @throws IOException
*/
@Test(expected = HadoopIllegalArgumentException.class)
public void testStopNonRegisteredProxy() throws IOException {
RPC.stopProxy(null);
}
private void testCallsInternal(Configuration conf) throws IOException {
Server server =
new RPC.Builder(conf)
.setProtocol(TestProtocol.class)
.setInstance(new TestImpl())
.setBindAddress(ADDRESS)
.setPort(0)
.build();
TestProtocol proxy = null;
try {
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
proxy.ping();
String stringResult = proxy.echo("foo");
assertEquals(stringResult, "foo");
stringResult = proxy.echo((String) null);
assertEquals(stringResult, null);
// Check rpcMetrics
MetricsRecordBuilder rb = getMetrics(server.rpcMetrics.name());
assertCounter("RpcProcessingTimeNumOps", 3L, rb);
assertCounterGt("SentBytes", 0L, rb);
assertCounterGt("ReceivedBytes", 0L, rb);
// Number of calls to echo method should be 2
rb = getMetrics(server.rpcDetailedMetrics.name());
assertCounter("EchoNumOps", 2L, rb);
// Number of calls to ping method should be 1
assertCounter("PingNumOps", 1L, rb);
String[] stringResults = proxy.echo(new String[] {"foo", "bar"});
assertTrue(Arrays.equals(stringResults, new String[] {"foo", "bar"}));
stringResults = proxy.echo((String[]) null);
assertTrue(Arrays.equals(stringResults, null));
UTF8 utf8Result = (UTF8) proxy.echo(new UTF8("hello world"));
assertEquals(utf8Result, new UTF8("hello world"));
utf8Result = (UTF8) proxy.echo((UTF8) null);
assertEquals(utf8Result, null);
int intResult = proxy.add(1, 2);
assertEquals(intResult, 3);
intResult = proxy.add(new int[] {1, 2});
assertEquals(intResult, 3);
// Test protobufs
EnumDescriptorProto sendProto = EnumDescriptorProto.newBuilder().setName("test").build();
EnumDescriptorProto retProto = proxy.exchangeProto(sendProto);
assertEquals(sendProto, retProto);
assertNotSame(sendProto, retProto);
boolean caught = false;
try {
proxy.error();
} catch (IOException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Caught " + e);
}
caught = true;
}
assertTrue(caught);
proxy.testServerGet();
// create multiple threads and make them do large data transfers
System.out.println("Starting multi-threaded RPC test...");
server.setSocketSendBufSize(1024);
Thread threadId[] = new Thread[numThreads];
for (int i = 0; i < numThreads; i++) {
Transactions trans = new Transactions(proxy, datasize);
threadId[i] = new Thread(trans, "TransactionThread-" + i);
threadId[i].start();
}
// wait for all transactions to get over
System.out.println("Waiting for all threads to finish RPCs...");
for (int i = 0; i < numThreads; i++) {
try {
threadId[i].join();
} catch (InterruptedException e) {
i--; // retry
}
}
} finally {
server.stop();
if (proxy != null) RPC.stopProxy(proxy);
}
}
@Override
public void destroy() {
RPC.stopProxy(proxy);
}
