public void process(WatchedEvent event) {
if (event.getType() == Event.EventType.NodeChildrenChanged) {
LOG.debug("Running children changed [" + event.getPath() + "]");
try {
getZkRunning();
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
}
} else if (event.getType() == Event.EventType.NodeDeleted) {
String znodePath = event.getPath();
LOG.debug("Running znode deleted [" + znodePath + "]");
try {
restartServer(znodePath);
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
}
}
}
@Override
public ScriptContext call() throws Exception {
try {
Scanner scn = new Scanner(znodePath);
scn.useDelimiter(":");
String hostName = scn.next(); // host name
String instance = scn.next(); // instance
int infoPort = Integer.parseInt(scn.next()); // UI info port
long serverStartTimestamp = Long.parseLong(scn.next());
scn.close();
// Get the --config property from classpath...it's always first
// in the classpath
String cp = System.getProperty("java.class.path");
scn = new Scanner(cp);
scn.useDelimiter(":");
String confDir = scn.next();
scn.close();
LOG.debug("conf dir [" + confDir + "]");
// Get -Dwms.home.dir
String wmsHome = System.getProperty("wms.home.dir");
// If stop-wms.sh is executed and WMS_MANAGES_ZK then zookeeper
// is stopped abruptly.
// Second scenario is when ZooKeeper fails for some reason
// regardless of whether WMS
// manages it. When either happens the WmsServer running znodes
// still exist in ZooKeeper
// and we see them at next startup. When they eventually timeout
// we get node deleted events for a server that no longer
// exists. So, only recognize
// WmsServer running znodes that have timestamps after last
// WmsMaster startup.
if (serverStartTimestamp > startupTimestamp) {
scriptContext.setHostName(hostName);
scriptContext.setScriptName("sys_shell.py");
if (hostName.equalsIgnoreCase(ia.getCanonicalHostName()))
scriptContext.setCommand(
"bin/wms-daemon.sh --config " + confDir + " start server " + instance);
else
scriptContext.setCommand(
"pdsh -w "
+ hostName
+ " \"cd "
+ wmsHome
+ ";bin/wms-daemon.sh --config "
+ confDir
+ " start server "
+ instance
+ "\"");
RetryCounter retryCounter = retryCounterFactory.create();
while (true) {
if (scriptContext.getStdOut().length() > 0)
scriptContext.getStdOut().delete(0, scriptContext.getStdOut().length());
if (scriptContext.getStdErr().length() > 0)
scriptContext.getStdErr().delete(0, scriptContext.getStdErr().length());
LOG.info(
"Restarting WmsServer ["
+ hostName
+ ":"
+ instance
+ "], script [ "
+ scriptContext.toString()
+ " ]");
ScriptManager.getInstance().runScript(scriptContext);
if (scriptContext.getExitCode() == 0) {
LOG.info("WmsServer [" + hostName + ":" + instance + "] restarted");
break;
} else {
StringBuilder sb = new StringBuilder();
sb.append("exit code [" + scriptContext.getExitCode() + "]");
if (!scriptContext.getStdOut().toString().isEmpty())
sb.append(", stdout [" + scriptContext.getStdOut().toString() + "]");
if (!scriptContext.getStdErr().toString().isEmpty())
sb.append(", stderr [" + scriptContext.getStdErr().toString() + "]");
LOG.error(sb.toString());
if (!retryCounter.shouldRetry()) {
LOG.error(
"WmsServer ["
+ hostName
+ ":"
+ instance
+ "] restart failed after "
+ retryCounter.getMaxRetries()
+ " retries");
break;
} else {
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
}
} else {
LOG.debug(
"No restart for "
+ znodePath
+ "\nbecause WmsServer start time ["
+ DateFormat.getDateTimeInstance().format(new Date(serverStartTimestamp))
+ "] was before WmsMaster start time ["
+ DateFormat.getDateTimeInstance().format(new Date(startupTimestamp))
+ "]");
}
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
}
return scriptContext;
}
@Override
public Writable call(
org.apache.hadoop.ipc.RPC.Server server,
String protocolName,
Writable rpcRequest,
long receivedTime)
throws IOException {
try {
Invocation call = (Invocation) rpcRequest;
if (server.verbose) log("Call: " + call);
// Verify rpc version
if (call.getRpcVersion() != writableRpcVersion) {
// Client is using a different version of WritableRpc
throw new IOException(
"WritableRpc version mismatch, client side version="
+ call.getRpcVersion()
+ ", server side version="
+ writableRpcVersion);
}
long clientVersion = call.getProtocolVersion();
final String protoName;
ProtoClassProtoImpl protocolImpl;
if (call.declaringClassProtocolName.equals(VersionedProtocol.class.getName())) {
// VersionProtocol methods are often used by client to figure out
// which version of protocol to use.
//
// Versioned protocol methods should go the protocolName protocol
// rather than the declaring class of the method since the
// the declaring class is VersionedProtocol which is not
// registered directly.
// Send the call to the highest protocol version
VerProtocolImpl highest =
server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protocolName);
if (highest == null) {
throw new IOException("Unknown protocol: " + protocolName);
}
protocolImpl = highest.protocolTarget;
} else {
protoName = call.declaringClassProtocolName;
// Find the right impl for the protocol based on client version.
ProtoNameVer pv = new ProtoNameVer(call.declaringClassProtocolName, clientVersion);
protocolImpl = server.getProtocolImplMap(RPC.RpcKind.RPC_WRITABLE).get(pv);
if (protocolImpl == null) { // no match for Protocol AND Version
VerProtocolImpl highest =
server.getHighestSupportedProtocol(RPC.RpcKind.RPC_WRITABLE, protoName);
if (highest == null) {
throw new IOException("Unknown protocol: " + protoName);
} else { // protocol supported but not the version that client wants
throw new RPC.VersionMismatch(protoName, clientVersion, highest.version);
}
}
}
// Invoke the protocol method
long startTime = Time.now();
Method method =
protocolImpl.protocolClass.getMethod(
call.getMethodName(), call.getParameterClasses());
method.setAccessible(true);
server.rpcDetailedMetrics.init(protocolImpl.protocolClass);
Object value = method.invoke(protocolImpl.protocolImpl, call.getParameters());
int processingTime = (int) (Time.now() - startTime);
int qTime = (int) (startTime - receivedTime);
if (LOG.isDebugEnabled()) {
LOG.debug(
"Served: "
+ call.getMethodName()
+ " queueTime= "
+ qTime
+ " procesingTime= "
+ processingTime);
}
server.rpcMetrics.addRpcQueueTime(qTime);
server.rpcMetrics.addRpcProcessingTime(processingTime);
server.rpcDetailedMetrics.addProcessingTime(call.getMethodName(), processingTime);
if (server.verbose) log("Return: " + value);
return new ObjectWritable(method.getReturnType(), value);
} catch (InvocationTargetException e) {
Throwable target = e.getTargetException();
if (target instanceof IOException) {
throw (IOException) target;
} else {
IOException ioe = new IOException(target.toString());
ioe.setStackTrace(target.getStackTrace());
throw ioe;
}
} catch (Throwable e) {
if (!(e instanceof IOException)) {
LOG.error("Unexpected throwable object ", e);
}
IOException ioe = new IOException(e.toString());
ioe.setStackTrace(e.getStackTrace());
throw ioe;
}
}