@Override
public Boolean call() throws Exception {
long timeoutMillis = 5000;
try {
getServersFile();
getZkRunning();
while (true) {
while (!restartQueue.isEmpty()) {
LOG.debug("Restart queue size [" + restartQueue.size() + "]");
RestartHandler handler = restartQueue.poll();
Future<ScriptContext> runner = pool.submit(handler);
ScriptContext scriptContext = runner.get(); // blocking call
if (scriptContext.getExitCode() != 0) restartQueue.add(handler);
}
try {
Thread.sleep(timeoutMillis);
} catch (InterruptedException e) {
}
}
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
pool.shutdown();
throw e;
}
}
public ServerManager(WmsMaster wmsMaster) throws Exception {
try {
this.wmsMaster = wmsMaster;
this.conf = wmsMaster.getConfiguration();
this.zkc = wmsMaster.getZkClient();
this.ia = wmsMaster.getInetAddress();
this.startupTimestamp = wmsMaster.getStartTime();
this.metrics = wmsMaster.getMetrics();
maxRestartAttempts =
conf.getInt(
Constants.WMS_MASTER_SERVER_RESTART_HANDLER_ATTEMPTS,
Constants.DEFAULT_WMS_MASTER_SERVER_RESTART_HANDLER_ATTEMPTS);
retryIntervalMillis =
conf.getInt(
Constants.WMS_MASTER_SERVER_RESTART_HANDLER_RETRY_INTERVAL_MILLIS,
Constants.DEFAULT_WMS_MASTER_SERVER_RESTART_HANDLER_RETRY_INTERVAL_MILLIS);
retryCounterFactory = new RetryCounterFactory(maxRestartAttempts, retryIntervalMillis);
parentZnode =
conf.get(Constants.ZOOKEEPER_ZNODE_PARENT, Constants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
pool = Executors.newSingleThreadExecutor();
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
throw e;
}
}
public void setWatchWorkers() {
try {
List<String> workerList = zooKeeper.getChildren(ZK_WORKER, zkWatcher);
for (String worker : workerList) {
zooKeeper.exists(ZK_WORKER + "/" + worker, zkWatcher);
}
} catch (Exception e) {
e.printStackTrace();
}
}
private static boolean unlock(String lock) {
logger.info("Releasing lock " + lock);
try {
zkInstance.delete(lock, -1);
} catch (Exception E) {
logger.debug("Error releasing lock: " + E.toString());
return true;
}
return true;
}
private static String lock(String lock) {
String realPath = "";
String parent = "/lock";
String lockName = parent + "/" + lock;
logger.debug("Getting lock " + lockName);
try {
if (zkInstance.exists(parent, false) == null)
zkInstance.create(parent, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.fromFlag(0));
} catch (Exception E) {
logger.error("Error creating lock node: " + E.toString());
return null;
}
List<String> children = new LinkedList<String>();
try {
// List <ACL> ACLList = zkInstance.getACL(lockName, zkInstance.exists(lock, false));
realPath =
zkInstance.create(
lockName, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
// children = zkInstance.getChildren(realPath, false);
checkLock:
while (true) {
children = zkInstance.getChildren(parent, false);
for (String curChild : children) {
String child = parent + "/" + curChild;
// System.out.println(child + " " + realPath + " " +
// Integer.toString(child.compareTo(realPath)));
if (child.compareTo(realPath) < 0
&& child.length() == realPath.length()
&& curChild.startsWith(lock)) {
// System.out.println(child + " cmp to " + realPath);
Thread.sleep(300);
continue checkLock;
}
}
logger.info("Got lock " + lockName);
return realPath;
}
} catch (Exception E) {
logger.error("Exception while trying to get lock " + lockName + " :" + E.toString());
E.printStackTrace();
return null;
}
}
/*
* public synchronized ArrayList<Request> getWorkloadsList() {
* ArrayList<Request> workloads = new ArrayList<Request>();
*
* LOG.debug("Reading " + parentZnode +
* Constants.DEFAULT_ZOOKEEPER_ZNODE_WORKLOADS);
*
* try { List<String> children = getChildren(parentZnode +
* Constants.DEFAULT_ZOOKEEPER_ZNODE_WORKLOADS, null);
*
* if (!children.isEmpty()) { for (String child : children) { Request
* request = new Request(); String workloadZnode = parentZnode +
* Constants.DEFAULT_ZOOKEEPER_ZNODE_WORKLOADS + "/" + child; Stat stat =
* zkc.exists(workloadZnode, false); if (stat != null) { byte[] bytes =
* zkc.getData(workloadZnode, false, stat); try {
* deserializer.deserialize(request, bytes); workloads.add(request); } catch
* (TException e) { e.printStackTrace(); } } } } } catch (Exception e) {
* e.printStackTrace(); }
*
* return workloads; }
*/
public synchronized ArrayList<String> getClientsList() {
ArrayList<String> clients = new ArrayList<String>();
LOG.debug("Reading " + parentZnode + Constants.DEFAULT_ZOOKEEPER_ZNODE_CLIENTS);
try {
List<String> children =
getChildren(parentZnode + Constants.DEFAULT_ZOOKEEPER_ZNODE_CLIENTS, null);
if (!children.isEmpty()) {
for (String child : children) {
clients.add(child);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return clients;
}
@Subscribe
public void handleJob(JobPacket jobPacket) throws Exception {
JobPacket packetToClient = new JobPacket();
if (jobPacket.type == JobPacket.JOB_REQ) {
jobQueue.add(jobPacket.hash);
packetToClient.type = JobPacket.JOB_ACCEPTED;
packetToClient.result = "none";
}
if (jobPacket.type == JobPacket.JOB_STATUS) {
// check under /result/<hash>
try {
if ((zooKeeper.exists(Joiner.on("/").join(ZK_JOBS, jobPacket.hash), false) != null)
&& (zooKeeper.exists(Joiner.on("/").join(ZK_RESULT, jobPacket.hash), false) == null)) {
System.out.println("Job in progress, please wait!");
packetToClient.type = JobPacket.JOB_PROGRESS;
packetToClient.result = "none";
}
if ((zooKeeper.exists(Joiner.on("/").join(ZK_JOBS, jobPacket.hash), false) == null)
&& (zooKeeper.exists(Joiner.on("/").join(ZK_RESULT, jobPacket.hash), false) == null)) {
System.out.println("No such Job, please enter your job again!");
packetToClient.type = JobPacket.JOB_NOTFOUND;
packetToClient.result = "none";
}
if (zooKeeper.exists(Joiner.on("/").join(ZK_RESULT, jobPacket.hash), false) != null) {
byte[] data =
zooKeeper.getData(Joiner.on("/").join(ZK_RESULT, jobPacket.hash), false, null);
packetToClient.type = JobPacket.JOB_RESULT;
if (data == null) {
System.out.println("Result not found!");
packetToClient.result = null;
} else {
String result = new String(data);
packetToClient.result = result;
System.out.println("Result found!");
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
socket.send(SerializationUtils.serialize(packetToClient), 0);
}
public void process(WatchedEvent event) {
if (event.getType() == Event.EventType.NodeChildrenChanged) {
LOG.debug("Running children changed [" + event.getPath() + "]");
try {
getZkRunning();
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
}
} else if (event.getType() == Event.EventType.NodeDeleted) {
String znodePath = event.getPath();
LOG.debug("Running znode deleted [" + znodePath + "]");
try {
restartServer(znodePath);
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
}
}
}
public static void main(String[] args) {
String myID = null;
if (args.length == 4) {
try {
zooHost = args[0];
zooPort = Integer.parseInt(args[1]);
myPort = Integer.parseInt(args[2]);
myID = args[3];
} catch (Exception e) {
e.printStackTrace();
}
} else {
System.err.println("Usage tracker [zooHost] [zooPort] [myPort] [myID]");
System.exit(-1);
}
// initialize ZMQ
context = ZMQ.context(1);
socket = context.socket(ZMQ.REP);
socket.bind("tcp://*:" + myPort);
eventBus = new EventBus("Tracker");
JobTracker t = new JobTracker(myID);
eventBus.register(t);
System.out.println("Starting thread");
new Thread(t.manageWorker()).start();
while (true) {
// wait for client req then respond
JobPacket packetFromServer = (JobPacket) SerializationUtils.deserialize(socket.recv(0));
System.out.println("From client" + packetFromServer.type);
eventBus.post(packetFromServer);
}
}
@Override
public ScriptContext call() throws Exception {
try {
Scanner scn = new Scanner(znodePath);
scn.useDelimiter(":");
String hostName = scn.next(); // host name
String instance = scn.next(); // instance
int infoPort = Integer.parseInt(scn.next()); // UI info port
long serverStartTimestamp = Long.parseLong(scn.next());
scn.close();
// Get the --config property from classpath...it's always first
// in the classpath
String cp = System.getProperty("java.class.path");
scn = new Scanner(cp);
scn.useDelimiter(":");
String confDir = scn.next();
scn.close();
LOG.debug("conf dir [" + confDir + "]");
// Get -Dwms.home.dir
String wmsHome = System.getProperty("wms.home.dir");
// If stop-wms.sh is executed and WMS_MANAGES_ZK then zookeeper
// is stopped abruptly.
// Second scenario is when ZooKeeper fails for some reason
// regardless of whether WMS
// manages it. When either happens the WmsServer running znodes
// still exist in ZooKeeper
// and we see them at next startup. When they eventually timeout
// we get node deleted events for a server that no longer
// exists. So, only recognize
// WmsServer running znodes that have timestamps after last
// WmsMaster startup.
if (serverStartTimestamp > startupTimestamp) {
scriptContext.setHostName(hostName);
scriptContext.setScriptName("sys_shell.py");
if (hostName.equalsIgnoreCase(ia.getCanonicalHostName()))
scriptContext.setCommand(
"bin/wms-daemon.sh --config " + confDir + " start server " + instance);
else
scriptContext.setCommand(
"pdsh -w "
+ hostName
+ " \"cd "
+ wmsHome
+ ";bin/wms-daemon.sh --config "
+ confDir
+ " start server "
+ instance
+ "\"");
RetryCounter retryCounter = retryCounterFactory.create();
while (true) {
if (scriptContext.getStdOut().length() > 0)
scriptContext.getStdOut().delete(0, scriptContext.getStdOut().length());
if (scriptContext.getStdErr().length() > 0)
scriptContext.getStdErr().delete(0, scriptContext.getStdErr().length());
LOG.info(
"Restarting WmsServer ["
+ hostName
+ ":"
+ instance
+ "], script [ "
+ scriptContext.toString()
+ " ]");
ScriptManager.getInstance().runScript(scriptContext);
if (scriptContext.getExitCode() == 0) {
LOG.info("WmsServer [" + hostName + ":" + instance + "] restarted");
break;
} else {
StringBuilder sb = new StringBuilder();
sb.append("exit code [" + scriptContext.getExitCode() + "]");
if (!scriptContext.getStdOut().toString().isEmpty())
sb.append(", stdout [" + scriptContext.getStdOut().toString() + "]");
if (!scriptContext.getStdErr().toString().isEmpty())
sb.append(", stderr [" + scriptContext.getStdErr().toString() + "]");
LOG.error(sb.toString());
if (!retryCounter.shouldRetry()) {
LOG.error(
"WmsServer ["
+ hostName
+ ":"
+ instance
+ "] restart failed after "
+ retryCounter.getMaxRetries()
+ " retries");
break;
} else {
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
}
} else {
LOG.debug(
"No restart for "
+ znodePath
+ "\nbecause WmsServer start time ["
+ DateFormat.getDateTimeInstance().format(new Date(serverStartTimestamp))
+ "] was before WmsMaster start time ["
+ DateFormat.getDateTimeInstance().format(new Date(startupTimestamp))
+ "]");
}
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
}
return scriptContext;
}
public JobTracker(String myID) {
// connect zooKeeper client zk server
try {
zkWatcher = new ZkWatcher();
zkConnected = new CountDownLatch(1);
zooKeeper =
new ZooKeeper(
zooHost + ":" + zooPort,
ZK_TIMEOUT,
new Watcher() {
@Override
public void process(WatchedEvent event) {
/* Release Lock if ZooKeeper is connected */
if (event.getState() == Event.KeeperState.SyncConnected) {
zkConnected.countDown();
} else {
System.err.println("Could not connect to ZooKeeper!");
System.exit(0);
}
}
});
zkConnected.await();
// Create /jobs if it doesn't exists
if (zooKeeper.exists(ZK_JOBS, false) == null) {
zooKeeper.create(ZK_JOBS, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
// Create /result if it doesn't exists, to stores already processed jobs with results
if (zooKeeper.exists(ZK_RESULT, false) == null) {
zooKeeper.create(ZK_RESULT, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}
// if /tracker does not exists, create one
if (zooKeeper.exists(ZK_TRACKER, false) == null) {
zooKeeper.create(
ZK_TRACKER,
Joiner.on(":").join(InetAddress.getLocalHost().getHostAddress(), myPort).getBytes(),
ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
// create myself as leader
zooKeeper.create(
Joiner.on("/").join(ZK_TRACKER, myID),
"primary".getBytes(),
ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL_SEQUENTIAL);
} else {
// if no children then create myself as leader
System.out.println("tracker has no children");
if (zooKeeper.getChildren(ZK_TRACKER, zkWatcher, null).isEmpty()) {
// create myself as leader and update data of /tracker
zooKeeper.setData(
ZK_TRACKER,
Joiner.on(":").join(InetAddress.getLocalHost().getHostAddress(), myPort).getBytes(),
-1);
zooKeeper.create(
Joiner.on("/").join(ZK_TRACKER, myID),
"primary".getBytes(),
ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL_SEQUENTIAL);
}
/* else if there is a child then
become the backup */
else {
// set watch on "primary"
zooKeeper.exists(
Joiner.on("/").join(ZK_TRACKER, zooKeeper.getChildren(ZK_TRACKER, zkWatcher).get(0)),
zkWatcher);
zooKeeper.create(
Joiner.on("/").join(ZK_TRACKER, myID),
"backup".getBytes(),
ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL_SEQUENTIAL);
}
}
// set watch at /worker
zooKeeper.getChildren(ZK_WORKER, zkWatcher);
zooKeeper.getData(ZK_WORKER, zkWatcher, null);
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public void process(WatchedEvent event) {
Event.EventType type = event.getType();
String path = event.getPath();
System.out.println("Path: " + path + ", Event type:" + type);
// set watch on workers
setWatchWorkers();
switch (type) {
case NodeDeleted:
try {
// Check if node deleted is from /tracker
if (path.contains(ZK_TRACKER)) {
String node = zooKeeper.getChildren(ZK_TRACKER, false).get(0);
// If primary is dead then I become primary
if (!zooKeeper
.getData(Joiner.on("/").join(ZK_TRACKER, node), false, null)
.equals("primary")) {
zooKeeper.setData(Joiner.on("/").join(ZK_TRACKER, node), "primary".getBytes(), -1);
zooKeeper.setData(
ZK_TRACKER,
Joiner.on(":")
.join(InetAddress.getLocalHost().getHostAddress(), myPort)
.getBytes(),
-1);
}
}
if (path.contains(ZK_WORKER)) {
List<String> currJobs = zooKeeper.getChildren(ZK_JOBS, false);
// /worker/<id>
String workerId = path.split("/")[2];
System.out.println("Dead worker id " + workerId);
for (String job : currJobs) {
while (true) {
String currData =
new String(zooKeeper.getData(Joiner.on("/").join(ZK_JOBS, job), false, null));
int currVersion =
zooKeeper.exists(Joiner.on("/").join(ZK_JOBS, job), false).getVersion();
System.out.println("Version curr" + currVersion);
// de-serialize
WorkerInfo workerInfo = gson.fromJson(currData, WorkerInfo.class);
HashMap<String, List<Integer>> newMap = workerInfo.getWorkerInfo();
List<Integer> deadWorkerList = newMap.get(workerId);
System.out.println("dead worker List" + deadWorkerList);
if (deadWorkerList == null) {
break;
}
// remove it from current info as its dead
newMap.remove(workerId);
List<String> currWorker = zooKeeper.getChildren(ZK_WORKER, false);
// Pick a worker ramdomly and assign the task to it by adding to its current task
// list
int rand = randGen.nextInt(currWorker.size());
List<Integer> newWorkerList = newMap.get(currWorker.get(rand));
if (newWorkerList == null) newWorkerList = deadWorkerList;
else newWorkerList.addAll(deadWorkerList);
newMap.put(currWorker.get(rand), newWorkerList);
WorkerInfo newWorkerInfo = new WorkerInfo(newMap, job);
// serialize
String newData = gson.toJson(newWorkerInfo);
// setdata on the znode /jobs/<hash>
try {
if (zooKeeper.setData(
Joiner.on("/").join(ZK_JOBS, job), newData.getBytes(), currVersion)
!= null) {
System.out.println(
"Update data for " + workerId + " on to " + currWorker.get(rand));
break;
}
} catch (KeeperException e) {
// Ignore
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
break;
}
}