public Result makeRequest(
RemoteInstanceRequestClient client, String methodName, Object... values) {
String remoteResponse;
String errorMessage;
{
try {
URI remoteUri =
UriBuilder.fromPath(getPath())
.scheme(exhibitor.getRestScheme())
.host(hostname)
.port(exhibitor.getRestPort())
.path(ClusterResource.class, methodName)
.build(values);
remoteResponse =
client.getWebResource(remoteUri, MediaType.APPLICATION_JSON_TYPE, String.class);
errorMessage = "";
} catch (Exception e) {
remoteResponse = "{}";
errorMessage = e.getMessage();
if (errorMessage == null) {
errorMessage = "Unknown";
}
}
}
return new Result(remoteResponse, errorMessage);
}
@VisibleForTesting
protected void restartZooKeeper(InstanceState currentInstanceState) throws Exception {
if (currentInstanceState != null) {
currentInstanceState.updateTimestampMs();
}
if (!exhibitor.getControlPanelValues().isSet(ControlPanelTypes.RESTARTS)) {
exhibitor
.getLog()
.add(ActivityLog.Type.INFO, "Restart of ZooKeeper skipped due to control panel setting");
return;
}
exhibitor.getActivityQueue().add(QueueGroups.MAIN, new KillRunningInstance(exhibitor, true));
}
@Override
public void contextInitialized(ServletContextEvent event) {
Map<String, String> argsBuilder = makeArgsBuilder();
try {
exhibitorCreator = new ExhibitorCreator(toArgsArray(argsBuilder));
exhibitor =
new Exhibitor(
exhibitorCreator.getConfigProvider(),
null,
exhibitorCreator.getBackupProvider(),
exhibitorCreator.getBuilder().build());
exhibitor.start();
event
.getServletContext()
.setAttribute(ExhibitorServletContextListener.class.getName(), exhibitor);
} catch (MissingConfigurationTypeException exit) {
log.error(
"Configuration type (" + OUR_PREFIX + ExhibitorCLI.CONFIG_TYPE + ") must be specified");
exit.getCli().logHelp(OUR_PREFIX);
throw new RuntimeException(exit);
} catch (ExhibitorCreatorExit exit) {
if (exit.getError() != null) {
log.error(exit.getError());
}
exit.getCli().logHelp(OUR_PREFIX);
throw new RuntimeException(exit);
} catch (Exception e) {
log.error("Trying to create Exhibitor", e);
throw new RuntimeException(e);
}
}
private String getPath() {
StringBuilder thisPath = new StringBuilder();
if (exhibitor.getRestPath() != null) {
if (!exhibitor.getRestPath().startsWith("/")) {
thisPath.append("/");
}
thisPath.append(exhibitor.getRestPath());
if (!exhibitor.getRestPath().endsWith("/")) {
thisPath.append("/");
}
} else {
thisPath.append("/");
}
thisPath.append(clusterResourcePath);
return thisPath.toString();
}
public void start() {
repeatingActivity.start();
exhibitor
.getConfigManager()
.addConfigListener(
new ConfigListener() {
@Override
public void configUpdated() {
repeatingActivity.setTimePeriodMs(
exhibitor.getConfigManager().getConfig().getInt(IntConfigs.CHECK_MS));
}
});
}
public MonitorRunningInstance(Exhibitor exhibitor) {
this.exhibitor = exhibitor;
Activity activity =
new Activity() {
@Override
public void completed(boolean wasSuccessful) {
// NOP
}
@Override
public Boolean call() throws Exception {
doWork();
return true;
}
};
repeatingActivity =
new RepeatingActivity(
exhibitor.getLog(),
exhibitor.getActivityQueue(),
QueueGroups.MAIN,
activity,
exhibitor.getConfigManager().getConfig().getInt(IntConfigs.CHECK_MS));
}
private int getDownInstanceRestartMs(InstanceConfig config) {
EncodedConfigParser parser =
new EncodedConfigParser(
exhibitor.getConfigManager().getConfig().getString(StringConfigs.ZOO_CFG_EXTRA));
int tickTime = parseInt(parser.getValues().get("tickTime"));
int initLimit = parseInt(parser.getValues().get("initLimit"));
int syncLimit = parseInt(parser.getValues().get("syncLimit"));
if ((tickTime > 0) && ((initLimit > 0) || (syncLimit > 0))) {
return 2
* tickTime
* Math.max(initLimit, syncLimit); // ZK should sync or fail within the initLimit/syncLimit
}
return (config.getInt(IntConfigs.CHECK_MS) * DOWN_RECHECK_FACTOR);
}
@VisibleForTesting
void doWork() throws Exception {
InstanceConfig config = exhibitor.getConfigManager().getConfig();
InstanceState instanceState =
new InstanceState(
new ServerList(config.getString(StringConfigs.SERVERS_SPEC)),
new Checker(exhibitor).calculateState(),
new RestartSignificantConfig(config));
exhibitor.getConfigManager().checkRollingConfig(instanceState);
InstanceState localCurrentInstanceState = currentInstanceState.get();
if (instanceState.equals(localCurrentInstanceState)) {
if ((localCurrentInstanceState.getState() == InstanceStateTypes.DOWN)
|| (localCurrentInstanceState.getState() == InstanceStateTypes.NOT_SERVING)) {
if (!exhibitor.getConfigManager().isRolling()) {
long elapsedMs = System.currentTimeMillis() - localCurrentInstanceState.getTimestampMs();
int downInstanceRestartMs = getDownInstanceRestartMs(config);
if (elapsedMs > downInstanceRestartMs) {
exhibitor
.getLog()
.add(
ActivityLog.Type.INFO,
"Restarting down/not-serving ZooKeeper after " + elapsedMs + " ms pause");
restartZooKeeper(localCurrentInstanceState);
} else {
exhibitor
.getLog()
.add(
ActivityLog.Type.INFO,
"ZooKeeper down/not-serving waiting "
+ elapsedMs
+ " of "
+ downInstanceRestartMs
+ " ms before restarting");
}
}
}
} else {
boolean serverListChange =
(localCurrentInstanceState != null)
&& !localCurrentInstanceState.getServerList().equals(instanceState.getServerList());
boolean configChange =
(localCurrentInstanceState != null)
&& !localCurrentInstanceState
.getCurrentConfig()
.equals(instanceState.getCurrentConfig());
currentInstanceState.set(instanceState);
exhibitor
.getLog()
.add(ActivityLog.Type.INFO, "State: " + instanceState.getState().getDescription());
if (serverListChange) {
exhibitor.getLog().add(ActivityLog.Type.INFO, "Server list has changed");
restartZooKeeper(localCurrentInstanceState);
} else if (configChange) {
exhibitor
.getLog()
.add(ActivityLog.Type.INFO, "ZooKeeper related configuration has changed");
restartZooKeeper(localCurrentInstanceState);
} else {
switch (instanceState.getState()) {
case DOWN:
{
restartZooKeeper(localCurrentInstanceState);
break;
}
default:
{
// nop
break;
}
}
}
}
}
