public static String stateToString(ManagedCluster cluster) {
StringBuilder buf = new StringBuilder("\n");
for (HighlyAvailableGraphDatabase database : cluster.getAllMembers()) {
ClusterClient client =
database.getDependencyResolver().resolveDependency(ClusterClient.class);
buf.append("Instance ")
.append(client.getServerId())
.append(":State ")
.append(database.getInstanceState())
.append(" (")
.append(client.getClusterServer())
.append("):")
.append("\n");
ClusterMembers members =
database.getDependencyResolver().resolveDependency(ClusterMembers.class);
for (ClusterMember clusterMember : members.getMembers()) {
buf.append(" ")
.append(clusterMember.getInstanceId())
.append(":")
.append(clusterMember.getHARole())
.append(" (is alive = ")
.append(clusterMember.isAlive())
.append(")")
.append("\n");
}
}
return buf.toString();
}
@Override
protected RemoteTxHook createTxHook() {
clusterEventsDelegateInvocationHandler = new DelegateInvocationHandler();
memberContextDelegateInvocationHandler = new DelegateInvocationHandler();
clusterMemberAvailabilityDelegateInvocationHandler = new DelegateInvocationHandler();
clusterEvents =
(ClusterMemberEvents)
Proxy.newProxyInstance(
ClusterMemberEvents.class.getClassLoader(),
new Class[] {ClusterMemberEvents.class, Lifecycle.class},
clusterEventsDelegateInvocationHandler);
memberContext =
(HighAvailabilityMemberContext)
Proxy.newProxyInstance(
HighAvailabilityMemberContext.class.getClassLoader(),
new Class[] {HighAvailabilityMemberContext.class},
memberContextDelegateInvocationHandler);
clusterMemberAvailability =
(ClusterMemberAvailability)
Proxy.newProxyInstance(
ClusterMemberAvailability.class.getClassLoader(),
new Class[] {ClusterMemberAvailability.class},
clusterMemberAvailabilityDelegateInvocationHandler);
ElectionCredentialsProvider electionCredentialsProvider =
config.get(HaSettings.slave_only)
? new NotElectableElectionCredentialsProvider()
: new DefaultElectionCredentialsProvider(
config.get(ClusterSettings.server_id),
new OnDiskLastTxIdGetter(new File(getStoreDir())),
new HighAvailabilityMemberInfoProvider() {
@Override
public HighAvailabilityMemberState getHighAvailabilityMemberState() {
return memberStateMachine.getCurrentState();
}
});
ObjectStreamFactory objectStreamFactory = new ObjectStreamFactory();
clusterClient =
new ClusterClient(
ClusterClient.adapt(config),
logging,
electionCredentialsProvider,
objectStreamFactory,
objectStreamFactory);
PaxosClusterMemberEvents localClusterEvents =
new PaxosClusterMemberEvents(
clusterClient,
clusterClient,
clusterClient,
clusterClient,
logging,
new Predicate<PaxosClusterMemberEvents.ClusterMembersSnapshot>() {
@Override
public boolean accept(PaxosClusterMemberEvents.ClusterMembersSnapshot item) {
for (MemberIsAvailable member : item.getCurrentAvailableMembers()) {
if (member.getRoleUri().getScheme().equals("ha")) {
if (HighAvailabilityModeSwitcher.getServerId(member.getRoleUri())
== config.get(ClusterSettings.server_id)) {
msgLog.error(
String.format(
"Instance %s has the same serverId as ours (%d) - will not "
+ "join this cluster",
member.getRoleUri(), config.get(ClusterSettings.server_id)));
return true;
}
}
}
return true;
}
},
new HANewSnapshotFunction(),
objectStreamFactory,
objectStreamFactory);
// Force a reelection after we enter the cluster
// and when that election is finished refresh the snapshot
clusterClient.addClusterListener(
new ClusterListener.Adapter() {
boolean hasRequestedElection =
true; // This ensures that the election result is (at least) from our
// request or thereafter
@Override
public void enteredCluster(ClusterConfiguration clusterConfiguration) {
clusterClient.performRoleElections();
}
@Override
public void elected(String role, InstanceId instanceId, URI electedMember) {
if (hasRequestedElection && role.equals(ClusterConfiguration.COORDINATOR)) {
clusterClient.removeClusterListener(this);
}
}
});
HighAvailabilityMemberContext localMemberContext =
new SimpleHighAvailabilityMemberContext(clusterClient.getServerId());
PaxosClusterMemberAvailability localClusterMemberAvailability =
new PaxosClusterMemberAvailability(
clusterClient.getServerId(),
clusterClient,
clusterClient,
logging,
objectStreamFactory,
objectStreamFactory);
memberContextDelegateInvocationHandler.setDelegate(localMemberContext);
clusterEventsDelegateInvocationHandler.setDelegate(localClusterEvents);
clusterMemberAvailabilityDelegateInvocationHandler.setDelegate(localClusterMemberAvailability);
members =
new ClusterMembers(
clusterClient,
clusterClient,
clusterEvents,
new InstanceId(config.get(ClusterSettings.server_id)));
memberStateMachine =
new HighAvailabilityMemberStateMachine(
memberContext,
availabilityGuard,
members,
clusterEvents,
clusterClient,
logging.getMessagesLog(HighAvailabilityMemberStateMachine.class));
HighAvailabilityConsoleLogger highAvailabilityConsoleLogger =
new HighAvailabilityConsoleLogger(
logging.getConsoleLog(HighAvailabilityConsoleLogger.class),
new InstanceId(config.get(ClusterSettings.server_id)));
availabilityGuard.addListener(highAvailabilityConsoleLogger);
clusterEvents.addClusterMemberListener(highAvailabilityConsoleLogger);
clusterClient.addClusterListener(highAvailabilityConsoleLogger);
paxosLife.add(clusterClient);
paxosLife.add(memberStateMachine);
paxosLife.add(clusterEvents);
paxosLife.add(localClusterMemberAvailability);
DelegateInvocationHandler<RemoteTxHook> txHookDelegate = new DelegateInvocationHandler<>();
RemoteTxHook txHook =
(RemoteTxHook)
Proxy.newProxyInstance(
RemoteTxHook.class.getClassLoader(),
new Class[] {RemoteTxHook.class},
txHookDelegate);
new TxHookModeSwitcher(
memberStateMachine,
txHookDelegate,
master,
new TxHookModeSwitcher.RequestContextFactoryResolver() {
@Override
public RequestContextFactory get() {
return requestContextFactory;
}
},
logging.getMessagesLog(TxHookModeSwitcher.class),
dependencyResolver);
return txHook;
}
private void startMember(InstanceId serverId) throws URISyntaxException, IOException {
Clusters.Member member = spec.getMembers().get(serverId.toIntegerIndex() - 1);
StringBuilder initialHosts = new StringBuilder(spec.getMembers().get(0).getHost());
for (int i = 1; i < spec.getMembers().size(); i++) {
initialHosts.append(",").append(spec.getMembers().get(i).getHost());
}
File parent = new File(root, name);
URI clusterUri = new URI("cluster://" + member.getHost());
if (member.isFullHaMember()) {
int clusterPort = clusterUri.getPort();
int haPort = clusterUri.getPort() + 3000;
File storeDir = new File(parent, "server" + serverId);
if (storeDirInitializer != null) {
storeDirInitializer.initializeStoreDir(serverId.toIntegerIndex(), storeDir);
}
GraphDatabaseBuilder builder =
dbFactory.newHighlyAvailableDatabaseBuilder(storeDir.getAbsolutePath());
builder.setConfig(ClusterSettings.cluster_name, name);
builder.setConfig(ClusterSettings.initial_hosts, initialHosts.toString());
builder.setConfig(ClusterSettings.server_id, serverId + "");
builder.setConfig(ClusterSettings.cluster_server, "0.0.0.0:" + clusterPort);
builder.setConfig(HaSettings.ha_server, ":" + haPort);
builder.setConfig(OnlineBackupSettings.online_backup_enabled, Settings.FALSE);
builder.setConfig(commonConfig);
if (instanceConfig.containsKey(serverId.toIntegerIndex())) {
builder.setConfig(instanceConfig.get(serverId.toIntegerIndex()));
}
config(builder, name, serverId);
final HighlyAvailableGraphDatabaseProxy graphDatabase =
new HighlyAvailableGraphDatabaseProxy(builder);
members.put(serverId, graphDatabase);
life.add(
new LifecycleAdapter() {
@Override
public void stop() throws Throwable {
graphDatabase.get().shutdown();
}
});
} else {
Map<String, String> config =
MapUtil.stringMap(
ClusterSettings.cluster_name.name(),
name,
ClusterSettings.initial_hosts.name(),
initialHosts.toString(),
ClusterSettings.server_id.name(),
serverId + "",
ClusterSettings.cluster_server.name(),
"0.0.0.0:" + clusterUri.getPort(),
GraphDatabaseSettings.store_dir.name(),
new File(parent, "arbiter" + serverId).getAbsolutePath());
Config config1 =
new Config(
config,
InternalAbstractGraphDatabase.Configuration.class,
GraphDatabaseSettings.class);
ObjectStreamFactory objectStreamFactory = new ObjectStreamFactory();
ClusterClient clusterClient =
new ClusterClient(
new Monitors(),
ClusterClient.adapt(config1),
NullLogService.getInstance(),
new NotElectableElectionCredentialsProvider(),
objectStreamFactory,
objectStreamFactory);
arbiters.add(
new ClusterMembers(
clusterClient,
clusterClient,
new ClusterMemberEvents() {
@Override
public void addClusterMemberListener(ClusterMemberListener listener) {
// noop
}
@Override
public void removeClusterMemberListener(ClusterMemberListener listener) {
// noop
}
},
clusterClient.getServerId()));
life.add(new FutureLifecycleAdapter<>(clusterClient));
}
}
@Override
protected TxHook createTxHook() {
clusterEventsDelegateInvocationHandler = new DelegateInvocationHandler();
memberContextDelegateInvocationHandler = new DelegateInvocationHandler();
clusterMemberAvailabilityDelegateInvocationHandler = new DelegateInvocationHandler();
clusterEvents =
(ClusterMemberEvents)
Proxy.newProxyInstance(
ClusterMemberEvents.class.getClassLoader(),
new Class[] {ClusterMemberEvents.class, Lifecycle.class},
clusterEventsDelegateInvocationHandler);
memberContext =
(HighAvailabilityMemberContext)
Proxy.newProxyInstance(
HighAvailabilityMemberContext.class.getClassLoader(),
new Class[] {HighAvailabilityMemberContext.class},
memberContextDelegateInvocationHandler);
clusterMemberAvailability =
(ClusterMemberAvailability)
Proxy.newProxyInstance(
ClusterMemberAvailability.class.getClassLoader(),
new Class[] {ClusterMemberAvailability.class},
clusterMemberAvailabilityDelegateInvocationHandler);
/*
* We need to create these anyway since even in compatibility mode we'll use them for switchover. If it turns
* out we are not going to need zookeeper, just assign them to the class fields. The difference is in when
* they start().
*/
ElectionCredentialsProvider electionCredentialsProvider =
config.get(HaSettings.slave_only)
? new NotElectableElectionCredentialsProvider()
: new DefaultElectionCredentialsProvider(
config.get(ClusterSettings.server_id),
new OnDiskLastTxIdGetter(new File(getStoreDir())),
new HighAvailabilityMemberInfoProvider() {
@Override
public HighAvailabilityMemberState getHighAvailabilityMemberState() {
return memberStateMachine.getCurrentState();
}
});
clusterClient =
new ClusterClient(ClusterClient.adapt(config), logging, electionCredentialsProvider);
PaxosClusterMemberEvents localClusterEvents =
new PaxosClusterMemberEvents(
clusterClient,
clusterClient,
clusterClient,
clusterClient,
logging,
new Predicate<PaxosClusterMemberEvents.ClusterMembersSnapshot>() {
@Override
public boolean accept(PaxosClusterMemberEvents.ClusterMembersSnapshot item) {
for (MemberIsAvailable member : item.getCurrentAvailableMembers()) {
if (member.getRoleUri().getScheme().equals("ha")) {
if (HighAvailabilityModeSwitcher.getServerId(member.getRoleUri())
== config.get(ClusterSettings.server_id)) {
msgLog.error(
String.format(
"Instance %s has the same serverId as ours (%d) - will not join this cluster",
member.getRoleUri(), config.get(ClusterSettings.server_id)));
return true;
}
}
}
return true;
}
});
// Force a reelection after we enter the cluster
// and when that election is finished refresh the snapshot
clusterClient.addClusterListener(
new ClusterListener.Adapter() {
@Override
public void enteredCluster(ClusterConfiguration clusterConfiguration) {
clusterClient.performRoleElections();
}
@Override
public void elected(String role, InstanceId instanceId, URI electedMember) {
if (role.equals(ClusterConfiguration.COORDINATOR)) {
clusterClient.refreshSnapshot();
clusterClient.removeClusterListener(this);
}
}
});
HighAvailabilityMemberContext localMemberContext =
new SimpleHighAvailabilityMemberContext(clusterClient.getServerId());
PaxosClusterMemberAvailability localClusterMemberAvailability =
new PaxosClusterMemberAvailability(
clusterClient.getServerId(), clusterClient, clusterClient, logging);
// Here we decide whether to start in compatibility mode or mode or not
if (!config.get(HaSettings.coordinators).isEmpty()
&& !config.get(HaSettings.coordinators).get(0).toString().trim().equals("")) {
compatibilityMode = true;
compatibilityLifecycle = new LinkedList<Lifecycle>();
Switchover switchover =
new ZooToPaxosSwitchover(
life,
paxosLife,
compatibilityLifecycle,
clusterEventsDelegateInvocationHandler,
memberContextDelegateInvocationHandler,
clusterMemberAvailabilityDelegateInvocationHandler,
localClusterEvents,
localMemberContext,
localClusterMemberAvailability);
ZooKeeperHighAvailabilityEvents zkEvents =
new ZooKeeperHighAvailabilityEvents(logging, config, switchover);
compatibilityLifecycle.add(zkEvents);
memberContextDelegateInvocationHandler.setDelegate(
new SimpleHighAvailabilityMemberContext(zkEvents.getInstanceId()));
clusterEventsDelegateInvocationHandler.setDelegate(zkEvents);
clusterMemberAvailabilityDelegateInvocationHandler.setDelegate(zkEvents);
// Paxos Events added to life, won't be stopped because it isn't started yet
paxosLife.add(localClusterEvents);
} else {
memberContextDelegateInvocationHandler.setDelegate(localMemberContext);
clusterEventsDelegateInvocationHandler.setDelegate(localClusterEvents);
clusterMemberAvailabilityDelegateInvocationHandler.setDelegate(
localClusterMemberAvailability);
}
members =
new ClusterMembers(
clusterClient,
clusterClient,
clusterEvents,
new InstanceId(config.get(ClusterSettings.server_id)));
memberStateMachine =
new HighAvailabilityMemberStateMachine(
memberContext,
accessGuard,
members,
clusterEvents,
clusterClient,
logging.getLogger(HighAvailabilityMemberStateMachine.class));
if (compatibilityMode) {
/*
* In here goes stuff that needs to stop when switching. If added in paxosLife too they will be restarted.
* Adding to life starts them when life.start is called - adding them to compatibilityLifeCycle shuts them
* down on switchover
*/
compatibilityLifecycle.add(memberStateMachine);
// compatibilityLifecycle.add( highAvailabilityModeSwitcher );
compatibilityLifecycle.add((Lifecycle) clusterEvents);
life.add(memberStateMachine);
// life.add( highAvailabilityModeSwitcher );
life.add(clusterEvents);
}
/*
* Here goes stuff that needs to start when paxos kicks in:
* In Normal (non compatibility mode): That means they start normally
* In Compatibility Mode: That means they start when switchover happens. If added to life too they will be
* restarted
*/
paxosLife.add(memberStateMachine);
paxosLife.add(clusterEvents);
paxosLife.add(clusterClient);
paxosLife.add(localClusterMemberAvailability);
DelegateInvocationHandler<TxHook> txHookDelegate = new DelegateInvocationHandler<TxHook>();
TxHook txHook =
(TxHook)
Proxy.newProxyInstance(
TxHook.class.getClassLoader(), new Class[] {TxHook.class}, txHookDelegate);
new TxHookModeSwitcher(
memberStateMachine,
txHookDelegate,
master,
new TxHookModeSwitcher.RequestContextFactoryResolver() {
@Override
public RequestContextFactory get() {
return requestContextFactory;
}
},
dependencyResolver);
return txHook;
}