private void copyStoreFromMaster(URI masterUri) throws Throwable {
FileSystemAbstraction fs = resolver.resolveDependency(FileSystemAbstraction.class);
// Must be called under lock on XaDataSourceManager
LifeSupport life = new LifeSupport();
try {
// Remove the current store - neostore file is missing, nothing we can really do
// stopServicesAndHandleBranchedStore( BranchedDataPolicy.keep_none );
final MasterClient copyMaster = newMasterClient(masterUri, null, life);
life.start();
// This will move the copied db to the graphdb location
console.log("Copying store from master");
new StoreCopyClient(config, kernelExtensions, console, fs)
.copyStore(
new StoreCopyClient.StoreCopyRequester() {
@Override
public Response<?> copyStore(StoreWriter writer) {
return copyMaster.copyStore(
new RequestContext(
0, config.get(ClusterSettings.server_id).toIntegerIndex(), 0, 0, 0, 0),
writer);
}
@Override
public void done() { // Nothing to clean up here
}
});
startServicesAgain();
console.log("Finished copying store from master");
} finally {
life.stop();
}
}
private URI startHaCommunication(
LifeSupport haCommunicationLife,
NeoStoreXaDataSource nioneoDataSource,
URI me,
URI masterUri) {
MasterClient master =
newMasterClient(masterUri, nioneoDataSource.getStoreId(), haCommunicationLife);
Slave slaveImpl =
new SlaveImpl(
nioneoDataSource.getStoreId(), resolver.resolveDependency(UpdatePuller.class));
SlaveServer server =
new SlaveServer(
slaveImpl, serverConfig(), logging, resolver.resolveDependency(Monitors.class));
masterDelegateHandler.setDelegate(master);
haCommunicationLife.add(slaveImpl);
haCommunicationLife.add(server);
haCommunicationLife.start();
URI slaveHaURI = createHaURI(me, server);
clusterMemberAvailability.memberIsAvailable(HighAvailabilityModeSwitcher.SLAVE, slaveHaURI);
return slaveHaURI;
}
public BatchingNeoStore(
FileSystemAbstraction fileSystem,
String storeDir,
Configuration config,
Monitor writeMonitor,
Logging logging,
WriterFactory writerFactory,
Monitors monitors) {
this.config = config;
this.writeMonitor = writeMonitor;
this.writerFactory = writerFactory;
this.monitors = monitors;
this.fileSystem = life.add(new ChannelReusingFileSystemAbstraction(fileSystem));
this.logger = logging.getMessagesLog(getClass());
this.neo4jConfig =
configForStoreDir(
new Config(
stringMap(dense_node_threshold.name(), valueOf(config.denseNodeThreshold())),
GraphDatabaseSettings.class),
new File(storeDir));
this.neoStore = newBatchWritingNeoStore();
this.propertyKeyRepository =
new BatchingPropertyKeyTokenRepository(neoStore.getPropertyKeyTokenStore());
this.labelRepository = new BatchingLabelTokenRepository(neoStore.getLabelTokenStore());
this.relationshipTypeRepository =
new BatchingRelationshipTypeTokenRepository(neoStore.getRelationshipTypeTokenStore());
life.start();
}
/**
* WARNING: beware of hacks.
*
* <p>Fails a member of this cluster by making it not respond to heart beats. A {@link
* RepairKit} is returned which is able to repair the instance (i.e start the network) again.
*
* @param db the {@link HighlyAvailableGraphDatabase} to fail.
* @return a {@link RepairKit} which can repair the failure.
* @throws IllegalArgumentException if the given db isn't a member of this cluster.
*/
public RepairKit fail(HighlyAvailableGraphDatabase db) throws Throwable {
assertMember(db);
ClusterClient clusterClient =
db.getDependencyResolver().resolveDependency(ClusterClient.class);
LifeSupport clusterClientLife =
(LifeSupport)
accessible(clusterClient.getClass().getDeclaredField("life")).get(clusterClient);
NetworkReceiver receiver =
instance(NetworkReceiver.class, clusterClientLife.getLifecycleInstances());
receiver.stop();
ExecutorLifecycleAdapter statemachineExecutor =
instance(ExecutorLifecycleAdapter.class, clusterClientLife.getLifecycleInstances());
statemachineExecutor.stop();
NetworkSender sender =
instance(NetworkSender.class, clusterClientLife.getLifecycleInstances());
sender.stop();
List<Lifecycle> stoppedServices = new ArrayList<>();
stoppedServices.add(sender);
stoppedServices.add(statemachineExecutor);
stoppedServices.add(receiver);
return new StartNetworkAgainKit(db, stoppedServices);
}
@Override
public void init() throws Throwable {
client = new ZooClient(logger.getLogger(ZooClient.class), config);
life.add(client);
client.addZooListener(new ZooHaEventListener());
client.addCompatibilityModeListener(new ZooCompatibilityModeListener());
life.init();
}
private IndexingModule buildIndexing(
Config config,
JobScheduler scheduler,
SchemaIndexProvider indexProvider,
LockService lockService,
TokenNameLookup tokenNameLookup,
Logging logging,
IndexingService.Monitor indexingServiceMonitor,
NeoStore neoStore,
UpdateableSchemaState updateableSchemaState) {
final DefaultSchemaIndexProviderMap providerMap =
new DefaultSchemaIndexProviderMap(indexProvider);
final IndexingService indexingService =
IndexingService.create(
new IndexSamplingConfig(config),
scheduler,
providerMap,
new NeoStoreIndexStoreView(lockService, neoStore),
tokenNameLookup,
updateableSchemaState,
toList(new SchemaStorage(neoStore.getSchemaStore()).allIndexRules()),
logging,
indexingServiceMonitor);
final IntegrityValidator integrityValidator = new IntegrityValidator(neoStore, indexingService);
// TODO Move to constructor
final LabelScanStore labelScanStore =
dependencyResolver
.resolveDependency(
LabelScanStoreProvider.class, LabelScanStoreProvider.HIGHEST_PRIORITIZED)
.getLabelScanStore();
life.add(indexingService);
life.add(labelScanStore);
return new IndexingModule() {
@Override
public IndexingService indexingService() {
return indexingService;
}
@Override
public LabelScanStore labelScanStore() {
return labelScanStore;
}
@Override
public IntegrityValidator integrityValidator() {
return integrityValidator;
}
@Override
public SchemaIndexProviderMap schemaIndexProviderMap() {
return providerMap;
}
};
}
@Override
public void init() {
luceneDataSource = life.add(new LuceneDataSource(config, indexStore, fileSystemAbstraction));
// TODO Don't do this here, do proper life cycle management
life.start();
LuceneIndexImplementation indexImplementation = new LuceneIndexImplementation(luceneDataSource);
indexProviders.registerIndexProvider(
LuceneIndexImplementation.SERVICE_NAME, indexImplementation);
}
@Test
public void shouldSendAMessageFromAClientWhichIsReceivedByAServer() throws Exception {
// given
CountDownLatch latch = new CountDownLatch(1);
LifeSupport life = new LifeSupport();
Server server1 =
new Server(
latch,
MapUtil.stringMap(
ClusterSettings.cluster_server.name(),
"localhost:1234",
ClusterSettings.server_id.name(),
"1",
ClusterSettings.initial_hosts.name(),
"localhost:1234,localhost:1235"));
life.add(server1);
Server server2 =
new Server(
latch,
MapUtil.stringMap(
ClusterSettings.cluster_server.name(),
"localhost:1235",
ClusterSettings.server_id.name(),
"2",
ClusterSettings.initial_hosts.name(),
"localhost:1234,localhost:1235"));
life.add(server2);
life.start();
// when
server1.process(
Message.to(TestMessage.helloWorld, URI.create("cluster://127.0.0.1:1235"), "Hello World"));
// then
latch.await(5, TimeUnit.SECONDS);
assertTrue("server1 should have processed the message", server1.processedMessage());
assertTrue("server2 should have processed the message", server2.processedMessage());
life.shutdown();
}
@Override
public void start() throws Throwable {
Clusters clusters = clustersProvider.clusters();
life = new LifeSupport();
for (int i = 0; i < clusters.getClusters().size(); i++) {
Clusters.Cluster cluster = clusters.getClusters().get(i);
ManagedCluster managedCluster = new ManagedCluster(cluster);
clusterMap.put(cluster.getName(), managedCluster);
life.add(managedCluster);
}
life.start();
}
private NeoStoreModule buildNeoStore(
StoreFactory storeFactory,
final LabelTokenHolder labelTokens,
final RelationshipTypeTokenHolder relationshipTypeTokens,
final PropertyKeyTokenHolder propertyKeyTokenHolder) {
final NeoStore neoStore = storeFactory.newNeoStore(false, true);
life.add(
new LifecycleAdapter() {
@Override
public void start() {
if (startupStatistics.numberOfRecoveredTransactions() > 0) {
neoStore.rebuildIdGenerators();
}
neoStoreModule.neoStore().makeStoreOk();
propertyKeyTokenHolder.addTokens(
neoStoreModule.neoStore().getPropertyKeyTokenStore().getTokens(Integer.MAX_VALUE));
relationshipTypeTokens.addTokens(
neoStoreModule
.neoStore()
.getRelationshipTypeTokenStore()
.getTokens(Integer.MAX_VALUE));
labelTokens.addTokens(
neoStoreModule.neoStore().getLabelTokenStore().getTokens(Integer.MAX_VALUE));
}
});
return new NeoStoreModule() {
@Override
public NeoStore neoStore() {
return neoStore;
}
};
}
/** Public for testing purpose. Do not use. */
public synchronized void registerDataSource(final XaDataSource dataSource) {
dataSources.put(dataSource.getName(), dataSource);
branchIdMapping.put(UTF8.decode(dataSource.getBranchId()), dataSource);
sourceIdMapping.put(dataSource.getName(), dataSource.getBranchId());
life.add(dataSource);
if (life.getStatus().equals(LifecycleStatus.STARTED)) {
Listeners.notifyListeners(
dsRegistrationListeners,
new Listeners.Notification<DataSourceRegistrationListener>() {
@Override
public void notify(DataSourceRegistrationListener listener) {
listener.registeredDataSource(dataSource);
}
});
}
}
@Override
protected IdGeneratorFactory createIdGeneratorFactory() {
idGeneratorFactory = new HaIdGeneratorFactory(master, logging);
highAvailabilityModeSwitcher =
new HighAvailabilityModeSwitcher(
clusterClient,
masterDelegateInvocationHandler,
clusterMemberAvailability,
memberStateMachine,
this,
(HaIdGeneratorFactory) idGeneratorFactory,
config,
logging,
updateableSchemaState,
kernelExtensions.listFactories(),
monitors);
/*
* We always need the mode switcher and we need it to restart on switchover.
*/
paxosLife.add(highAvailabilityModeSwitcher);
/*
* We don't really switch to master here. We just need to initialize the idGenerator so the initial store
* can be started (if required). In any case, the rest of the database is in pending state, so nothing will
* happen until events start arriving and that will set us to the proper state anyway.
*/
((HaIdGeneratorFactory) idGeneratorFactory).switchToMaster();
return idGeneratorFactory;
}
/**
* WARNING: beware of hacks.
*
* <p>Fails a member of this cluster by making it not respond to heart beats. A {@link
* RepairKit} is returned which is able to repair the instance (i.e start the network) again.
*
* @param db the {@link HighlyAvailableGraphDatabase} to fail.
* @return a {@link RepairKit} which can repair the failure.
* @throws IllegalArgumentException if the given db isn't a member of this cluster.
*/
public RepairKit fail(HighlyAvailableGraphDatabase db) throws Throwable {
assertMember(db);
ClusterClient clusterClient =
db.getDependencyResolver().resolveDependency(ClusterClient.class);
LifeSupport clusterClientLife =
(LifeSupport)
accessible(clusterClient.getClass().getDeclaredField("life")).get(clusterClient);
NetworkInstance network =
instance(NetworkInstance.class, clusterClientLife.getLifecycleInstances());
network.stop();
int serverId =
db.getDependencyResolver().resolveDependency(Config.class).get(HaSettings.server_id);
db.shutdown();
return new StartDatabaseAgainKit(this, serverId);
}
@Override
public void shutdown() {
recordAccess.close();
if (isShutdown) {
throw new IllegalStateException("Batch inserter already has shutdown");
}
isShutdown = true;
try {
repopulateAllIndexes();
} catch (IOException e) {
throw new RuntimeException(e);
}
neoStore.close();
try {
storeLocker.release();
} catch (IOException e) {
throw new UnderlyingStorageException("Could not release store lock", e);
}
msgLog.logMessage(
Thread.currentThread() + " Clean shutdown on BatchInserter(" + this + ")", true);
msgLog.close();
life.shutdown();
}
@Override
protected IdGeneratorFactory createIdGeneratorFactory() {
idGeneratorFactory = new HaIdGeneratorFactory(master, memberStateMachine, logging);
highAvailabilityModeSwitcher =
new HighAvailabilityModeSwitcher(
masterDelegateInvocationHandler,
clusterMemberAvailability,
memberStateMachine,
this,
(HaIdGeneratorFactory) idGeneratorFactory,
config,
logging);
/*
* We always need the mode switcher and we need it to restart on switchover. So:
* 1) if in compatibility mode, it must be added in all 3 - to start on start and restart on switchover
* 2) if not in compatibility mode it must be added in paxosLife, which is started anyway.
*/
paxosLife.add(highAvailabilityModeSwitcher);
if (compatibilityMode) {
compatibilityLifecycle.add(1, highAvailabilityModeSwitcher);
life.add(highAvailabilityModeSwitcher);
}
return idGeneratorFactory;
}
@Override
public void start() throws Throwable {
life.start(); // Binds ZooClient so we have a local address to use next
Listeners.notifyListeners(
bindingListeners,
new Listeners.Notification<BindingListener>() {
@Override
public void notify(BindingListener listener) {
listener.listeningAt(URI.create(client.getClusterServer()));
}
});
client.refreshMasterFromZooKeeper();
Listeners.notifyListeners(
haListeners,
new Listeners.Notification<ClusterMemberListener>() {
@Override
public void notify(ClusterMemberListener listener) {
listener.coordinatorIsElected(new InstanceId(client.getCachedMaster().getMachineId()));
}
});
Listeners.notifyListeners(
haListeners,
new Listeners.Notification<ClusterMemberListener>() {
@Override
public void notify(ClusterMemberListener listener) {
listener.memberIsAvailable(
HighAvailabilityModeSwitcher.MASTER,
new InstanceId(client.getCachedMaster().getMachineId()),
URI.create("ha://" + client.getCachedMaster().getServerAsString()));
}
});
}
private Server(final CountDownLatch latch, final Map<String, String> config) {
final Config conf = new Config(config, ClusterSettings.class);
networkReceiver =
life.add(
new NetworkReceiver(
new NetworkReceiver.Configuration() {
@Override
public HostnamePort clusterServer() {
return conf.get(ClusterSettings.cluster_server);
}
},
new DevNullLoggingService()));
networkSender =
life.add(
new NetworkSender(
new NetworkSender.Configuration() {
@Override
public int defaultPort() {
return 5001;
}
@Override
public int port() {
return conf.get(ClusterSettings.cluster_server).getPort();
}
},
networkReceiver,
new DevNullLoggingService()));
life.add(
new LifecycleAdapter() {
@Override
public void start() throws Throwable {
networkReceiver.addMessageProcessor(
new MessageProcessor() {
@Override
public boolean process(Message<? extends MessageType> message) {
// server receives a message
processedMessage.set(true);
latch.countDown();
return true;
}
});
}
});
}
@Override
public void start() throws Throwable {
life = new LifeSupport();
for (XaDataSource ds : dataSources.values()) {
life.add(ds);
}
life.start();
for (DataSourceRegistrationListener listener : dsRegistrationListeners) {
try {
for (XaDataSource ds : dataSources.values()) {
listener.registeredDataSource(ds);
}
} catch (Throwable t) {
msgLog.logMessage("Failed when notifying registering listener", t);
}
}
}
@Override
public void close() {
if (neoStore != null) {
neoStore.close();
neoStore = null;
}
life.shutdown();
}
@Override
public void shutdown() throws Throwable {
dsRegistrationListeners = null;
life.shutdown();
dataSources.clear();
branchIdMapping.clear();
sourceIdMapping.clear();
isShutdown = true;
}
@Override
public void start() throws Throwable {
Clusters clusters = clustersProvider.clusters();
life = new LifeSupport();
// Started so instances added here will be started immediately, and in case of exceptions they
// can be
// shutdown() or stop()ped properly
life.start();
for (int i = 0; i < clusters.getClusters().size(); i++) {
Clusters.Cluster cluster = clusters.getClusters().get(i);
ManagedCluster managedCluster = new ManagedCluster(cluster);
clusterMap.put(cluster.getName(), managedCluster);
life.add(managedCluster);
}
}
@Override
public void stop() {
if (!readOnly) {
forceEverything();
}
life.shutdown();
neoStore.close();
msgLog.info("NeoStore closed");
}
@Override
public synchronized void shutdown() throws Throwable {
modeSwitcherExecutor.shutdown();
modeSwitcherExecutor.awaitTermination(60, TimeUnit.SECONDS);
haCommunicationLife.shutdown();
switchToMaster.close();
switchToMaster = null;
switchToSlave = null;
}
public void addDataSourceRegistrationListener(DataSourceRegistrationListener listener) {
if (life.getStatus().equals(LifecycleStatus.STARTED)) {
try {
for (XaDataSource ds : dataSources.values()) {
listener.registeredDataSource(ds);
}
} catch (Throwable t) {
msgLog.logMessage("Failed when notifying registering listener", t);
}
}
dsRegistrationListeners = Listeners.addListener(listener, dsRegistrationListeners);
}
@Override
public void stop() {
super.stop();
if (!readOnly) {
forceEverything();
}
life.shutdown();
xaContainer.close();
if (logApplied) {
neoStore.rebuildIdGenerators();
logApplied = false;
}
neoStore.close();
msgLog.info("NeoStore closed");
}
@Override
public void stop() {
transactionLogModule.logRotationControl().awaitAllTransactionsClosed();
transactionLogModule.logRotationControl().forceEverything();
/*
* We simply increment the version, essentially "rotating" away
* the current active log file, to avoid having a recovery on
* next startup. Not necessary, simply speeds up the startup
* process.
*/
neoStoreModule.neoStore().incrementAndGetVersion();
life.shutdown();
neoStoreModule.neoStore().close();
msgLog.info("NeoStore closed");
}
/** Public for testing purpose. Do not use. */
public synchronized void unregisterDataSource(String name) {
final XaDataSource dataSource = dataSources.get(name);
if (dataSource == null) {
return;
}
byte branchId[] = getBranchId(dataSource.getXaConnection().getXaResource());
dataSources.remove(name);
branchIdMapping.remove(UTF8.decode(branchId));
sourceIdMapping.remove(name);
Listeners.notifyListeners(
dsRegistrationListeners,
new Listeners.Notification<DataSourceRegistrationListener>() {
@Override
public void notify(DataSourceRegistrationListener listener) {
listener.unregisteredDataSource(dataSource);
}
});
life.remove(dataSource);
// No need for shutdown, removing does that
}
@Override
public void stop() throws Throwable {
life.stop();
}
@Override
public synchronized void stop() throws Throwable {
haCommunicationLife.stop();
}
@Override
public synchronized void init() throws Throwable {
modeSwitcherExecutor = createExecutor();
haCommunicationLife.init();
}