Esempio n. 1
0
  private void copyStoreFromMaster(URI masterUri) throws Throwable {
    FileSystemAbstraction fs = resolver.resolveDependency(FileSystemAbstraction.class);
    // Must be called under lock on XaDataSourceManager
    LifeSupport life = new LifeSupport();
    try {
      // Remove the current store - neostore file is missing, nothing we can really do
      //            stopServicesAndHandleBranchedStore( BranchedDataPolicy.keep_none );
      final MasterClient copyMaster = newMasterClient(masterUri, null, life);
      life.start();

      // This will move the copied db to the graphdb location
      console.log("Copying store from master");
      new StoreCopyClient(config, kernelExtensions, console, fs)
          .copyStore(
              new StoreCopyClient.StoreCopyRequester() {
                @Override
                public Response<?> copyStore(StoreWriter writer) {
                  return copyMaster.copyStore(
                      new RequestContext(
                          0, config.get(ClusterSettings.server_id).toIntegerIndex(), 0, 0, 0, 0),
                      writer);
                }

                @Override
                public void done() { // Nothing to clean up here
                }
              });

      startServicesAgain();
      console.log("Finished copying store from master");
    } finally {
      life.stop();
    }
  }
Esempio n. 2
0
  private URI startHaCommunication(
      LifeSupport haCommunicationLife,
      NeoStoreXaDataSource nioneoDataSource,
      URI me,
      URI masterUri) {
    MasterClient master =
        newMasterClient(masterUri, nioneoDataSource.getStoreId(), haCommunicationLife);

    Slave slaveImpl =
        new SlaveImpl(
            nioneoDataSource.getStoreId(), resolver.resolveDependency(UpdatePuller.class));

    SlaveServer server =
        new SlaveServer(
            slaveImpl, serverConfig(), logging, resolver.resolveDependency(Monitors.class));

    masterDelegateHandler.setDelegate(master);

    haCommunicationLife.add(slaveImpl);
    haCommunicationLife.add(server);
    haCommunicationLife.start();

    URI slaveHaURI = createHaURI(me, server);
    clusterMemberAvailability.memberIsAvailable(HighAvailabilityModeSwitcher.SLAVE, slaveHaURI);

    return slaveHaURI;
  }
Esempio n. 3
0
  public BatchingNeoStore(
      FileSystemAbstraction fileSystem,
      String storeDir,
      Configuration config,
      Monitor writeMonitor,
      Logging logging,
      WriterFactory writerFactory,
      Monitors monitors) {
    this.config = config;
    this.writeMonitor = writeMonitor;
    this.writerFactory = writerFactory;
    this.monitors = monitors;
    this.fileSystem = life.add(new ChannelReusingFileSystemAbstraction(fileSystem));

    this.logger = logging.getMessagesLog(getClass());
    this.neo4jConfig =
        configForStoreDir(
            new Config(
                stringMap(dense_node_threshold.name(), valueOf(config.denseNodeThreshold())),
                GraphDatabaseSettings.class),
            new File(storeDir));

    this.neoStore = newBatchWritingNeoStore();
    this.propertyKeyRepository =
        new BatchingPropertyKeyTokenRepository(neoStore.getPropertyKeyTokenStore());
    this.labelRepository = new BatchingLabelTokenRepository(neoStore.getLabelTokenStore());
    this.relationshipTypeRepository =
        new BatchingRelationshipTypeTokenRepository(neoStore.getRelationshipTypeTokenStore());
    life.start();
  }
Esempio n. 4
0
    /**
     * WARNING: beware of hacks.
     *
     * <p>Fails a member of this cluster by making it not respond to heart beats. A {@link
     * RepairKit} is returned which is able to repair the instance (i.e start the network) again.
     *
     * @param db the {@link HighlyAvailableGraphDatabase} to fail.
     * @return a {@link RepairKit} which can repair the failure.
     * @throws IllegalArgumentException if the given db isn't a member of this cluster.
     */
    public RepairKit fail(HighlyAvailableGraphDatabase db) throws Throwable {
      assertMember(db);
      ClusterClient clusterClient =
          db.getDependencyResolver().resolveDependency(ClusterClient.class);
      LifeSupport clusterClientLife =
          (LifeSupport)
              accessible(clusterClient.getClass().getDeclaredField("life")).get(clusterClient);

      NetworkReceiver receiver =
          instance(NetworkReceiver.class, clusterClientLife.getLifecycleInstances());
      receiver.stop();

      ExecutorLifecycleAdapter statemachineExecutor =
          instance(ExecutorLifecycleAdapter.class, clusterClientLife.getLifecycleInstances());
      statemachineExecutor.stop();

      NetworkSender sender =
          instance(NetworkSender.class, clusterClientLife.getLifecycleInstances());
      sender.stop();

      List<Lifecycle> stoppedServices = new ArrayList<>();
      stoppedServices.add(sender);
      stoppedServices.add(statemachineExecutor);
      stoppedServices.add(receiver);

      return new StartNetworkAgainKit(db, stoppedServices);
    }
 @Override
 public void init() throws Throwable {
   client = new ZooClient(logger.getLogger(ZooClient.class), config);
   life.add(client);
   client.addZooListener(new ZooHaEventListener());
   client.addCompatibilityModeListener(new ZooCompatibilityModeListener());
   life.init();
 }
Esempio n. 6
0
  private IndexingModule buildIndexing(
      Config config,
      JobScheduler scheduler,
      SchemaIndexProvider indexProvider,
      LockService lockService,
      TokenNameLookup tokenNameLookup,
      Logging logging,
      IndexingService.Monitor indexingServiceMonitor,
      NeoStore neoStore,
      UpdateableSchemaState updateableSchemaState) {
    final DefaultSchemaIndexProviderMap providerMap =
        new DefaultSchemaIndexProviderMap(indexProvider);

    final IndexingService indexingService =
        IndexingService.create(
            new IndexSamplingConfig(config),
            scheduler,
            providerMap,
            new NeoStoreIndexStoreView(lockService, neoStore),
            tokenNameLookup,
            updateableSchemaState,
            toList(new SchemaStorage(neoStore.getSchemaStore()).allIndexRules()),
            logging,
            indexingServiceMonitor);
    final IntegrityValidator integrityValidator = new IntegrityValidator(neoStore, indexingService);

    // TODO Move to constructor
    final LabelScanStore labelScanStore =
        dependencyResolver
            .resolveDependency(
                LabelScanStoreProvider.class, LabelScanStoreProvider.HIGHEST_PRIORITIZED)
            .getLabelScanStore();

    life.add(indexingService);
    life.add(labelScanStore);

    return new IndexingModule() {
      @Override
      public IndexingService indexingService() {
        return indexingService;
      }

      @Override
      public LabelScanStore labelScanStore() {
        return labelScanStore;
      }

      @Override
      public IntegrityValidator integrityValidator() {
        return integrityValidator;
      }

      @Override
      public SchemaIndexProviderMap schemaIndexProviderMap() {
        return providerMap;
      }
    };
  }
Esempio n. 7
0
 @Override
 public void init() {
   luceneDataSource = life.add(new LuceneDataSource(config, indexStore, fileSystemAbstraction));
   // TODO Don't do this here, do proper life cycle management
   life.start();
   LuceneIndexImplementation indexImplementation = new LuceneIndexImplementation(luceneDataSource);
   indexProviders.registerIndexProvider(
       LuceneIndexImplementation.SERVICE_NAME, indexImplementation);
 }
  @Test
  public void shouldSendAMessageFromAClientWhichIsReceivedByAServer() throws Exception {

    // given

    CountDownLatch latch = new CountDownLatch(1);

    LifeSupport life = new LifeSupport();

    Server server1 =
        new Server(
            latch,
            MapUtil.stringMap(
                ClusterSettings.cluster_server.name(),
                "localhost:1234",
                ClusterSettings.server_id.name(),
                "1",
                ClusterSettings.initial_hosts.name(),
                "localhost:1234,localhost:1235"));

    life.add(server1);

    Server server2 =
        new Server(
            latch,
            MapUtil.stringMap(
                ClusterSettings.cluster_server.name(),
                "localhost:1235",
                ClusterSettings.server_id.name(),
                "2",
                ClusterSettings.initial_hosts.name(),
                "localhost:1234,localhost:1235"));

    life.add(server2);

    life.start();

    // when

    server1.process(
        Message.to(TestMessage.helloWorld, URI.create("cluster://127.0.0.1:1235"), "Hello World"));

    // then

    latch.await(5, TimeUnit.SECONDS);

    assertTrue("server1 should have processed the message", server1.processedMessage());
    assertTrue("server2 should have processed the message", server2.processedMessage());

    life.shutdown();
  }
Esempio n. 9
0
  @Override
  public void start() throws Throwable {
    Clusters clusters = clustersProvider.clusters();

    life = new LifeSupport();

    for (int i = 0; i < clusters.getClusters().size(); i++) {
      Clusters.Cluster cluster = clusters.getClusters().get(i);
      ManagedCluster managedCluster = new ManagedCluster(cluster);
      clusterMap.put(cluster.getName(), managedCluster);
      life.add(managedCluster);
    }

    life.start();
  }
Esempio n. 10
0
  private NeoStoreModule buildNeoStore(
      StoreFactory storeFactory,
      final LabelTokenHolder labelTokens,
      final RelationshipTypeTokenHolder relationshipTypeTokens,
      final PropertyKeyTokenHolder propertyKeyTokenHolder) {
    final NeoStore neoStore = storeFactory.newNeoStore(false, true);

    life.add(
        new LifecycleAdapter() {
          @Override
          public void start() {
            if (startupStatistics.numberOfRecoveredTransactions() > 0) {
              neoStore.rebuildIdGenerators();
            }
            neoStoreModule.neoStore().makeStoreOk();

            propertyKeyTokenHolder.addTokens(
                neoStoreModule.neoStore().getPropertyKeyTokenStore().getTokens(Integer.MAX_VALUE));
            relationshipTypeTokens.addTokens(
                neoStoreModule
                    .neoStore()
                    .getRelationshipTypeTokenStore()
                    .getTokens(Integer.MAX_VALUE));
            labelTokens.addTokens(
                neoStoreModule.neoStore().getLabelTokenStore().getTokens(Integer.MAX_VALUE));
          }
        });

    return new NeoStoreModule() {
      @Override
      public NeoStore neoStore() {
        return neoStore;
      }
    };
  }
Esempio n. 11
0
 /** Public for testing purpose. Do not use. */
 public synchronized void registerDataSource(final XaDataSource dataSource) {
   dataSources.put(dataSource.getName(), dataSource);
   branchIdMapping.put(UTF8.decode(dataSource.getBranchId()), dataSource);
   sourceIdMapping.put(dataSource.getName(), dataSource.getBranchId());
   life.add(dataSource);
   if (life.getStatus().equals(LifecycleStatus.STARTED)) {
     Listeners.notifyListeners(
         dsRegistrationListeners,
         new Listeners.Notification<DataSourceRegistrationListener>() {
           @Override
           public void notify(DataSourceRegistrationListener listener) {
             listener.registeredDataSource(dataSource);
           }
         });
   }
 }
  @Override
  protected IdGeneratorFactory createIdGeneratorFactory() {
    idGeneratorFactory = new HaIdGeneratorFactory(master, logging);
    highAvailabilityModeSwitcher =
        new HighAvailabilityModeSwitcher(
            clusterClient,
            masterDelegateInvocationHandler,
            clusterMemberAvailability,
            memberStateMachine,
            this,
            (HaIdGeneratorFactory) idGeneratorFactory,
            config,
            logging,
            updateableSchemaState,
            kernelExtensions.listFactories(),
            monitors);
    /*
     * We always need the mode switcher and we need it to restart on switchover.
     */
    paxosLife.add(highAvailabilityModeSwitcher);

    /*
     * We don't really switch to master here. We just need to initialize the idGenerator so the initial store
     * can be started (if required). In any case, the rest of the database is in pending state, so nothing will
     * happen until events start arriving and that will set us to the proper state anyway.
     */
    ((HaIdGeneratorFactory) idGeneratorFactory).switchToMaster();

    return idGeneratorFactory;
  }
Esempio n. 13
0
    /**
     * WARNING: beware of hacks.
     *
     * <p>Fails a member of this cluster by making it not respond to heart beats. A {@link
     * RepairKit} is returned which is able to repair the instance (i.e start the network) again.
     *
     * @param db the {@link HighlyAvailableGraphDatabase} to fail.
     * @return a {@link RepairKit} which can repair the failure.
     * @throws IllegalArgumentException if the given db isn't a member of this cluster.
     */
    public RepairKit fail(HighlyAvailableGraphDatabase db) throws Throwable {
      assertMember(db);
      ClusterClient clusterClient =
          db.getDependencyResolver().resolveDependency(ClusterClient.class);
      LifeSupport clusterClientLife =
          (LifeSupport)
              accessible(clusterClient.getClass().getDeclaredField("life")).get(clusterClient);
      NetworkInstance network =
          instance(NetworkInstance.class, clusterClientLife.getLifecycleInstances());
      network.stop();

      int serverId =
          db.getDependencyResolver().resolveDependency(Config.class).get(HaSettings.server_id);
      db.shutdown();
      return new StartDatabaseAgainKit(this, serverId);
    }
Esempio n. 14
0
  @Override
  public void shutdown() {
    recordAccess.close();

    if (isShutdown) {
      throw new IllegalStateException("Batch inserter already has shutdown");
    }
    isShutdown = true;

    try {
      repopulateAllIndexes();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    neoStore.close();

    try {
      storeLocker.release();
    } catch (IOException e) {
      throw new UnderlyingStorageException("Could not release store lock", e);
    }

    msgLog.logMessage(
        Thread.currentThread() + " Clean shutdown on BatchInserter(" + this + ")", true);
    msgLog.close();
    life.shutdown();
  }
Esempio n. 15
0
  @Override
  protected IdGeneratorFactory createIdGeneratorFactory() {

    idGeneratorFactory = new HaIdGeneratorFactory(master, memberStateMachine, logging);
    highAvailabilityModeSwitcher =
        new HighAvailabilityModeSwitcher(
            masterDelegateInvocationHandler,
            clusterMemberAvailability,
            memberStateMachine,
            this,
            (HaIdGeneratorFactory) idGeneratorFactory,
            config,
            logging);
    /*
     * We always need the mode switcher and we need it to restart on switchover. So:
     * 1) if in compatibility mode, it must be added in all 3 - to start on start and restart on switchover
     * 2) if not in compatibility mode it must be added in paxosLife, which is started anyway.
     */
    paxosLife.add(highAvailabilityModeSwitcher);
    if (compatibilityMode) {
      compatibilityLifecycle.add(1, highAvailabilityModeSwitcher);
      life.add(highAvailabilityModeSwitcher);
    }

    return idGeneratorFactory;
  }
 @Override
 public void start() throws Throwable {
   life.start(); // Binds ZooClient so we have a local address to use next
   Listeners.notifyListeners(
       bindingListeners,
       new Listeners.Notification<BindingListener>() {
         @Override
         public void notify(BindingListener listener) {
           listener.listeningAt(URI.create(client.getClusterServer()));
         }
       });
   client.refreshMasterFromZooKeeper();
   Listeners.notifyListeners(
       haListeners,
       new Listeners.Notification<ClusterMemberListener>() {
         @Override
         public void notify(ClusterMemberListener listener) {
           listener.coordinatorIsElected(new InstanceId(client.getCachedMaster().getMachineId()));
         }
       });
   Listeners.notifyListeners(
       haListeners,
       new Listeners.Notification<ClusterMemberListener>() {
         @Override
         public void notify(ClusterMemberListener listener) {
           listener.memberIsAvailable(
               HighAvailabilityModeSwitcher.MASTER,
               new InstanceId(client.getCachedMaster().getMachineId()),
               URI.create("ha://" + client.getCachedMaster().getServerAsString()));
         }
       });
 }
    private Server(final CountDownLatch latch, final Map<String, String> config) {
      final Config conf = new Config(config, ClusterSettings.class);
      networkReceiver =
          life.add(
              new NetworkReceiver(
                  new NetworkReceiver.Configuration() {
                    @Override
                    public HostnamePort clusterServer() {
                      return conf.get(ClusterSettings.cluster_server);
                    }
                  },
                  new DevNullLoggingService()));

      networkSender =
          life.add(
              new NetworkSender(
                  new NetworkSender.Configuration() {
                    @Override
                    public int defaultPort() {
                      return 5001;
                    }

                    @Override
                    public int port() {
                      return conf.get(ClusterSettings.cluster_server).getPort();
                    }
                  },
                  networkReceiver,
                  new DevNullLoggingService()));

      life.add(
          new LifecycleAdapter() {
            @Override
            public void start() throws Throwable {
              networkReceiver.addMessageProcessor(
                  new MessageProcessor() {
                    @Override
                    public boolean process(Message<? extends MessageType> message) {
                      // server receives a message
                      processedMessage.set(true);
                      latch.countDown();
                      return true;
                    }
                  });
            }
          });
    }
Esempio n. 18
0
 @Override
 public void start() throws Throwable {
   life = new LifeSupport();
   for (XaDataSource ds : dataSources.values()) {
     life.add(ds);
   }
   life.start();
   for (DataSourceRegistrationListener listener : dsRegistrationListeners) {
     try {
       for (XaDataSource ds : dataSources.values()) {
         listener.registeredDataSource(ds);
       }
     } catch (Throwable t) {
       msgLog.logMessage("Failed when notifying registering listener", t);
     }
   }
 }
Esempio n. 19
0
 @Override
 public void close() {
   if (neoStore != null) {
     neoStore.close();
     neoStore = null;
   }
   life.shutdown();
 }
Esempio n. 20
0
 @Override
 public void shutdown() throws Throwable {
   dsRegistrationListeners = null;
   life.shutdown();
   dataSources.clear();
   branchIdMapping.clear();
   sourceIdMapping.clear();
   isShutdown = true;
 }
Esempio n. 21
0
  @Override
  public void start() throws Throwable {
    Clusters clusters = clustersProvider.clusters();

    life = new LifeSupport();

    // Started so instances added here will be started immediately, and in case of exceptions they
    // can be
    // shutdown() or stop()ped properly
    life.start();

    for (int i = 0; i < clusters.getClusters().size(); i++) {
      Clusters.Cluster cluster = clusters.getClusters().get(i);
      ManagedCluster managedCluster = new ManagedCluster(cluster);
      clusterMap.put(cluster.getName(), managedCluster);
      life.add(managedCluster);
    }
  }
Esempio n. 22
0
 @Override
 public void stop() {
   if (!readOnly) {
     forceEverything();
   }
   life.shutdown();
   neoStore.close();
   msgLog.info("NeoStore closed");
 }
  @Override
  public synchronized void shutdown() throws Throwable {
    modeSwitcherExecutor.shutdown();

    modeSwitcherExecutor.awaitTermination(60, TimeUnit.SECONDS);

    haCommunicationLife.shutdown();

    switchToMaster.close();
    switchToMaster = null;
    switchToSlave = null;
  }
Esempio n. 24
0
 public void addDataSourceRegistrationListener(DataSourceRegistrationListener listener) {
   if (life.getStatus().equals(LifecycleStatus.STARTED)) {
     try {
       for (XaDataSource ds : dataSources.values()) {
         listener.registeredDataSource(ds);
       }
     } catch (Throwable t) {
       msgLog.logMessage("Failed when notifying registering listener", t);
     }
   }
   dsRegistrationListeners = Listeners.addListener(listener, dsRegistrationListeners);
 }
Esempio n. 25
0
 @Override
 public void stop() {
   super.stop();
   if (!readOnly) {
     forceEverything();
   }
   life.shutdown();
   xaContainer.close();
   if (logApplied) {
     neoStore.rebuildIdGenerators();
     logApplied = false;
   }
   neoStore.close();
   msgLog.info("NeoStore closed");
 }
Esempio n. 26
0
  @Override
  public void stop() {
    transactionLogModule.logRotationControl().awaitAllTransactionsClosed();
    transactionLogModule.logRotationControl().forceEverything();
    /*
     * We simply increment the version, essentially "rotating" away
     * the current active log file, to avoid having a recovery on
     * next startup. Not necessary, simply speeds up the startup
     * process.
     */
    neoStoreModule.neoStore().incrementAndGetVersion();

    life.shutdown();
    neoStoreModule.neoStore().close();
    msgLog.info("NeoStore closed");
  }
Esempio n. 27
0
  /** Public for testing purpose. Do not use. */
  public synchronized void unregisterDataSource(String name) {
    final XaDataSource dataSource = dataSources.get(name);
    if (dataSource == null) {
      return;
    }

    byte branchId[] = getBranchId(dataSource.getXaConnection().getXaResource());
    dataSources.remove(name);
    branchIdMapping.remove(UTF8.decode(branchId));
    sourceIdMapping.remove(name);
    Listeners.notifyListeners(
        dsRegistrationListeners,
        new Listeners.Notification<DataSourceRegistrationListener>() {
          @Override
          public void notify(DataSourceRegistrationListener listener) {
            listener.unregisteredDataSource(dataSource);
          }
        });
    life.remove(dataSource);
    // No need for shutdown, removing does that
  }
Esempio n. 28
0
 @Override
 public void stop() throws Throwable {
   life.stop();
 }
 @Override
 public synchronized void stop() throws Throwable {
   haCommunicationLife.stop();
 }
  @Override
  public synchronized void init() throws Throwable {
    modeSwitcherExecutor = createExecutor();

    haCommunicationLife.init();
  }