Exemplo n.º 1
0
  private ClusterState rejoin(ClusterState clusterState, String reason) {
    logger.warn(reason + ", current nodes: {}", clusterState.nodes());
    nodesFD.stop();
    masterFD.stop(reason);
    master = false;

    ClusterBlocks clusterBlocks =
        ClusterBlocks.builder()
            .blocks(clusterState.blocks())
            .addGlobalBlock(NO_MASTER_BLOCK)
            .addGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)
            .build();

    // clear the routing table, we have no master, so we need to recreate the routing when we reform
    // the cluster
    RoutingTable routingTable = RoutingTable.builder().build();
    // we also clean the metadata, since we are going to recover it if we become master
    MetaData metaData = MetaData.builder().build();

    // clean the nodes, we are now not connected to anybody, since we try and reform the cluster
    latestDiscoNodes =
        new DiscoveryNodes.Builder().put(localNode).localNodeId(localNode.id()).build();

    asyncJoinCluster();

    return ClusterState.builder(clusterState)
        .blocks(clusterBlocks)
        .nodes(latestDiscoNodes)
        .routingTable(routingTable)
        .metaData(metaData)
        .build();
  }
Exemplo n.º 2
0
 @Override
 public void publish(ClusterState clusterState, AckListener ackListener) {
   if (!master) {
     throw new ElasticsearchIllegalStateException("Shouldn't publish state when not master");
   }
   latestDiscoNodes = clusterState.nodes();
   nodesFD.updateNodes(clusterState.nodes());
   publishClusterState.publish(clusterState, ackListener);
 }
Exemplo n.º 3
0
  void handleNewClusterStateFromMaster(
      final ClusterState newState,
      final PublishClusterStateAction.NewClusterStateListener.NewStateProcessed newStateProcessed) {
    if (master) {
      clusterService.submitStateUpdateTask(
          "zen-disco-master_receive_cluster_state_from_another_master ["
              + newState.nodes().masterNode()
              + "]",
          Priority.URGENT,
          new ProcessedClusterStateUpdateTask() {
            @Override
            public ClusterState execute(ClusterState currentState) {
              if (newState.version() > currentState.version()) {
                logger.warn(
                    "received cluster state from [{}] which is also master but with a newer cluster_state, rejoining to cluster...",
                    newState.nodes().masterNode());
                return rejoin(
                    currentState,
                    "zen-disco-master_receive_cluster_state_from_another_master ["
                        + newState.nodes().masterNode()
                        + "]");
              } else {
                logger.warn(
                    "received cluster state from [{}] which is also master but with an older cluster_state, telling [{}] to rejoin the cluster",
                    newState.nodes().masterNode(),
                    newState.nodes().masterNode());
                transportService.sendRequest(
                    newState.nodes().masterNode(),
                    RejoinClusterRequestHandler.ACTION,
                    new RejoinClusterRequest(currentState.nodes().localNodeId()),
                    new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
                      @Override
                      public void handleException(TransportException exp) {
                        logger.warn(
                            "failed to send rejoin request to [{}]",
                            exp,
                            newState.nodes().masterNode());
                      }
                    });
                return currentState;
              }
            }

            @Override
            public void clusterStateProcessed(
                String source, ClusterState oldState, ClusterState newState) {
              newStateProcessed.onNewClusterStateProcessed();
            }

            @Override
            public void onFailure(String source, Throwable t) {
              logger.error("unexpected failure during [{}]", t, source);
              newStateProcessed.onNewClusterStateFailed(t);
            }
          });
    } else {
      if (newState.nodes().localNode() == null) {
        logger.warn(
            "received a cluster state from [{}] and not part of the cluster, should not happen",
            newState.nodes().masterNode());
        newStateProcessed.onNewClusterStateFailed(
            new ElasticsearchIllegalStateException(
                "received state from a node that is not part of the cluster"));
      } else {
        if (currentJoinThread != null) {
          logger.debug(
              "got a new state from master node, though we are already trying to rejoin the cluster");
        }

        clusterService.submitStateUpdateTask(
            "zen-disco-receive(from master [" + newState.nodes().masterNode() + "])",
            Priority.URGENT,
            new ProcessedClusterStateUpdateTask() {
              @Override
              public ClusterState execute(ClusterState currentState) {

                // we don't need to do this, since we ping the master, and get notified when it has
                // moved from being a master
                // because it doesn't have enough master nodes...
                // if (!electMaster.hasEnoughMasterNodes(newState.nodes())) {
                //    return disconnectFromCluster(newState, "not enough master nodes on new cluster
                // state received from [" + newState.nodes().masterNode() + "]");
                // }

                latestDiscoNodes = newState.nodes();

                // check to see that we monitor the correct master of the cluster
                if (masterFD.masterNode() == null
                    || !masterFD.masterNode().equals(latestDiscoNodes.masterNode())) {
                  masterFD.restart(
                      latestDiscoNodes.masterNode(),
                      "new cluster state received and we are monitoring the wrong master ["
                          + masterFD.masterNode()
                          + "]");
                }

                ClusterState.Builder builder = ClusterState.builder(newState);
                // if the routing table did not change, use the original one
                if (newState.routingTable().version() == currentState.routingTable().version()) {
                  builder.routingTable(currentState.routingTable());
                }
                // same for metadata
                if (newState.metaData().version() == currentState.metaData().version()) {
                  builder.metaData(currentState.metaData());
                } else {
                  // if its not the same version, only copy over new indices or ones that changed
                  // the version
                  MetaData.Builder metaDataBuilder =
                      MetaData.builder(newState.metaData()).removeAllIndices();
                  for (IndexMetaData indexMetaData : newState.metaData()) {
                    IndexMetaData currentIndexMetaData =
                        currentState.metaData().index(indexMetaData.index());
                    if (currentIndexMetaData == null
                        || currentIndexMetaData.version() != indexMetaData.version()) {
                      metaDataBuilder.put(indexMetaData, false);
                    } else {
                      metaDataBuilder.put(currentIndexMetaData, false);
                    }
                  }
                  builder.metaData(metaDataBuilder);
                }

                return builder.build();
              }

              @Override
              public void onFailure(String source, Throwable t) {
                logger.error("unexpected failure during [{}]", t, source);
                newStateProcessed.onNewClusterStateFailed(t);
              }

              @Override
              public void clusterStateProcessed(
                  String source, ClusterState oldState, ClusterState newState) {
                sendInitialStateEventIfNeeded();
                newStateProcessed.onNewClusterStateProcessed();
              }
            });
      }
    }
  }
Exemplo n.º 4
0
  void handleNewClusterStateFromMaster(
      ClusterState newClusterState,
      final PublishClusterStateAction.NewClusterStateListener.NewStateProcessed newStateProcessed) {
    if (master) {
      final ClusterState newState = newClusterState;
      clusterService.submitStateUpdateTask(
          "zen-disco-master_receive_cluster_state_from_another_master ["
              + newState.nodes().masterNode()
              + "]",
          Priority.URGENT,
          new ProcessedClusterStateUpdateTask() {
            @Override
            public ClusterState execute(ClusterState currentState) {
              if (newState.version() > currentState.version()) {
                logger.warn(
                    "received cluster state from [{}] which is also master but with a newer cluster_state, rejoining to cluster...",
                    newState.nodes().masterNode());
                return rejoin(
                    currentState,
                    "zen-disco-master_receive_cluster_state_from_another_master ["
                        + newState.nodes().masterNode()
                        + "]");
              } else {
                logger.warn(
                    "received cluster state from [{}] which is also master but with an older cluster_state, telling [{}] to rejoin the cluster",
                    newState.nodes().masterNode(),
                    newState.nodes().masterNode());
                transportService.sendRequest(
                    newState.nodes().masterNode(),
                    RejoinClusterRequestHandler.ACTION,
                    new RejoinClusterRequest(currentState.nodes().localNodeId()),
                    new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
                      @Override
                      public void handleException(TransportException exp) {
                        logger.warn(
                            "failed to send rejoin request to [{}]",
                            exp,
                            newState.nodes().masterNode());
                      }
                    });
                return currentState;
              }
            }

            @Override
            public void clusterStateProcessed(
                String source, ClusterState oldState, ClusterState newState) {
              newStateProcessed.onNewClusterStateProcessed();
            }

            @Override
            public void onFailure(String source, Throwable t) {
              logger.error("unexpected failure during [{}]", t, source);
              newStateProcessed.onNewClusterStateFailed(t);
            }
          });
    } else {
      if (newClusterState.nodes().localNode() == null) {
        logger.warn(
            "received a cluster state from [{}] and not part of the cluster, should not happen",
            newClusterState.nodes().masterNode());
        newStateProcessed.onNewClusterStateFailed(
            new ElasticsearchIllegalStateException(
                "received state from a node that is not part of the cluster"));
      } else {
        if (currentJoinThread != null) {
          logger.debug(
              "got a new state from master node, though we are already trying to rejoin the cluster");
        }

        final ProcessClusterState processClusterState =
            new ProcessClusterState(newClusterState, newStateProcessed);
        processNewClusterStates.add(processClusterState);

        clusterService.submitStateUpdateTask(
            "zen-disco-receive(from master [" + newClusterState.nodes().masterNode() + "])",
            Priority.URGENT,
            new ProcessedClusterStateUpdateTask() {
              @Override
              public ClusterState execute(ClusterState currentState) {
                // we already processed it in a previous event
                if (processClusterState.processed) {
                  return currentState;
                }

                // TODO: once improvement that we can do is change the message structure to include
                // version and masterNodeId
                // at the start, this will allow us to keep the "compressed bytes" around, and only
                // parse the first page
                // to figure out if we need to use it or not, and only once we picked the latest
                // one, parse the whole state

                // try and get the state with the highest version out of all the ones with the same
                // master node id
                ProcessClusterState stateToProcess = processNewClusterStates.poll();
                if (stateToProcess == null) {
                  return currentState;
                }
                stateToProcess.processed = true;
                while (true) {
                  ProcessClusterState potentialState = processNewClusterStates.peek();
                  // nothing else in the queue, bail
                  if (potentialState == null) {
                    break;
                  }
                  // if its not from the same master, then bail
                  if (!Objects.equal(
                      stateToProcess.clusterState.nodes().masterNodeId(),
                      potentialState.clusterState.nodes().masterNodeId())) {
                    break;
                  }

                  // we are going to use it for sure, poll (remove) it
                  potentialState = processNewClusterStates.poll();
                  potentialState.processed = true;

                  if (potentialState.clusterState.version()
                      > stateToProcess.clusterState.version()) {
                    // we found a new one
                    stateToProcess = potentialState;
                  }
                }

                ClusterState updatedState = stateToProcess.clusterState;

                // if the new state has a smaller version, and it has the same master node, then no
                // need to process it
                if (updatedState.version() < currentState.version()
                    && Objects.equal(
                        updatedState.nodes().masterNodeId(), currentState.nodes().masterNodeId())) {
                  return currentState;
                }

                // we don't need to do this, since we ping the master, and get notified when it has
                // moved from being a master
                // because it doesn't have enough master nodes...
                // if (!electMaster.hasEnoughMasterNodes(newState.nodes())) {
                //    return disconnectFromCluster(newState, "not enough master nodes on new cluster
                // state received from [" + newState.nodes().masterNode() + "]");
                // }

                latestDiscoNodes = updatedState.nodes();

                // check to see that we monitor the correct master of the cluster
                if (masterFD.masterNode() == null
                    || !masterFD.masterNode().equals(latestDiscoNodes.masterNode())) {
                  masterFD.restart(
                      latestDiscoNodes.masterNode(),
                      "new cluster state received and we are monitoring the wrong master ["
                          + masterFD.masterNode()
                          + "]");
                }

                ClusterState.Builder builder = ClusterState.builder(updatedState);
                // if the routing table did not change, use the original one
                if (updatedState.routingTable().version()
                    == currentState.routingTable().version()) {
                  builder.routingTable(currentState.routingTable());
                }
                // same for metadata
                if (updatedState.metaData().version() == currentState.metaData().version()) {
                  builder.metaData(currentState.metaData());
                } else {
                  // if its not the same version, only copy over new indices or ones that changed
                  // the version
                  MetaData.Builder metaDataBuilder =
                      MetaData.builder(updatedState.metaData()).removeAllIndices();
                  for (IndexMetaData indexMetaData : updatedState.metaData()) {
                    IndexMetaData currentIndexMetaData =
                        currentState.metaData().index(indexMetaData.index());
                    if (currentIndexMetaData == null
                        || currentIndexMetaData.version() != indexMetaData.version()) {
                      metaDataBuilder.put(indexMetaData, false);
                    } else {
                      metaDataBuilder.put(currentIndexMetaData, false);
                    }
                  }
                  builder.metaData(metaDataBuilder);
                }

                return builder.build();
              }

              @Override
              public void onFailure(String source, Throwable t) {
                logger.error("unexpected failure during [{}]", t, source);
                newStateProcessed.onNewClusterStateFailed(t);
              }

              @Override
              public void clusterStateProcessed(
                  String source, ClusterState oldState, ClusterState newState) {
                sendInitialStateEventIfNeeded();
                newStateProcessed.onNewClusterStateProcessed();
              }
            });
      }
    }
  }