コード例 #1
0
  /**
   * Helper method to perform idempotent operation to refresh all watches (related to real-time
   * segments): - Data change listener for all existing real-time tables. - Child creation listener
   * for all existing real-time tables. - Data change listener for all existing real-time segments
   *
   * @param path
   */
  private void refreshWatchers(String path) {
    LOGGER.info("Received change notification for path: {}", path);
    List<Stat> stats = new ArrayList<>();
    List<ZNRecord> tableConfigs =
        _pinotHelixResourceManager.getPropertyStore().getChildren(TABLE_CONFIG, stats, 0);

    if (tableConfigs == null) {
      return;
    }

    for (ZNRecord tableConfig : tableConfigs) {
      try {
        AbstractTableConfig abstractTableConfig = AbstractTableConfig.fromZnRecord(tableConfig);
        if (abstractTableConfig.isRealTime()) {
          String realtimeTable = abstractTableConfig.getTableName();
          String realtimeSegmentsPathForTable =
              _propertyStorePath + SEGMENTS_PATH + "/" + realtimeTable;

          LOGGER.info("Setting data/child changes watch for real-time table '{}'", realtimeTable);
          _zkClient.subscribeDataChanges(realtimeSegmentsPathForTable, this);
          _zkClient.subscribeChildChanges(realtimeSegmentsPathForTable, this);

          List<String> childNames =
              _pinotHelixResourceManager
                  .getPropertyStore()
                  .getChildNames(SEGMENTS_PATH + "/" + realtimeTable, 0);

          if (childNames != null && !childNames.isEmpty()) {
            for (String segmentName : childNames) {
              String segmentPath = realtimeSegmentsPathForTable + "/" + segmentName;
              LOGGER.info("Setting data change watch for real-time segment: {}", segmentPath);
              _zkClient.subscribeDataChanges(segmentPath, this);
            }
          }
        }
      } catch (JSONException e) {
        LOGGER.error("Caught exception while reading table config", e);
      } catch (IOException e) {
        LOGGER.error(
            "Caught exception while setting change listeners for realtime tables/segments", e);
      }
    }
  }
コード例 #2
0
  private synchronized void assignRealtimeSegmentsToServerInstancesIfNecessary()
      throws JSONException, IOException {
    // Fetch current ideal state snapshot
    Map<String, IdealState> idealStateMap = new HashMap<String, IdealState>();

    for (String resource : _pinotHelixResourceManager.getAllRealtimeTables()) {
      idealStateMap.put(
          resource,
          _pinotHelixResourceManager
              .getHelixAdmin()
              .getResourceIdealState(_pinotHelixResourceManager.getHelixClusterName(), resource));
    }

    List<String> listOfSegmentsToAdd = new ArrayList<String>();

    for (String resource : idealStateMap.keySet()) {
      IdealState state = idealStateMap.get(resource);

      // Are there any partitions?
      if (state.getPartitionSet().size() == 0) {
        // No, this is a brand new ideal state, so we will add one new segment to every partition
        // and replica
        List<String> instancesInResource = new ArrayList<String>();
        try {
          instancesInResource.addAll(
              _pinotHelixResourceManager.getServerInstancesForTable(resource, TableType.REALTIME));
        } catch (Exception e) {
          LOGGER.error("Caught exception while fetching instances for resource {}", resource, e);
        }

        // Assign a new segment to all server instances
        for (String instanceId : instancesInResource) {
          InstanceZKMetadata instanceZKMetadata =
              _pinotHelixResourceManager.getInstanceZKMetadata(instanceId);
          String groupId = instanceZKMetadata.getGroupId(resource);
          String partitionId = instanceZKMetadata.getPartition(resource);
          listOfSegmentsToAdd.add(
              SegmentNameBuilder.Realtime.build(
                  resource,
                  instanceId,
                  groupId,
                  partitionId,
                  String.valueOf(System.currentTimeMillis())));
        }
      } else {
        // Add all server instances to the list of instances for which to assign a realtime segment
        Set<String> instancesToAssignRealtimeSegment = new HashSet<String>();
        instancesToAssignRealtimeSegment.addAll(
            _pinotHelixResourceManager.getServerInstancesForTable(resource, TableType.REALTIME));

        // Remove server instances that are currently processing a segment
        for (String partition : state.getPartitionSet()) {
          RealtimeSegmentZKMetadata realtimeSegmentZKMetadata =
              ZKMetadataProvider.getRealtimeSegmentZKMetadata(
                  _pinotHelixResourceManager.getPropertyStore(),
                  SegmentNameBuilder.Realtime.extractTableName(partition),
                  partition);
          if (realtimeSegmentZKMetadata.getStatus() == Status.IN_PROGRESS) {
            String instanceName = SegmentNameBuilder.Realtime.extractInstanceName(partition);
            instancesToAssignRealtimeSegment.remove(instanceName);
          }
        }

        // Assign a new segment to the server instances not currently processing this segment
        for (String instanceId : instancesToAssignRealtimeSegment) {
          InstanceZKMetadata instanceZKMetadata =
              _pinotHelixResourceManager.getInstanceZKMetadata(instanceId);
          String groupId = instanceZKMetadata.getGroupId(resource);
          String partitionId = instanceZKMetadata.getPartition(resource);
          listOfSegmentsToAdd.add(
              SegmentNameBuilder.Realtime.build(
                  resource,
                  instanceId,
                  groupId,
                  partitionId,
                  String.valueOf(System.currentTimeMillis())));
        }
      }
    }

    LOGGER.info(
        "Computed list of new segments to add : " + Arrays.toString(listOfSegmentsToAdd.toArray()));

    // Add the new segments to the server instances
    for (String segmentId : listOfSegmentsToAdd) {
      String resourceName = SegmentNameBuilder.Realtime.extractTableName(segmentId);
      String instanceName = SegmentNameBuilder.Realtime.extractInstanceName(segmentId);

      // Does the ideal state already contain this segment?
      if (!idealStateMap.get(resourceName).getPartitionSet().contains(segmentId)) {
        // No, add it
        // Create the realtime segment metadata
        RealtimeSegmentZKMetadata realtimeSegmentMetadataToAdd = new RealtimeSegmentZKMetadata();
        realtimeSegmentMetadataToAdd.setTableName(
            TableNameBuilder.extractRawTableName(resourceName));
        realtimeSegmentMetadataToAdd.setSegmentType(SegmentType.REALTIME);
        realtimeSegmentMetadataToAdd.setStatus(Status.IN_PROGRESS);
        realtimeSegmentMetadataToAdd.setSegmentName(segmentId);

        // Add the new metadata to the property store
        ZKMetadataProvider.setRealtimeSegmentZKMetadata(
            _pinotHelixResourceManager.getPropertyStore(), realtimeSegmentMetadataToAdd);

        // Update the ideal state to add the new realtime segment
        HelixHelper.updateIdealState(
            _pinotHelixResourceManager.getHelixZkManager(),
            resourceName,
            idealState ->
                PinotTableIdealStateBuilder.addNewRealtimeSegmentToIdealState(
                    segmentId, idealState, instanceName),
            RetryPolicies.exponentialBackoffRetryPolicy(5, 500L, 2.0f));
      }
    }
  }
コード例 #3
0
 public void stop() {
   LOGGER.info("Stopping realtime segments manager, stopping property store.");
   _pinotHelixResourceManager.getPropertyStore().stop();
 }