private static Map<Integer, Long> findLocalSources(
      Collection<IndexSnapshotRequestConfig.PartitionRanges> partitionRanges, SiteTracker tracker) {
    Set<Integer> partitions = Sets.newHashSet();
    for (IndexSnapshotRequestConfig.PartitionRanges partitionRange : partitionRanges) {
      partitions.add(partitionRange.partitionId);
    }

    Map<Integer, Long> pidToLocalHSId = Maps.newHashMap();
    List<Long> localSites = Longs.asList(tracker.getLocalSites());

    for (long hsId : localSites) {
      int pid = tracker.getPartitionForSite(hsId);
      if (partitions.contains(pid)) {
        pidToLocalHSId.put(pid, hsId);
      }
    }

    return pidToLocalHSId;
  }
示例#2
0
  public static String getLiveSystemOverview() {
    // get the start time
    long t = SystemStatsCollector.getStartTime();
    Date date = new Date(t);
    long duration = System.currentTimeMillis() - t;
    long minutes = duration / 60000;
    long hours = minutes / 60;
    minutes -= hours * 60;
    long days = hours / 24;
    hours -= days * 24;
    String starttime = String.format("%s (%dd %dh %dm)", date.toString(), days, hours, minutes);

    // handle the basic info page below this
    SiteTracker st = VoltDB.instance().getSiteTrackerForSnapshot();

    // get the cluster info
    String clusterinfo = st.getAllHosts().size() + " hosts ";
    clusterinfo += " with " + st.getAllSites().size() + " sites ";
    clusterinfo += " (" + st.getAllSites().size() / st.getAllHosts().size();
    clusterinfo += " per host)";

    StringBuilder sb = new StringBuilder();
    sb.append("<table class='table table-condensed'>\n");
    sb.append(
        "<tr><td>Mode                     </td><td>"
            + VoltDB.instance().getMode().toString()
            + "</td><td>\n");
    sb.append(
        "<tr><td>VoltDB Version           </td><td>"
            + VoltDB.instance().getVersionString()
            + "</td><td>\n");
    sb.append(
        "<tr><td>Buildstring              </td><td>"
            + VoltDB.instance().getBuildString()
            + "</td><td>\n");
    sb.append("<tr><td>Cluster Composition      </td><td>" + clusterinfo + "</td><td>\n");
    sb.append("<tr><td>Running Since            </td><td>" + starttime + "</td><td>\n");

    sb.append("</table>\n");

    return sb.toString();
  }
示例#3
0
  private static List<Long> computeDedupedLocalSites(long txnId, SiteTracker tracker) {
    MessageDigest digest;
    try {
      digest = MessageDigest.getInstance("SHA-1");
    } catch (NoSuchAlgorithmException e) {
      throw new AssertionError(e);
    }

    /*
     * List of partitions to include if this snapshot is
     * going to be deduped. Attempts to break up the work
     * by seeding and RNG selecting
     * a random replica to do the work. Will not work in failure
     * cases, but we don't use dedupe when we want durability.
     *
     * Originally used the partition id as the seed, but it turns out
     * that nextInt(2) returns a 1 for seeds 0-4095. Now use SHA-1
     * on the txnid + partition id.
     */
    List<Long> sitesToInclude = new ArrayList<Long>();
    for (long localSite : tracker.getLocalSites()) {
      final int partitionId = tracker.getPartitionForSite(localSite);
      List<Long> sites =
          new ArrayList<Long>(tracker.getSitesForPartition(tracker.getPartitionForSite(localSite)));
      Collections.sort(sites);

      digest.update(Longs.toByteArray(txnId));
      final long seed =
          Longs.fromByteArray(Arrays.copyOf(digest.digest(Ints.toByteArray(partitionId)), 8));

      int siteIndex = new java.util.Random(seed).nextInt(sites.size());
      if (localSite == sites.get(siteIndex)) {
        sitesToInclude.add(localSite);
      }
    }
    return sitesToInclude;
  }
示例#4
0
  protected boolean createSetupInternal(
      String file_path,
      String file_nonce,
      long txnId,
      Map<Integer, Long> partitionTransactionIds,
      JSONObject jsData,
      SystemProcedureExecutionContext context,
      String hostname,
      final VoltTable result,
      Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers,
      SiteTracker tracker,
      long timestamp)
      throws IOException {
    assert (SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty());

    /*
     * List of partitions to include if this snapshot is
     * going to be deduped. Attempts to break up the work
     * by seeding and RNG selecting
     * a random replica to do the work. Will not work in failure
     * cases, but we don't use dedupe when we want durability.
     */
    List<Long> sitesToInclude = CSVSnapshotWritePlan.computeDedupedLocalSites(txnId, tracker);
    // If there's no work to do on this host, just claim success and get out:
    if (sitesToInclude.isEmpty() && !tracker.isFirstHost()) {
      return false;
    }

    NativeSnapshotWritePlan.createFileBasedCompletionTasks(
        file_path,
        file_nonce,
        txnId,
        partitionTransactionIds,
        context,
        exportSequenceNumbers,
        timestamp);

    final List<Table> tables = SnapshotUtil.getTablesToSave(context.getDatabase());
    final AtomicInteger numTables = new AtomicInteger(tables.size());
    final SnapshotRegistry.Snapshot snapshotRecord =
        SnapshotRegistry.startSnapshot(
            txnId,
            context.getHostId(),
            file_path,
            file_nonce,
            SnapshotFormat.CSV,
            tables.toArray(new Table[0]));

    SnapshotDataTarget sdt = null;
    boolean noTargetsCreated = true;

    final ArrayList<SnapshotTableTask> partitionedSnapshotTasks =
        new ArrayList<SnapshotTableTask>();
    final ArrayList<SnapshotTableTask> replicatedSnapshotTasks = new ArrayList<SnapshotTableTask>();
    for (final Table table : tables) {
      /*
       * For a deduped csv snapshot, only produce the replicated tables on the "leader"
       * host.
       */
      if (table.getIsreplicated() && !tracker.isFirstHost()) {
        snapshotRecord.removeTable(table.getTypeName());
        continue;
      }

      File saveFilePath = null;
      saveFilePath =
          SnapshotUtil.constructFileForTable(
              table, file_path, file_nonce, SnapshotFormat.CSV, context.getHostId());

      try {
        sdt = new SimpleFileSnapshotDataTarget(saveFilePath, !table.getIsreplicated());

        m_targets.add(sdt);
        final Runnable onClose =
            new TargetStatsClosure(sdt, table.getTypeName(), numTables, snapshotRecord);
        sdt.setOnCloseHandler(onClose);

        List<SnapshotDataFilter> filters = new ArrayList<SnapshotDataFilter>();
        filters.add(new CSVSnapshotFilter(CatalogUtil.getVoltTable(table), ',', null));

        final SnapshotTableTask task =
            new SnapshotTableTask(
                table.getRelativeIndex(),
                sdt,
                filters.toArray(new SnapshotDataFilter[filters.size()]),
                table.getIsreplicated(),
                table.getTypeName());

        if (table.getIsreplicated()) {
          replicatedSnapshotTasks.add(task);
        } else {
          partitionedSnapshotTasks.add(task);
        }

        noTargetsCreated = false;
        result.addRow(context.getHostId(), hostname, table.getTypeName(), "SUCCESS", "");
      } catch (IOException ex) {
        handleTargetCreationError(
            sdt, context, file_nonce, hostname, table.getTypeName(), ex, result);
      }
    }

    if (noTargetsCreated) {
      SnapshotRegistry.discardSnapshot(snapshotRecord);
    }

    // CSV snapshots do the partitioned work only on the specified sites for de-duping,
    // but since we've pre-filtered the replicated task list to only contain entries on
    // one node, we can go ahead and distribute them across all of the sites on that node.
    placePartitionedTasks(partitionedSnapshotTasks, sitesToInclude);
    placeReplicatedTasks(replicatedSnapshotTasks, tracker.getSitesForHost(context.getHostId()));
    return noTargetsCreated;
  }