@Override
  public Callable<Boolean> createSetup(
      String file_path,
      String file_nonce,
      long txnId,
      Map<Integer, Long> partitionTransactionIds,
      JSONObject jsData,
      SystemProcedureExecutionContext context,
      VoltTable result,
      Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers,
      SiteTracker tracker,
      HashinatorSnapshotData hashinatorData,
      long timestamp) {
    assert SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty();

    final IndexSnapshotRequestConfig config =
        new IndexSnapshotRequestConfig(jsData, context.getDatabase());
    final Map<Integer, Long> pidToLocalHSIds = findLocalSources(config.partitionRanges, tracker);

    // mark snapshot start in registry
    final AtomicInteger numTables = new AtomicInteger(config.tables.length);
    m_snapshotRecord =
        SnapshotRegistry.startSnapshot(
            txnId, context.getHostId(), file_path, file_nonce, SnapshotFormat.INDEX, config.tables);

    // create table tasks
    for (Table table : config.tables) {
      createTasksForTable(
          table, config.partitionRanges, pidToLocalHSIds, numTables, m_snapshotRecord);
      result.addRow(
          context.getHostId(),
          CoreUtils.getHostnameOrAddress(),
          table.getTypeName(),
          "SUCCESS",
          "");
    }

    return null;
  }
Example #2
0
  protected boolean createSetupInternal(
      String file_path,
      String file_nonce,
      long txnId,
      Map<Integer, Long> partitionTransactionIds,
      JSONObject jsData,
      SystemProcedureExecutionContext context,
      String hostname,
      final VoltTable result,
      Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers,
      SiteTracker tracker,
      long timestamp)
      throws IOException {
    assert (SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty());

    /*
     * List of partitions to include if this snapshot is
     * going to be deduped. Attempts to break up the work
     * by seeding and RNG selecting
     * a random replica to do the work. Will not work in failure
     * cases, but we don't use dedupe when we want durability.
     */
    List<Long> sitesToInclude = CSVSnapshotWritePlan.computeDedupedLocalSites(txnId, tracker);
    // If there's no work to do on this host, just claim success and get out:
    if (sitesToInclude.isEmpty() && !tracker.isFirstHost()) {
      return false;
    }

    NativeSnapshotWritePlan.createFileBasedCompletionTasks(
        file_path,
        file_nonce,
        txnId,
        partitionTransactionIds,
        context,
        exportSequenceNumbers,
        timestamp);

    final List<Table> tables = SnapshotUtil.getTablesToSave(context.getDatabase());
    final AtomicInteger numTables = new AtomicInteger(tables.size());
    final SnapshotRegistry.Snapshot snapshotRecord =
        SnapshotRegistry.startSnapshot(
            txnId,
            context.getHostId(),
            file_path,
            file_nonce,
            SnapshotFormat.CSV,
            tables.toArray(new Table[0]));

    SnapshotDataTarget sdt = null;
    boolean noTargetsCreated = true;

    final ArrayList<SnapshotTableTask> partitionedSnapshotTasks =
        new ArrayList<SnapshotTableTask>();
    final ArrayList<SnapshotTableTask> replicatedSnapshotTasks = new ArrayList<SnapshotTableTask>();
    for (final Table table : tables) {
      /*
       * For a deduped csv snapshot, only produce the replicated tables on the "leader"
       * host.
       */
      if (table.getIsreplicated() && !tracker.isFirstHost()) {
        snapshotRecord.removeTable(table.getTypeName());
        continue;
      }

      File saveFilePath = null;
      saveFilePath =
          SnapshotUtil.constructFileForTable(
              table, file_path, file_nonce, SnapshotFormat.CSV, context.getHostId());

      try {
        sdt = new SimpleFileSnapshotDataTarget(saveFilePath, !table.getIsreplicated());

        m_targets.add(sdt);
        final Runnable onClose =
            new TargetStatsClosure(sdt, table.getTypeName(), numTables, snapshotRecord);
        sdt.setOnCloseHandler(onClose);

        List<SnapshotDataFilter> filters = new ArrayList<SnapshotDataFilter>();
        filters.add(new CSVSnapshotFilter(CatalogUtil.getVoltTable(table), ',', null));

        final SnapshotTableTask task =
            new SnapshotTableTask(
                table.getRelativeIndex(),
                sdt,
                filters.toArray(new SnapshotDataFilter[filters.size()]),
                table.getIsreplicated(),
                table.getTypeName());

        if (table.getIsreplicated()) {
          replicatedSnapshotTasks.add(task);
        } else {
          partitionedSnapshotTasks.add(task);
        }

        noTargetsCreated = false;
        result.addRow(context.getHostId(), hostname, table.getTypeName(), "SUCCESS", "");
      } catch (IOException ex) {
        handleTargetCreationError(
            sdt, context, file_nonce, hostname, table.getTypeName(), ex, result);
      }
    }

    if (noTargetsCreated) {
      SnapshotRegistry.discardSnapshot(snapshotRecord);
    }

    // CSV snapshots do the partitioned work only on the specified sites for de-duping,
    // but since we've pre-filtered the replicated task list to only contain entries on
    // one node, we can go ahead and distribute them across all of the sites on that node.
    placePartitionedTasks(partitionedSnapshotTasks, sitesToInclude);
    placeReplicatedTasks(replicatedSnapshotTasks, tracker.getSitesForHost(context.getHostId()));
    return noTargetsCreated;
  }