@Override
public Callable<Boolean> createSetup(
String file_path,
String file_nonce,
long txnId,
Map<Integer, Long> partitionTransactionIds,
JSONObject jsData,
SystemProcedureExecutionContext context,
VoltTable result,
Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers,
SiteTracker tracker,
HashinatorSnapshotData hashinatorData,
long timestamp) {
assert SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty();
final IndexSnapshotRequestConfig config =
new IndexSnapshotRequestConfig(jsData, context.getDatabase());
final Map<Integer, Long> pidToLocalHSIds = findLocalSources(config.partitionRanges, tracker);
// mark snapshot start in registry
final AtomicInteger numTables = new AtomicInteger(config.tables.length);
m_snapshotRecord =
SnapshotRegistry.startSnapshot(
txnId, context.getHostId(), file_path, file_nonce, SnapshotFormat.INDEX, config.tables);
// create table tasks
for (Table table : config.tables) {
createTasksForTable(
table, config.partitionRanges, pidToLocalHSIds, numTables, m_snapshotRecord);
result.addRow(
context.getHostId(),
CoreUtils.getHostnameOrAddress(),
table.getTypeName(),
"SUCCESS",
"");
}
return null;
}
protected boolean createSetupInternal(
String file_path,
String file_nonce,
long txnId,
Map<Integer, Long> partitionTransactionIds,
JSONObject jsData,
SystemProcedureExecutionContext context,
String hostname,
final VoltTable result,
Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers,
SiteTracker tracker,
long timestamp)
throws IOException {
assert (SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty());
/*
* List of partitions to include if this snapshot is
* going to be deduped. Attempts to break up the work
* by seeding and RNG selecting
* a random replica to do the work. Will not work in failure
* cases, but we don't use dedupe when we want durability.
*/
List<Long> sitesToInclude = CSVSnapshotWritePlan.computeDedupedLocalSites(txnId, tracker);
// If there's no work to do on this host, just claim success and get out:
if (sitesToInclude.isEmpty() && !tracker.isFirstHost()) {
return false;
}
NativeSnapshotWritePlan.createFileBasedCompletionTasks(
file_path,
file_nonce,
txnId,
partitionTransactionIds,
context,
exportSequenceNumbers,
timestamp);
final List<Table> tables = SnapshotUtil.getTablesToSave(context.getDatabase());
final AtomicInteger numTables = new AtomicInteger(tables.size());
final SnapshotRegistry.Snapshot snapshotRecord =
SnapshotRegistry.startSnapshot(
txnId,
context.getHostId(),
file_path,
file_nonce,
SnapshotFormat.CSV,
tables.toArray(new Table[0]));
SnapshotDataTarget sdt = null;
boolean noTargetsCreated = true;
final ArrayList<SnapshotTableTask> partitionedSnapshotTasks =
new ArrayList<SnapshotTableTask>();
final ArrayList<SnapshotTableTask> replicatedSnapshotTasks = new ArrayList<SnapshotTableTask>();
for (final Table table : tables) {
/*
* For a deduped csv snapshot, only produce the replicated tables on the "leader"
* host.
*/
if (table.getIsreplicated() && !tracker.isFirstHost()) {
snapshotRecord.removeTable(table.getTypeName());
continue;
}
File saveFilePath = null;
saveFilePath =
SnapshotUtil.constructFileForTable(
table, file_path, file_nonce, SnapshotFormat.CSV, context.getHostId());
try {
sdt = new SimpleFileSnapshotDataTarget(saveFilePath, !table.getIsreplicated());
m_targets.add(sdt);
final Runnable onClose =
new TargetStatsClosure(sdt, table.getTypeName(), numTables, snapshotRecord);
sdt.setOnCloseHandler(onClose);
List<SnapshotDataFilter> filters = new ArrayList<SnapshotDataFilter>();
filters.add(new CSVSnapshotFilter(CatalogUtil.getVoltTable(table), ',', null));
final SnapshotTableTask task =
new SnapshotTableTask(
table.getRelativeIndex(),
sdt,
filters.toArray(new SnapshotDataFilter[filters.size()]),
table.getIsreplicated(),
table.getTypeName());
if (table.getIsreplicated()) {
replicatedSnapshotTasks.add(task);
} else {
partitionedSnapshotTasks.add(task);
}
noTargetsCreated = false;
result.addRow(context.getHostId(), hostname, table.getTypeName(), "SUCCESS", "");
} catch (IOException ex) {
handleTargetCreationError(
sdt, context, file_nonce, hostname, table.getTypeName(), ex, result);
}
}
if (noTargetsCreated) {
SnapshotRegistry.discardSnapshot(snapshotRecord);
}
// CSV snapshots do the partitioned work only on the specified sites for de-duping,
// but since we've pre-filtered the replicated task list to only contain entries on
// one node, we can go ahead and distribute them across all of the sites on that node.
placePartitionedTasks(partitionedSnapshotTasks, sitesToInclude);
placeReplicatedTasks(replicatedSnapshotTasks, tracker.getSitesForHost(context.getHostId()));
return noTargetsCreated;
}