private static Map<Integer, Long> findLocalSources(
Collection<IndexSnapshotRequestConfig.PartitionRanges> partitionRanges, SiteTracker tracker) {
Set<Integer> partitions = Sets.newHashSet();
for (IndexSnapshotRequestConfig.PartitionRanges partitionRange : partitionRanges) {
partitions.add(partitionRange.partitionId);
}
Map<Integer, Long> pidToLocalHSId = Maps.newHashMap();
List<Long> localSites = Longs.asList(tracker.getLocalSites());
for (long hsId : localSites) {
int pid = tracker.getPartitionForSite(hsId);
if (partitions.contains(pid)) {
pidToLocalHSId.put(pid, hsId);
}
}
return pidToLocalHSId;
}
public static String getLiveSystemOverview() {
// get the start time
long t = SystemStatsCollector.getStartTime();
Date date = new Date(t);
long duration = System.currentTimeMillis() - t;
long minutes = duration / 60000;
long hours = minutes / 60;
minutes -= hours * 60;
long days = hours / 24;
hours -= days * 24;
String starttime = String.format("%s (%dd %dh %dm)", date.toString(), days, hours, minutes);
// handle the basic info page below this
SiteTracker st = VoltDB.instance().getSiteTrackerForSnapshot();
// get the cluster info
String clusterinfo = st.getAllHosts().size() + " hosts ";
clusterinfo += " with " + st.getAllSites().size() + " sites ";
clusterinfo += " (" + st.getAllSites().size() / st.getAllHosts().size();
clusterinfo += " per host)";
StringBuilder sb = new StringBuilder();
sb.append("<table class='table table-condensed'>\n");
sb.append(
"<tr><td>Mode </td><td>"
+ VoltDB.instance().getMode().toString()
+ "</td><td>\n");
sb.append(
"<tr><td>VoltDB Version </td><td>"
+ VoltDB.instance().getVersionString()
+ "</td><td>\n");
sb.append(
"<tr><td>Buildstring </td><td>"
+ VoltDB.instance().getBuildString()
+ "</td><td>\n");
sb.append("<tr><td>Cluster Composition </td><td>" + clusterinfo + "</td><td>\n");
sb.append("<tr><td>Running Since </td><td>" + starttime + "</td><td>\n");
sb.append("</table>\n");
return sb.toString();
}
private static List<Long> computeDedupedLocalSites(long txnId, SiteTracker tracker) {
MessageDigest digest;
try {
digest = MessageDigest.getInstance("SHA-1");
} catch (NoSuchAlgorithmException e) {
throw new AssertionError(e);
}
/*
* List of partitions to include if this snapshot is
* going to be deduped. Attempts to break up the work
* by seeding and RNG selecting
* a random replica to do the work. Will not work in failure
* cases, but we don't use dedupe when we want durability.
*
* Originally used the partition id as the seed, but it turns out
* that nextInt(2) returns a 1 for seeds 0-4095. Now use SHA-1
* on the txnid + partition id.
*/
List<Long> sitesToInclude = new ArrayList<Long>();
for (long localSite : tracker.getLocalSites()) {
final int partitionId = tracker.getPartitionForSite(localSite);
List<Long> sites =
new ArrayList<Long>(tracker.getSitesForPartition(tracker.getPartitionForSite(localSite)));
Collections.sort(sites);
digest.update(Longs.toByteArray(txnId));
final long seed =
Longs.fromByteArray(Arrays.copyOf(digest.digest(Ints.toByteArray(partitionId)), 8));
int siteIndex = new java.util.Random(seed).nextInt(sites.size());
if (localSite == sites.get(siteIndex)) {
sitesToInclude.add(localSite);
}
}
return sitesToInclude;
}
protected boolean createSetupInternal(
String file_path,
String file_nonce,
long txnId,
Map<Integer, Long> partitionTransactionIds,
JSONObject jsData,
SystemProcedureExecutionContext context,
String hostname,
final VoltTable result,
Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers,
SiteTracker tracker,
long timestamp)
throws IOException {
assert (SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty());
/*
* List of partitions to include if this snapshot is
* going to be deduped. Attempts to break up the work
* by seeding and RNG selecting
* a random replica to do the work. Will not work in failure
* cases, but we don't use dedupe when we want durability.
*/
List<Long> sitesToInclude = CSVSnapshotWritePlan.computeDedupedLocalSites(txnId, tracker);
// If there's no work to do on this host, just claim success and get out:
if (sitesToInclude.isEmpty() && !tracker.isFirstHost()) {
return false;
}
NativeSnapshotWritePlan.createFileBasedCompletionTasks(
file_path,
file_nonce,
txnId,
partitionTransactionIds,
context,
exportSequenceNumbers,
timestamp);
final List<Table> tables = SnapshotUtil.getTablesToSave(context.getDatabase());
final AtomicInteger numTables = new AtomicInteger(tables.size());
final SnapshotRegistry.Snapshot snapshotRecord =
SnapshotRegistry.startSnapshot(
txnId,
context.getHostId(),
file_path,
file_nonce,
SnapshotFormat.CSV,
tables.toArray(new Table[0]));
SnapshotDataTarget sdt = null;
boolean noTargetsCreated = true;
final ArrayList<SnapshotTableTask> partitionedSnapshotTasks =
new ArrayList<SnapshotTableTask>();
final ArrayList<SnapshotTableTask> replicatedSnapshotTasks = new ArrayList<SnapshotTableTask>();
for (final Table table : tables) {
/*
* For a deduped csv snapshot, only produce the replicated tables on the "leader"
* host.
*/
if (table.getIsreplicated() && !tracker.isFirstHost()) {
snapshotRecord.removeTable(table.getTypeName());
continue;
}
File saveFilePath = null;
saveFilePath =
SnapshotUtil.constructFileForTable(
table, file_path, file_nonce, SnapshotFormat.CSV, context.getHostId());
try {
sdt = new SimpleFileSnapshotDataTarget(saveFilePath, !table.getIsreplicated());
m_targets.add(sdt);
final Runnable onClose =
new TargetStatsClosure(sdt, table.getTypeName(), numTables, snapshotRecord);
sdt.setOnCloseHandler(onClose);
List<SnapshotDataFilter> filters = new ArrayList<SnapshotDataFilter>();
filters.add(new CSVSnapshotFilter(CatalogUtil.getVoltTable(table), ',', null));
final SnapshotTableTask task =
new SnapshotTableTask(
table.getRelativeIndex(),
sdt,
filters.toArray(new SnapshotDataFilter[filters.size()]),
table.getIsreplicated(),
table.getTypeName());
if (table.getIsreplicated()) {
replicatedSnapshotTasks.add(task);
} else {
partitionedSnapshotTasks.add(task);
}
noTargetsCreated = false;
result.addRow(context.getHostId(), hostname, table.getTypeName(), "SUCCESS", "");
} catch (IOException ex) {
handleTargetCreationError(
sdt, context, file_nonce, hostname, table.getTypeName(), ex, result);
}
}
if (noTargetsCreated) {
SnapshotRegistry.discardSnapshot(snapshotRecord);
}
// CSV snapshots do the partitioned work only on the specified sites for de-duping,
// but since we've pre-filtered the replicated task list to only contain entries on
// one node, we can go ahead and distribute them across all of the sites on that node.
placePartitionedTasks(partitionedSnapshotTasks, sitesToInclude);
placeReplicatedTasks(replicatedSnapshotTasks, tracker.getSitesForHost(context.getHostId()));
return noTargetsCreated;
}