public long getQueuedExportBytes(int partitionId, String signature) {
// assert(m_dataSourcesByPartition.containsKey(partitionId));
// assert(m_dataSourcesByPartition.get(partitionId).containsKey(delegateId));
HashMap<String, ExportDataSource> sources = m_dataSourcesByPartition.get(partitionId);
if (sources == null) {
/*
* This is fine. If the table is dropped it won't have an entry in the generation created
* after the table was dropped.
*/
// exportLog.error("Could not find export data sources for generation " +
// m_timestamp + " partition "
// + partitionId);
return 0;
}
ExportDataSource source = sources.get(signature);
if (source == null) {
/*
* This is fine. If the table is dropped it won't have an entry in the generation created
* after the table was dropped.
*/
// exportLog.error("Could not find export data source for generation " + m_timestamp + "
// partition " + partitionId +
// " signature " + signature);
return 0;
}
return source.sizeInBytes();
}
/*
* An unfortunate test only method for supplying a mock source
*/
public void addDataSource(ExportDataSource source) {
HashMap<String, ExportDataSource> dataSourcesForPartition =
m_dataSourcesByPartition.get(source.getPartitionId());
if (dataSourcesForPartition == null) {
dataSourcesForPartition = new HashMap<String, ExportDataSource>();
m_dataSourcesByPartition.put(source.getPartitionId(), dataSourcesForPartition);
}
dataSourcesForPartition.put(source.getSignature(), source);
}
/**
* Indicate to all associated {@link ExportDataSource}to assume mastership role for the given
* partition id
*
* @param partitionId
*/
public void acceptMastershipTask(int partitionId) {
HashMap<String, ExportDataSource> partitionDataSourceMap =
m_dataSourcesByPartition.get(partitionId);
exportLog.info(
"Export generation " + m_timestamp + " accepting mastership for partition " + partitionId);
for (ExportDataSource eds : partitionDataSourceMap.values()) {
try {
eds.acceptMastership();
} catch (Exception e) {
exportLog.error("Unable to start exporting", e);
}
}
}
public void closeAndDelete() throws IOException {
List<ListenableFuture<?>> tasks = new ArrayList<ListenableFuture<?>>();
for (HashMap<String, ExportDataSource> map : m_dataSourcesByPartition.values()) {
for (ExportDataSource source : map.values()) {
tasks.add(source.closeAndDelete());
}
}
try {
Futures.allAsList(tasks).get();
} catch (Exception e) {
Throwables.propagateIfPossible(e, IOException.class);
}
shutdown = true;
VoltFile.recursivelyDelete(m_directory);
}
public void close() {
List<ListenableFuture<?>> tasks = new ArrayList<ListenableFuture<?>>();
for (HashMap<String, ExportDataSource> sources : m_dataSourcesByPartition.values()) {
for (ExportDataSource source : sources.values()) {
tasks.add(source.close());
}
}
try {
Futures.allAsList(tasks).get();
} catch (Exception e) {
// Logging of errors is done inside the tasks so nothing to do here
// intentionally not failing if there is an issue with close
exportLog.error("Error closing export data sources", e);
}
shutdown = true;
}
public void pushExportBuffer(
int partitionId,
String signature,
long uso,
long bufferPtr,
ByteBuffer buffer,
boolean sync,
boolean endOfStream) {
// System.out.println("In generation " + m_timestamp + " partition " + partitionId + "
// signature " + signature + (buffer == null ? " null buffer " : (" buffer length " +
// buffer.remaining())));
// for (Integer i : m_dataSourcesByPartition.keySet()) {
// System.out.println("Have partition " + i);
// }
assert (m_dataSourcesByPartition.containsKey(partitionId));
assert (m_dataSourcesByPartition.get(partitionId).containsKey(signature));
HashMap<String, ExportDataSource> sources = m_dataSourcesByPartition.get(partitionId);
if (sources == null) {
exportLog.error(
"Could not find export data sources for partition "
+ partitionId
+ " generation "
+ m_timestamp
+ " the export data is being discarded");
DBBPool.deleteCharArrayMemory(bufferPtr);
return;
}
ExportDataSource source = sources.get(signature);
if (source == null) {
exportLog.error(
"Could not find export data source for partition "
+ partitionId
+ " signature "
+ signature
+ " generation "
+ m_timestamp
+ " the export data is being discarded");
DBBPool.deleteCharArrayMemory(bufferPtr);
return;
}
source.pushExportBuffer(uso, bufferPtr, buffer, sync, endOfStream);
}
private void handleLeaderChildrenUpdate(Integer partition, List<String> children) {
if (m_drainedSources.get() == m_numSources || children.isEmpty()) {
return;
}
String leader = Collections.min(children);
if (m_partitionLeaderZKName.get(partition).equals(leader)) {
if (m_partitionsIKnowIAmTheLeader.add(partition)) {
for (ExportDataSource eds : m_dataSourcesByPartition.get(partition).values()) {
try {
eds.acceptMastership();
} catch (Exception e) {
exportLog.error("Unable to start exporting", e);
}
}
}
}
}
/*
* Returns true if the generatino was completely truncated away
*/
public boolean truncateExportToTxnId(long txnId, long[] perPartitionTxnIds) {
// create an easy partitionId:txnId lookup.
HashMap<Integer, Long> partitionToTxnId = new HashMap<Integer, Long>();
for (long tid : perPartitionTxnIds) {
partitionToTxnId.put(TxnEgo.getPartitionId(tid), tid);
}
List<ListenableFuture<?>> tasks = new ArrayList<ListenableFuture<?>>();
// pre-iv2, the truncation point is the snapshot transaction id.
// In iv2, truncation at the per-partition txn id recorded in the snapshot.
for (HashMap<String, ExportDataSource> dataSources : m_dataSourcesByPartition.values()) {
for (ExportDataSource source : dataSources.values()) {
if (VoltDB.instance().isIV2Enabled()) {
Long truncationPoint = partitionToTxnId.get(source.getPartitionId());
if (truncationPoint == null) {
exportLog.error(
"Snapshot "
+ txnId
+ " does not include truncation point for partition "
+ source.getPartitionId());
} else {
tasks.add(source.truncateExportToTxnId(truncationPoint));
}
} else {
tasks.add(source.truncateExportToTxnId(txnId));
}
}
}
try {
Futures.allAsList(tasks).get();
} catch (Exception e) {
VoltDB.crashLocalVoltDB(
"Unexpected exception truncating export data during snapshot restore. "
+ "You can back up export overflow data and start the "
+ "DB without it to get past this error",
true,
e);
}
return m_drainedSources.get() == m_numSources;
}
/*
* Create a datasource based on an ad file
*/
private void addDataSource(File adFile, Set<Integer> partitions) throws IOException {
m_numSources++;
ExportDataSource source = new ExportDataSource(m_onSourceDrained, adFile);
partitions.add(source.getPartitionId());
m_timestamp = source.getGeneration();
exportLog.info(
"Creating ExportDataSource for "
+ adFile
+ " table "
+ source.getTableName()
+ " signature "
+ source.getSignature()
+ " partition id "
+ source.getPartitionId()
+ " bytes "
+ source.sizeInBytes());
HashMap<String, ExportDataSource> dataSourcesForPartition =
m_dataSourcesByPartition.get(source.getPartitionId());
if (dataSourcesForPartition == null) {
dataSourcesForPartition = new HashMap<String, ExportDataSource>();
m_dataSourcesByPartition.put(source.getPartitionId(), dataSourcesForPartition);
}
dataSourcesForPartition.put(source.getSignature(), source);
}