public long getQueuedExportBytes(int partitionId, String signature) { // assert(m_dataSourcesByPartition.containsKey(partitionId)); // assert(m_dataSourcesByPartition.get(partitionId).containsKey(delegateId)); HashMap<String, ExportDataSource> sources = m_dataSourcesByPartition.get(partitionId); if (sources == null) { /* * This is fine. If the table is dropped it won't have an entry in the generation created * after the table was dropped. */ // exportLog.error("Could not find export data sources for generation " + // m_timestamp + " partition " // + partitionId); return 0; } ExportDataSource source = sources.get(signature); if (source == null) { /* * This is fine. If the table is dropped it won't have an entry in the generation created * after the table was dropped. */ // exportLog.error("Could not find export data source for generation " + m_timestamp + " // partition " + partitionId + // " signature " + signature); return 0; } return source.sizeInBytes(); }
/* * An unfortunate test only method for supplying a mock source */ public void addDataSource(ExportDataSource source) { HashMap<String, ExportDataSource> dataSourcesForPartition = m_dataSourcesByPartition.get(source.getPartitionId()); if (dataSourcesForPartition == null) { dataSourcesForPartition = new HashMap<String, ExportDataSource>(); m_dataSourcesByPartition.put(source.getPartitionId(), dataSourcesForPartition); } dataSourcesForPartition.put(source.getSignature(), source); }
/** * Indicate to all associated {@link ExportDataSource}to assume mastership role for the given * partition id * * @param partitionId */ public void acceptMastershipTask(int partitionId) { HashMap<String, ExportDataSource> partitionDataSourceMap = m_dataSourcesByPartition.get(partitionId); exportLog.info( "Export generation " + m_timestamp + " accepting mastership for partition " + partitionId); for (ExportDataSource eds : partitionDataSourceMap.values()) { try { eds.acceptMastership(); } catch (Exception e) { exportLog.error("Unable to start exporting", e); } } }
public void closeAndDelete() throws IOException { List<ListenableFuture<?>> tasks = new ArrayList<ListenableFuture<?>>(); for (HashMap<String, ExportDataSource> map : m_dataSourcesByPartition.values()) { for (ExportDataSource source : map.values()) { tasks.add(source.closeAndDelete()); } } try { Futures.allAsList(tasks).get(); } catch (Exception e) { Throwables.propagateIfPossible(e, IOException.class); } shutdown = true; VoltFile.recursivelyDelete(m_directory); }
public void close() { List<ListenableFuture<?>> tasks = new ArrayList<ListenableFuture<?>>(); for (HashMap<String, ExportDataSource> sources : m_dataSourcesByPartition.values()) { for (ExportDataSource source : sources.values()) { tasks.add(source.close()); } } try { Futures.allAsList(tasks).get(); } catch (Exception e) { // Logging of errors is done inside the tasks so nothing to do here // intentionally not failing if there is an issue with close exportLog.error("Error closing export data sources", e); } shutdown = true; }
public void pushExportBuffer( int partitionId, String signature, long uso, long bufferPtr, ByteBuffer buffer, boolean sync, boolean endOfStream) { // System.out.println("In generation " + m_timestamp + " partition " + partitionId + " // signature " + signature + (buffer == null ? " null buffer " : (" buffer length " + // buffer.remaining()))); // for (Integer i : m_dataSourcesByPartition.keySet()) { // System.out.println("Have partition " + i); // } assert (m_dataSourcesByPartition.containsKey(partitionId)); assert (m_dataSourcesByPartition.get(partitionId).containsKey(signature)); HashMap<String, ExportDataSource> sources = m_dataSourcesByPartition.get(partitionId); if (sources == null) { exportLog.error( "Could not find export data sources for partition " + partitionId + " generation " + m_timestamp + " the export data is being discarded"); DBBPool.deleteCharArrayMemory(bufferPtr); return; } ExportDataSource source = sources.get(signature); if (source == null) { exportLog.error( "Could not find export data source for partition " + partitionId + " signature " + signature + " generation " + m_timestamp + " the export data is being discarded"); DBBPool.deleteCharArrayMemory(bufferPtr); return; } source.pushExportBuffer(uso, bufferPtr, buffer, sync, endOfStream); }
private void handleLeaderChildrenUpdate(Integer partition, List<String> children) { if (m_drainedSources.get() == m_numSources || children.isEmpty()) { return; } String leader = Collections.min(children); if (m_partitionLeaderZKName.get(partition).equals(leader)) { if (m_partitionsIKnowIAmTheLeader.add(partition)) { for (ExportDataSource eds : m_dataSourcesByPartition.get(partition).values()) { try { eds.acceptMastership(); } catch (Exception e) { exportLog.error("Unable to start exporting", e); } } } } }
/* * Returns true if the generatino was completely truncated away */ public boolean truncateExportToTxnId(long txnId, long[] perPartitionTxnIds) { // create an easy partitionId:txnId lookup. HashMap<Integer, Long> partitionToTxnId = new HashMap<Integer, Long>(); for (long tid : perPartitionTxnIds) { partitionToTxnId.put(TxnEgo.getPartitionId(tid), tid); } List<ListenableFuture<?>> tasks = new ArrayList<ListenableFuture<?>>(); // pre-iv2, the truncation point is the snapshot transaction id. // In iv2, truncation at the per-partition txn id recorded in the snapshot. for (HashMap<String, ExportDataSource> dataSources : m_dataSourcesByPartition.values()) { for (ExportDataSource source : dataSources.values()) { if (VoltDB.instance().isIV2Enabled()) { Long truncationPoint = partitionToTxnId.get(source.getPartitionId()); if (truncationPoint == null) { exportLog.error( "Snapshot " + txnId + " does not include truncation point for partition " + source.getPartitionId()); } else { tasks.add(source.truncateExportToTxnId(truncationPoint)); } } else { tasks.add(source.truncateExportToTxnId(txnId)); } } } try { Futures.allAsList(tasks).get(); } catch (Exception e) { VoltDB.crashLocalVoltDB( "Unexpected exception truncating export data during snapshot restore. " + "You can back up export overflow data and start the " + "DB without it to get past this error", true, e); } return m_drainedSources.get() == m_numSources; }
/* * Create a datasource based on an ad file */ private void addDataSource(File adFile, Set<Integer> partitions) throws IOException { m_numSources++; ExportDataSource source = new ExportDataSource(m_onSourceDrained, adFile); partitions.add(source.getPartitionId()); m_timestamp = source.getGeneration(); exportLog.info( "Creating ExportDataSource for " + adFile + " table " + source.getTableName() + " signature " + source.getSignature() + " partition id " + source.getPartitionId() + " bytes " + source.sizeInBytes()); HashMap<String, ExportDataSource> dataSourcesForPartition = m_dataSourcesByPartition.get(source.getPartitionId()); if (dataSourcesForPartition == null) { dataSourcesForPartition = new HashMap<String, ExportDataSource>(); m_dataSourcesByPartition.put(source.getPartitionId(), dataSourcesForPartition); } dataSourcesForPartition.put(source.getSignature(), source); }