public JournalTransactionStore(
final String dataPath, final MessageStoreManager storeManager, final MetaConfig metaConfig)
throws Exception {
this.journalStore =
new JournalStore(
dataPath,
storeManager,
this,
metaConfig.getMaxCheckpoints(),
metaConfig.getFlushTxLogAtCommit());
this.scheduledExecutorService.scheduleAtFixedRate(
new Runnable() {
@Override
public void run() {
try {
JournalTransactionStore.this.makeCheckpoint();
} catch (final Exception e) {
log.error("Execute checkpoint failed", e);
}
}
},
metaConfig.getCheckpointInterval(),
metaConfig.getCheckpointInterval(),
TimeUnit.MILLISECONDS);
MetaMBeanServer.registMBean(this, null);
}
DeletePolicySelector(final MetaConfig metaConfig) {
for (final String topic : metaConfig.getTopics()) {
final TopicConfig topicConfig = metaConfig.getTopicConfig(topic);
final String deletePolicy =
topicConfig != null ? topicConfig.getDeletePolicy() : metaConfig.getDeletePolicy();
this.deletePolicyMap.put(topic, DeletePolicyFactory.getDeletePolicy(deletePolicy));
}
}
private Set<File> getDataDirSet(final MetaConfig metaConfig) throws IOException {
final Set<String> paths = new HashSet<String>();
// public data path
paths.add(metaConfig.getDataPath());
// topic data path
for (final String topic : metaConfig.getTopics()) {
final TopicConfig topicConfig = metaConfig.getTopicConfig(topic);
if (topicConfig != null) {
paths.add(topicConfig.getDataPath());
}
}
final Set<File> fileSet = new HashSet<File>();
for (final String path : paths) {
fileSet.add(this.getDataDir(path));
}
return fileSet;
}
private void loadDataDir(final MetaConfig metaConfig, final File dir)
throws IOException, InterruptedException {
log.warn("Begin to scan data path:" + dir.getAbsolutePath());
final long start = System.currentTimeMillis();
final File[] ls = dir.listFiles();
int nThreads = Runtime.getRuntime().availableProcessors() + 1;
ExecutorService executor = Executors.newFixedThreadPool(nThreads);
int count = 0;
List<Callable<MessageStore>> tasks = new ArrayList<Callable<MessageStore>>();
for (final File subDir : ls) {
if (!subDir.isDirectory()) {
log.warn("Ignore not directory path:" + subDir.getAbsolutePath());
} else {
final String name = subDir.getName();
final int index = name.lastIndexOf('-');
if (index < 0) {
log.warn("Ignore invlaid directory:" + subDir.getAbsolutePath());
continue;
}
tasks.add(
new Callable<MessageStore>() {
@Override
public MessageStore call() throws Exception {
log.warn("Loading data directory:" + subDir.getAbsolutePath() + "...");
final String topic = name.substring(0, index);
final int partition = Integer.parseInt(name.substring(index + 1));
final MessageStore messageStore =
new MessageStore(
topic,
partition,
metaConfig,
MessageStoreManager.this.deletePolicySelector.select(
topic, MessageStoreManager.this.deletePolicy));
return messageStore;
}
});
count++;
if (count % nThreads == 0 || count == ls.length) {
if (metaConfig.isLoadMessageStoresInParallel()) {
this.loadStoresInParallel(executor, tasks);
} else {
this.loadStores(tasks);
}
}
}
}
executor.shutdownNow();
log.warn("End to scan data path in " + (System.currentTimeMillis() - start) / 1000 + " secs");
}