Example #1
0
  /**
   * Creates a new RocksDB backed state and restores from the given backup directory. After
   * restoring the backup directory is deleted.
   *
   * @param keySerializer The serializer for the keys.
   * @param namespaceSerializer The serializer for the namespace.
   * @param basePath The path on the local system where RocksDB data should be stored.
   * @param restorePath The path to a backup directory from which to restore RocksDb database.
   */
  protected AbstractRocksDBState(
      TypeSerializer<K> keySerializer,
      TypeSerializer<N> namespaceSerializer,
      File basePath,
      String checkpointPath,
      String restorePath,
      Options options) {

    rocksDbPath = new File(basePath, "db" + UUID.randomUUID().toString());
    hadoopConfPath = new File(basePath, HADOOP_CONF_NAME);

    RocksDB.loadLibrary();

    // clean it, this will remove the last part of the path but RocksDB will recreate it
    try {
      if (rocksDbPath.exists()) {
        LOG.warn("Deleting already existing db directory {}.", rocksDbPath);
        FileUtils.deleteDirectory(rocksDbPath);
      }
    } catch (IOException e) {
      throw new RuntimeException("Error cleaning RocksDB data directory.", e);
    }

    try (BackupEngine backupEngine =
        BackupEngine.open(Env.getDefault(), new BackupableDBOptions(restorePath + "/"))) {
      backupEngine.restoreDbFromLatestBackup(
          rocksDbPath.getAbsolutePath(), rocksDbPath.getAbsolutePath(), new RestoreOptions(true));
    } catch (RocksDBException | IllegalArgumentException e) {
      throw new RuntimeException("Error while restoring RocksDB state from " + restorePath, e);
    } finally {
      try {
        FileUtils.deleteDirectory(new File(restorePath));
      } catch (IOException e) {
        LOG.error("Error cleaning up local restore directory " + restorePath, e);
      }
    }

    this.keySerializer = requireNonNull(keySerializer);
    this.namespaceSerializer = namespaceSerializer;
    this.basePath = basePath;
    this.checkpointPath = checkpointPath;

    if (!basePath.exists()) {
      if (!basePath.mkdirs()) {
        throw new RuntimeException("Could not create RocksDB data directory.");
      }
    }

    try {
      db = RocksDB.open(options, rocksDbPath.getAbsolutePath());
    } catch (RocksDBException e) {
      throw new RuntimeException("Error while opening RocksDB instance.", e);
    }

    writeHadoopConfig(hadoopConfPath);
  }
Example #2
0
  @Override
  public final KvStateSnapshot<K, N, S, SD, RocksDBStateBackend> snapshot(
      final long checkpointId, long timestamp) throws Exception {

    final File localBackupPath = new File(basePath, "local-chk-" + checkpointId);
    final URI backupUri = new URI(checkpointPath + "/chk-" + checkpointId);

    if (!localBackupPath.exists()) {
      if (!localBackupPath.mkdirs()) {
        throw new RuntimeException("Could not create local backup path " + localBackupPath);
      }
    }

    try (BackupEngine backupEngine =
        BackupEngine.open(
            Env.getDefault(), new BackupableDBOptions(localBackupPath.getAbsolutePath()))) {
      backupEngine.createNewBackup(db);
    }

    return new AsyncRocksDBSnapshot<>(localBackupPath, backupUri, checkpointId, this);
  }