protected void initialize(
     BlobStore blobStore, ClusterName clusterName, @Nullable ByteSizeValue defaultChunkSize)
     throws IOException {
   this.blobStore = blobStore;
   this.chunkSize = componentSettings.getAsBytesSize("chunk_size", defaultChunkSize);
   this.basePath = BlobPath.cleanPath().add(clusterName.value());
   this.metaDataBlobContainer = blobStore.immutableBlobContainer(basePath.add("metadata"));
   this.currentIndex = findLatestIndex();
   this.compress = componentSettings.getAsBoolean("compress", true);
   logger.debug("Latest metadata found at index [" + currentIndex + "]");
 }
 /**
  * Delete an arbitrary BlobPath from our store.
  *
  * @param path The blob path to delete
  */
 @Override
 public void delete(BlobPath path) {
   String keyPath = path.buildAsString("/");
   if (!keyPath.isEmpty()) {
     keyPath = keyPath + "/";
   }
   StoredObject obj = swift().getObject(keyPath);
   if (obj.exists()) {
     obj.delete();
   }
 }
Example #3
0
 /**
  * Builds URL using base URL and specified path
  *
  * @param path relative path
  * @return Base URL + path
  * @throws MalformedURLException
  */
 private URL buildPath(BlobPath path) throws MalformedURLException {
   String[] paths = path.toArray();
   if (paths.length == 0) {
     return path();
   }
   URL blobPath = new URL(this.path, paths[0] + "/");
   if (paths.length > 1) {
     for (int i = 1; i < paths.length; i++) {
       blobPath = new URL(blobPath, paths[i] + "/");
     }
   }
   return blobPath;
 }
  /**
   * Constructs new shared file system repository
   *
   * @param name repository name
   * @param repositorySettings repository settings
   * @param indexShardRepository index shard repository
   * @param s3Service S3 service
   * @throws IOException
   */
  @Inject
  public S3Repository(
      RepositoryName name,
      RepositorySettings repositorySettings,
      IndexShardRepository indexShardRepository,
      AwsS3Service s3Service)
      throws IOException {
    super(name.getName(), repositorySettings, indexShardRepository);

    String bucket = repositorySettings.settings().get("bucket", componentSettings.get("bucket"));
    if (bucket == null) {
      throw new RepositoryException(name.name(), "No bucket defined for s3 gateway");
    }

    String region = repositorySettings.settings().get("region", componentSettings.get("region"));
    if (region == null) {
      // Bucket setting is not set - use global region setting
      String regionSetting =
          repositorySettings.settings().get("cloud.aws.region", settings.get("cloud.aws.region"));
      if (regionSetting != null) {
        regionSetting = regionSetting.toLowerCase(Locale.ENGLISH);
        if ("us-east".equals(regionSetting)) {
          // Default bucket - setting region to null
          region = null;
        } else if ("us-east-1".equals(regionSetting)) {
          region = null;
        } else if ("us-west".equals(regionSetting)) {
          region = "us-west-1";
        } else if ("us-west-1".equals(regionSetting)) {
          region = "us-west-1";
        } else if ("us-west-2".equals(regionSetting)) {
          region = "us-west-2";
        } else if ("ap-southeast".equals(regionSetting)) {
          region = "ap-southeast-1";
        } else if ("ap-southeast-1".equals(regionSetting)) {
          region = "ap-southeast-1";
        } else if ("ap-southeast-2".equals(regionSetting)) {
          region = "ap-southeast-2";
        } else if ("ap-northeast".equals(regionSetting)) {
          region = "ap-northeast-1";
        } else if ("ap-northeast-1".equals(regionSetting)) {
          region = "ap-northeast-1";
        } else if ("eu-west".equals(regionSetting)) {
          region = "EU";
        } else if ("eu-west-1".equals(regionSetting)) {
          region = "EU";
        } else if ("sa-east".equals(regionSetting)) {
          region = "sa-east-1";
        } else if ("sa-east-1".equals(regionSetting)) {
          region = "sa-east-1";
        }
      }
    }
    int concurrentStreams =
        repositorySettings
            .settings()
            .getAsInt("concurrent_streams", componentSettings.getAsInt("concurrent_streams", 5));
    ExecutorService concurrentStreamPool =
        EsExecutors.newScaling(
            1,
            concurrentStreams,
            5,
            TimeUnit.SECONDS,
            EsExecutors.daemonThreadFactory(settings, "[s3_stream]"));

    boolean serverSideEncryption =
        repositorySettings
            .settings()
            .getAsBoolean(
                "server_side_encryption",
                componentSettings.getAsBoolean("server_side_encryption", false));

    logger.debug(
        "using bucket [{}], region [{}], chunk_size [{}], concurrent_streams [{}], server_side_encryption [{}]",
        bucket,
        region,
        chunkSize,
        concurrentStreams,
        serverSideEncryption);
    blobStore =
        new S3BlobStore(
            settings,
            s3Service.client(
                region,
                repositorySettings.settings().get("access_key"),
                repositorySettings.settings().get("secret_key")),
            bucket,
            region,
            concurrentStreamPool,
            serverSideEncryption);
    this.chunkSize =
        repositorySettings
            .settings()
            .getAsBytesSize(
                "chunk_size",
                componentSettings.getAsBytesSize(
                    "chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB)));
    this.compress =
        repositorySettings
            .settings()
            .getAsBoolean("compress", componentSettings.getAsBoolean("compress", false));
    String basePath = repositorySettings.settings().get("base_path", null);
    if (Strings.hasLength(basePath)) {
      BlobPath path = new BlobPath();
      for (String elem : Strings.splitStringToArray(basePath, '/')) {
        path = path.add(elem);
      }
      this.basePath = path;
    } else {
      this.basePath = BlobPath.cleanPath();
    }
  }
 public BlobPath shardPath(int shardId) {
   return indexPath.add(Integer.toString(shardId));
 }
 @Override
 public void reset() throws Exception {
   blobStore.delete(BlobPath.cleanPath());
 }