public void testSnapshotRecovery() throws Exception { logger.info("--> start node A"); String nodeA = internalCluster().startNode(); logger.info("--> create repository"); assertAcked( client() .admin() .cluster() .preparePutRepository(REPO_NAME) .setType("fs") .setSettings( Settings.builder().put("location", randomRepoPath()).put("compress", false)) .get()); ensureGreen(); logger.info("--> create index on node: {}", nodeA); createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client() .admin() .cluster() .prepareCreateSnapshot(REPO_NAME, SNAP_NAME) .setWaitForCompletion(true) .setIndices(INDEX_NAME) .get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat( createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat( client() .admin() .cluster() .prepareGetSnapshots(REPO_NAME) .setSnapshots(SNAP_NAME) .get() .getSnapshots() .get(0) .state(), equalTo(SnapshotState.SUCCESS)); client().admin().indices().prepareClose(INDEX_NAME).execute().actionGet(); logger.info("--> restore"); RestoreSnapshotResponse restoreSnapshotResponse = client() .admin() .cluster() .prepareRestoreSnapshot(REPO_NAME, SNAP_NAME) .setWaitForCompletion(true) .execute() .actionGet(); int totalShards = restoreSnapshotResponse.getRestoreInfo().totalShards(); assertThat(totalShards, greaterThan(0)); ensureGreen(); logger.info("--> request recoveries"); RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet(); for (Map.Entry<String, List<RecoveryState>> indexRecoveryStates : response.shardRecoveryStates().entrySet()) { assertThat(indexRecoveryStates.getKey(), equalTo(INDEX_NAME)); List<RecoveryState> recoveryStates = indexRecoveryStates.getValue(); assertThat(recoveryStates.size(), equalTo(totalShards)); for (RecoveryState recoveryState : recoveryStates) { SnapshotRecoverySource recoverySource = new SnapshotRecoverySource( new Snapshot(REPO_NAME, createSnapshotResponse.getSnapshotInfo().snapshotId()), Version.CURRENT, INDEX_NAME); assertRecoveryState(recoveryState, 0, recoverySource, true, Stage.DONE, null, nodeA); validateIndexRecoveryState(recoveryState.getIndex()); } } }
/** * Tests the case where we create an index without shadow replicas, snapshot it and then restore * into an index with shadow replicas enabled. */ public void testRestoreToShadow() throws ExecutionException, InterruptedException { final Path dataPath = createTempDir(); Settings nodeSettings = nodeSettings(dataPath); internalCluster().startNodesAsync(3, nodeSettings).get(); Settings idxSettings = Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .build(); assertAcked(prepareCreate("foo").setSettings(idxSettings)); ensureGreen(); final int numDocs = randomIntBetween(10, 100); for (int i = 0; i < numDocs; i++) { client().prepareIndex("foo", "doc", "" + i).setSource("foo", "bar").get(); } assertNoFailures( client().admin().indices().prepareFlush().setForce(true).execute().actionGet()); assertAcked( client() .admin() .cluster() .preparePutRepository("test-repo") .setType("fs") .setSettings(Settings.builder().put("location", randomRepoPath()))); CreateSnapshotResponse createSnapshotResponse = client() .admin() .cluster() .prepareCreateSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) .setIndices("foo") .get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat( createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat( client() .admin() .cluster() .prepareGetSnapshots("test-repo") .setSnapshots("test-snap") .get() .getSnapshots() .get(0) .state(), equalTo(SnapshotState.SUCCESS)); Settings shadowSettings = Settings.builder() .put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString()) .put(IndexMetaData.SETTING_SHADOW_REPLICAS, true) .put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2) .build(); logger.info("--> restore the index into shadow replica index"); RestoreSnapshotResponse restoreSnapshotResponse = client() .admin() .cluster() .prepareRestoreSnapshot("test-repo", "test-snap") .setIndexSettings(shadowSettings) .setWaitForCompletion(true) .setRenamePattern("(.+)") .setRenameReplacement("$1-copy") .execute() .actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); refresh(); Index index = resolveIndex("foo-copy"); for (IndicesService service : internalCluster().getDataNodeInstances(IndicesService.class)) { if (service.hasIndex(index)) { IndexShard shard = service.indexServiceSafe(index).getShardOrNull(0); if (shard.routingEntry().primary()) { assertFalse(shard instanceof ShadowIndexShard); } else { assertTrue(shard instanceof ShadowIndexShard); } } } logger.info("--> performing query"); SearchResponse resp = client().prepareSearch("foo-copy").setQuery(matchAllQuery()).get(); assertHitCount(resp, numDocs); }
/** * Tests that restoring of a corrupted shard fails and we get a partial snapshot. TODO once * checksum verification on snapshotting is implemented this test needs to be fixed or split into * several parts... We should also corrupt files on the actual snapshot and check that we don't * restore the corrupted shard. */ @TestLogging("monitor.fs:DEBUG") public void testCorruptFileThenSnapshotAndRestore() throws ExecutionException, InterruptedException, IOException { int numDocs = scaledRandomIntBetween(100, 1000); internalCluster().ensureAtLeastNumDataNodes(2); assertAcked( prepareCreate("test") .setSettings( Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") // no replicas for this test .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) .put( MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose .put( IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue( 1, ByteSizeUnit .PB)) // no translog based flush - it might change the .liv / // segments.N files )); ensureGreen(); IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; for (int i = 0; i < builders.length; i++) { builders[i] = client().prepareIndex("test", "type").setSource("field", "value"); } indexRandom(true, builders); ensureGreen(); assertAllSuccessful( client() .admin() .indices() .prepareFlush() .setForce(true) .setWaitIfOngoing(true) .execute() .actionGet()); // we have to flush at least once here since we don't corrupt the translog SearchResponse countResponse = client().prepareSearch().setSize(0).get(); assertHitCount(countResponse, numDocs); ShardRouting shardRouting = corruptRandomPrimaryFile(false); // we don't corrupt segments.gen since S/R doesn't snapshot this file // the other problem here why we can't corrupt segments.X files is that the snapshot flushes // again before // it snapshots and that will write a new segments.X+1 file logger.info("--> creating repository"); assertAcked( client() .admin() .cluster() .preparePutRepository("test-repo") .setType("fs") .setSettings( Settings.builder() .put("location", randomRepoPath().toAbsolutePath()) .put("compress", randomBoolean()) .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client() .admin() .cluster() .prepareCreateSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) .setIndices("test") .get(); assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.PARTIAL)); logger.info("failed during snapshot -- maybe SI file got corrupted"); final List<Path> files = listShardFiles(shardRouting); Path corruptedFile = null; for (Path file : files) { if (file.getFileName().toString().startsWith("corrupted_")) { corruptedFile = file; break; } } assertThat(corruptedFile, notNullValue()); }
@Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") public void testSimpleWorkflow() { Client client = client(); logger.info( "--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); PutRepositoryResponse putRepositoryResponse = client .admin() .cluster() .preparePutRepository("test-repo") .setType("s3") .setSettings( Settings.settingsBuilder() .put("base_path", basePath) .put("chunk_size", randomIntBetween(1000, 10000))) .get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); createIndex("test-idx-1", "test-idx-2", "test-idx-3"); ensureGreen(); logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i); index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client .admin() .cluster() .prepareCreateSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) .setIndices("test-idx-*", "-test-idx-3") .get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat( createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat( client .admin() .cluster() .prepareGetSnapshots("test-repo") .setSnapshots("test-snap") .get() .getSnapshots() .get(0) .state(), equalTo(SnapshotState.SUCCESS)); logger.info("--> delete some data"); for (int i = 0; i < 50; i++) { client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get(); } for (int i = 50; i < 100; i++) { client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get(); } for (int i = 0; i < 100; i += 2) { client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); } refresh(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); logger.info("--> close indices"); client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); logger.info("--> restore all indices from the snapshot"); RestoreSnapshotResponse restoreSnapshotResponse = client .admin() .cluster() .prepareRestoreSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) .execute() .actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); // Test restore after index deletion logger.info("--> delete indices"); cluster().wipeIndices("test-idx-1", "test-idx-2"); logger.info("--> restore one index after deletion"); restoreSnapshotResponse = client .admin() .cluster() .prepareRestoreSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) .setIndices("test-idx-*", "-test-idx-2") .execute() .actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); }
@Test public void testSimpleWorkflow() { Client client = client(); logger.info("--> creating hdfs repository with path [{}]", path); PutRepositoryResponse putRepositoryResponse = client .admin() .cluster() .preparePutRepository("test-repo") .setType("hdfs") .setSettings( Settings.settingsBuilder() .put("uri", "file://./") .put("path", path) .put("conf", "additional-cfg.xml, conf-2.xml") .put("chunk_size", randomIntBetween(100, 1000) + "k") .put("compress", randomBoolean())) .get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); createIndex("test-idx-1", "test-idx-2", "test-idx-3"); ensureGreen(); logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i); index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client .admin() .cluster() .prepareCreateSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) .setIndices("test-idx-*", "-test-idx-3") .get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat( createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat( client .admin() .cluster() .prepareGetSnapshots("test-repo") .setSnapshots("test-snap") .get() .getSnapshots() .get(0) .state(), equalTo(SnapshotState.SUCCESS)); logger.info("--> delete some data"); for (int i = 0; i < 50; i++) { client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get(); } for (int i = 50; i < 100; i++) { client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get(); } for (int i = 0; i < 100; i += 2) { client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); } refresh(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); logger.info("--> close indices"); client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); logger.info("--> restore all indices from the snapshot"); RestoreSnapshotResponse restoreSnapshotResponse = client .admin() .cluster() .prepareRestoreSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) .execute() .actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); // Test restore after index deletion logger.info("--> delete indices"); wipeIndices("test-idx-1", "test-idx-2"); logger.info("--> restore one index after deletion"); restoreSnapshotResponse = client .admin() .cluster() .prepareRestoreSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) .setIndices("test-idx-*", "-test-idx-2") .execute() .actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); }
private void assertRepositoryIsOperational(Client client, String repository) { createIndex("test-idx-1"); ensureGreen(); logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); } refresh(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client .admin() .cluster() .prepareCreateSnapshot(repository, "test-snap") .setWaitForCompletion(true) .setIndices("test-idx-*") .get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat( createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat( client .admin() .cluster() .prepareGetSnapshots(repository) .setSnapshots("test-snap") .get() .getSnapshots() .get(0) .state(), equalTo(SnapshotState.SUCCESS)); logger.info("--> delete some data"); for (int i = 0; i < 50; i++) { client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get(); } refresh(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); logger.info("--> close indices"); client.admin().indices().prepareClose("test-idx-1").get(); logger.info("--> restore all indices from the snapshot"); RestoreSnapshotResponse restoreSnapshotResponse = client .admin() .cluster() .prepareRestoreSnapshot(repository, "test-snap") .setWaitForCompletion(true) .execute() .actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); }
@Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") public void testEncryption() { Client client = client(); logger.info( "--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); PutRepositoryResponse putRepositoryResponse = client .admin() .cluster() .preparePutRepository("test-repo") .setType("s3") .setSettings( Settings.settingsBuilder() .put("base_path", basePath) .put("chunk_size", randomIntBetween(1000, 10000)) .put("server_side_encryption", true)) .get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); createIndex("test-idx-1", "test-idx-2", "test-idx-3"); ensureGreen(); logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i); index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client .admin() .cluster() .prepareCreateSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) .setIndices("test-idx-*", "-test-idx-3") .get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat( createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat( client .admin() .cluster() .prepareGetSnapshots("test-repo") .setSnapshots("test-snap") .get() .getSnapshots() .get(0) .state(), equalTo(SnapshotState.SUCCESS)); Settings settings = internalCluster().getInstance(Settings.class); Settings bucket = settings.getByPrefix("repositories.s3."); AmazonS3 s3Client = internalCluster() .getInstance(AwsS3Service.class) .client( null, null, bucket.get("region", settings.get("repositories.s3.region")), bucket.get("access_key", settings.get("cloud.aws.access_key")), bucket.get("secret_key", settings.get("cloud.aws.secret_key"))); String bucketName = bucket.get("bucket"); logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath); List<S3ObjectSummary> summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries(); for (S3ObjectSummary summary : summaries) { assertThat( s3Client.getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256")); } logger.info("--> delete some data"); for (int i = 0; i < 50; i++) { client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get(); } for (int i = 50; i < 100; i++) { client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get(); } for (int i = 0; i < 100; i += 2) { client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); } refresh(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); logger.info("--> close indices"); client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); logger.info("--> restore all indices from the snapshot"); RestoreSnapshotResponse restoreSnapshotResponse = client .admin() .cluster() .prepareRestoreSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) .execute() .actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); // Test restore after index deletion logger.info("--> delete indices"); cluster().wipeIndices("test-idx-1", "test-idx-2"); logger.info("--> restore one index after deletion"); restoreSnapshotResponse = client .admin() .cluster() .prepareRestoreSnapshot("test-repo", "test-snap") .setWaitForCompletion(true) .setIndices("test-idx-*", "-test-idx-2") .execute() .actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); }
@Test public void snapshotRecoveryTest() throws Exception { logger.info("--> start node A"); String nodeA = cluster().startNode(settingsBuilder().put("gateway.type", "local")); logger.info("--> create repository"); assertAcked( client() .admin() .cluster() .preparePutRepository(REPO_NAME) .setType("fs") .setSettings( ImmutableSettings.settingsBuilder() .put("location", newTempDir(LifecycleScope.SUITE)) .put("compress", false)) .get()); ensureGreen(); logger.info("--> create index on node: {}", nodeA); createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client() .admin() .cluster() .prepareCreateSnapshot(REPO_NAME, SNAP_NAME) .setWaitForCompletion(true) .setIndices(INDEX_NAME) .get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat( createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat( client() .admin() .cluster() .prepareGetSnapshots(REPO_NAME) .setSnapshots(SNAP_NAME) .get() .getSnapshots() .get(0) .state(), equalTo(SnapshotState.SUCCESS)); client().admin().indices().prepareClose(INDEX_NAME).execute().actionGet(); logger.info("--> restore"); RestoreSnapshotResponse restoreSnapshotResponse = client() .admin() .cluster() .prepareRestoreSnapshot(REPO_NAME, SNAP_NAME) .setWaitForCompletion(true) .execute() .actionGet(); int totalShards = restoreSnapshotResponse.getRestoreInfo().totalShards(); assertThat(totalShards, greaterThan(0)); ensureGreen(); logger.info("--> request recoveries"); RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet(); for (Map.Entry<String, List<ShardRecoveryResponse>> shardRecoveryResponse : response.shardResponses().entrySet()) { assertThat(shardRecoveryResponse.getKey(), equalTo(INDEX_NAME)); List<ShardRecoveryResponse> shardRecoveryResponses = shardRecoveryResponse.getValue(); assertThat(shardRecoveryResponses.size(), equalTo(totalShards)); for (ShardRecoveryResponse shardResponse : shardRecoveryResponses) { assertThat(shardResponse.recoveryState().getType(), equalTo(RecoveryState.Type.SNAPSHOT)); assertThat(shardResponse.recoveryState().getStage(), equalTo(RecoveryState.Stage.DONE)); assertNotNull(shardResponse.recoveryState().getRestoreSource()); assertThat(shardResponse.recoveryState().getTargetNode().getName(), equalTo(nodeA)); } } }
/** For issue #26: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/26 */ @Test public void testListBlobs_26() throws StorageException, URISyntaxException { createIndex("test-idx-1", "test-idx-2", "test-idx-3"); ensureGreen(); logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i); index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); ClusterAdminClient client = client().admin().cluster(); logger.info("--> creating azure repository without any path"); PutRepositoryResponse putRepositoryResponse = client .preparePutRepository("test-repo") .setType("azure") .setSettings(Settings.settingsBuilder().put(Repository.CONTAINER, getContainerName())) .get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); // Get all snapshots - should be empty assertThat(client.prepareGetSnapshots("test-repo").get().getSnapshots().size(), equalTo(0)); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client .prepareCreateSnapshot("test-repo", "test-snap-26") .setWaitForCompletion(true) .setIndices("test-idx-*") .get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); // Get all snapshots - should have one assertThat(client.prepareGetSnapshots("test-repo").get().getSnapshots().size(), equalTo(1)); // Clean the snapshot client.prepareDeleteSnapshot("test-repo", "test-snap-26").get(); client.prepareDeleteRepository("test-repo").get(); logger.info("--> creating azure repository path [{}]", getRepositoryPath()); putRepositoryResponse = client .preparePutRepository("test-repo") .setType("azure") .setSettings( Settings.settingsBuilder() .put(Repository.CONTAINER, getContainerName()) .put(Repository.BASE_PATH, getRepositoryPath())) .get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); // Get all snapshots - should be empty assertThat(client.prepareGetSnapshots("test-repo").get().getSnapshots().size(), equalTo(0)); logger.info("--> snapshot"); createSnapshotResponse = client .prepareCreateSnapshot("test-repo", "test-snap-26") .setWaitForCompletion(true) .setIndices("test-idx-*") .get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); // Get all snapshots - should have one assertThat(client.prepareGetSnapshots("test-repo").get().getSnapshots().size(), equalTo(1)); }
@Test public void testMultipleRepositories() { Client client = client(); logger.info("--> creating azure repository with path [{}]", getRepositoryPath()); PutRepositoryResponse putRepositoryResponse1 = client .admin() .cluster() .preparePutRepository("test-repo1") .setType("azure") .setSettings( Settings.settingsBuilder() .put(Repository.CONTAINER, getContainerName().concat("-1")) .put(Repository.BASE_PATH, getRepositoryPath()) .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000))) .get(); assertThat(putRepositoryResponse1.isAcknowledged(), equalTo(true)); PutRepositoryResponse putRepositoryResponse2 = client .admin() .cluster() .preparePutRepository("test-repo2") .setType("azure") .setSettings( Settings.settingsBuilder() .put(Repository.CONTAINER, getContainerName().concat("-2")) .put(Repository.BASE_PATH, getRepositoryPath()) .put(Repository.CHUNK_SIZE, randomIntBetween(1000, 10000))) .get(); assertThat(putRepositoryResponse2.isAcknowledged(), equalTo(true)); createIndex("test-idx-1", "test-idx-2"); ensureGreen(); logger.info("--> indexing some data"); for (int i = 0; i < 100; i++) { index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i); } refresh(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); logger.info("--> snapshot 1"); CreateSnapshotResponse createSnapshotResponse1 = client .admin() .cluster() .prepareCreateSnapshot("test-repo1", "test-snap") .setWaitForCompletion(true) .setIndices("test-idx-1") .get(); assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat( createSnapshotResponse1.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse1.getSnapshotInfo().totalShards())); logger.info("--> snapshot 2"); CreateSnapshotResponse createSnapshotResponse2 = client .admin() .cluster() .prepareCreateSnapshot("test-repo2", "test-snap") .setWaitForCompletion(true) .setIndices("test-idx-2") .get(); assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat( createSnapshotResponse2.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards())); assertThat( client .admin() .cluster() .prepareGetSnapshots("test-repo1") .setSnapshots("test-snap") .get() .getSnapshots() .get(0) .state(), equalTo(SnapshotState.SUCCESS)); assertThat( client .admin() .cluster() .prepareGetSnapshots("test-repo2") .setSnapshots("test-snap") .get() .getSnapshots() .get(0) .state(), equalTo(SnapshotState.SUCCESS)); // Test restore after index deletion logger.info("--> delete indices"); cluster().wipeIndices("test-idx-1", "test-idx-2"); logger.info("--> restore one index after deletion from snapshot 1"); RestoreSnapshotResponse restoreSnapshotResponse1 = client .admin() .cluster() .prepareRestoreSnapshot("test-repo1", "test-snap") .setWaitForCompletion(true) .setIndices("test-idx-1") .execute() .actionGet(); assertThat(restoreSnapshotResponse1.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); logger.info("--> restore other index after deletion from snapshot 2"); RestoreSnapshotResponse restoreSnapshotResponse2 = client .admin() .cluster() .prepareRestoreSnapshot("test-repo2", "test-snap") .setWaitForCompletion(true) .setIndices("test-idx-2") .execute() .actionGet(); assertThat(restoreSnapshotResponse2.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(true)); }