public void testBringSiteOnline(Method m) {
    final String key = k(m);
    final String value = v(m);

    assertAllCachesEmpty();
    setSitesStatus(false);
    assertSiteStatusInAllCaches(XSiteAdminOperations.OFFLINE);

    extractGlobalComponent(site(0).cacheManagers().get(0), GlobalXSiteAdminOperations.class)
        .bringSiteOnline(siteName(1));

    assertSiteStatus(0, 0, null, 1, XSiteAdminOperations.ONLINE);
    assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name(), 1, XSiteAdminOperations.ONLINE);
    assertSiteStatus(
        0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name(), 2, XSiteAdminOperations.OFFLINE);
    assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_2.name(), 2, XSiteAdminOperations.OFFLINE);

    // double check with data
    putInAllCache(key, value);
    assertValueInAllCachesInPrimarySite(
        key, value); // all caches should have the value in primary site

    assertValueInCache(1, null, key, value);
    assertValueInCache(1, CacheType.BACKUP_TO_SITE_1_AND_2.name(), key, value);
    assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name());
    assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_2.name());
  }
 private void assertAllCachesEmpty() {
   for (CacheType cacheType : CacheType.values()) {
     assertCacheEmpty(0, 0, cacheType.name());
   }
   assertCacheEmpty(1, 0, null);
   assertCacheEmpty(1, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name());
   assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name());
   assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_2.name());
 }
  public void testPushState(Method m) {
    final String key = k(m);
    final String value = v(m);

    assertAllCachesEmpty();
    setSitesStatus(false);
    assertSiteStatusInAllCaches(XSiteAdminOperations.OFFLINE);

    putInAllCache(key, value);
    assertValueInAllCachesInPrimarySite(key, value);

    // check the value is not in the backups
    assertCacheEmpty(1, 0, null);
    assertCacheEmpty(1, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name());
    assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name());
    assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_2.name());

    extractGlobalComponent(site(0).cacheManagers().get(0), GlobalXSiteAdminOperations.class)
        .pushState(siteName(1));
    awaitXSiteStateTransfer();

    // check state and data
    assertSiteStatus(0, 0, null, 1, XSiteAdminOperations.ONLINE);
    assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name(), 1, XSiteAdminOperations.ONLINE);
    assertSiteStatus(
        0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name(), 2, XSiteAdminOperations.OFFLINE);
    assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_2.name(), 2, XSiteAdminOperations.OFFLINE);

    assertValueInCache(1, null, key, value);
    assertValueInCache(1, CacheType.BACKUP_TO_SITE_1_AND_2.name(), key, value);
    assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name());
    assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_2.name());

    extractGlobalComponent(site(0).cacheManagers().get(0), GlobalXSiteAdminOperations.class)
        .pushState(siteName(2));
    awaitXSiteStateTransfer();

    // check state and data
    assertSiteStatus(0, 0, null, 1, XSiteAdminOperations.ONLINE);
    assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name(), 1, XSiteAdminOperations.ONLINE);
    assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name(), 2, XSiteAdminOperations.ONLINE);
    assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_2.name(), 2, XSiteAdminOperations.ONLINE);

    assertValueInCache(1, null, key, value);
    assertValueInCache(1, CacheType.BACKUP_TO_SITE_1_AND_2.name(), key, value);
    assertValueInCache(2, CacheType.BACKUP_TO_SITE_1_AND_2.name(), key, value);
    assertValueInCache(2, CacheType.BACKUP_TO_SITE_2.name(), key, value);
  }
 private void setSitesStatus(boolean online) {
   if (online) {
     xSiteAdminOperations(0, 0, null).bringSiteOnline(siteName(1));
     xSiteAdminOperations(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name())
         .bringSiteOnline(siteName(1));
     xSiteAdminOperations(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name())
         .bringSiteOnline(siteName(2));
     xSiteAdminOperations(0, 0, CacheType.BACKUP_TO_SITE_2.name()).bringSiteOnline(siteName(2));
   } else {
     xSiteAdminOperations(0, 0, null).takeSiteOffline(siteName(1));
     xSiteAdminOperations(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name())
         .takeSiteOffline(siteName(1));
     xSiteAdminOperations(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name())
         .takeSiteOffline(siteName(2));
     xSiteAdminOperations(0, 0, CacheType.BACKUP_TO_SITE_2.name()).takeSiteOffline(siteName(2));
   }
 }
  @Override
  protected void afterSitesCreated() {
    super.afterSitesCreated();

    ConfigurationBuilder builder = newConfiguration();
    builder.sites().addBackup().site(siteName(2)).strategy(BackupConfiguration.BackupStrategy.SYNC);
    defineInSite(site(0), CacheType.BACKUP_TO_SITE_2.name(), builder.build());
    defineInSite(site(2), CacheType.BACKUP_TO_SITE_2.name(), newConfiguration().build());

    builder = newConfiguration();
    builder.sites().addBackup().site(siteName(1)).strategy(BackupConfiguration.BackupStrategy.SYNC);
    builder.sites().addBackup().site(siteName(2)).strategy(BackupConfiguration.BackupStrategy.SYNC);
    defineInSite(site(0), CacheType.BACKUP_TO_SITE_1_AND_2.name(), builder.build());
    defineInSite(site(1), CacheType.BACKUP_TO_SITE_1_AND_2.name(), newConfiguration().build());
    defineInSite(site(2), CacheType.BACKUP_TO_SITE_1_AND_2.name(), newConfiguration().build());

    defineInSite(site(0), CacheType.NO_BACKUP.name(), newConfiguration().build());

    // wait for caches in primary cluster
    site(0).waitForClusterToForm(null);
    site(0).waitForClusterToForm(CacheType.BACKUP_TO_SITE_1_AND_2.name());
    site(0).waitForClusterToForm(CacheType.BACKUP_TO_SITE_1_AND_2.name());
    site(0).waitForClusterToForm(CacheType.BACKUP_TO_SITE_2.name());

    // wait for caches in backup site 1
    site(1).waitForClusterToForm(null);
    site(1).waitForClusterToForm(CacheType.BACKUP_TO_SITE_1_AND_2.name());

    // wait for caches in backup site 2
    site(2).waitForClusterToForm(CacheType.BACKUP_TO_SITE_1_AND_2.name());
    site(2).waitForClusterToForm(CacheType.BACKUP_TO_SITE_2.name());
  }
  public void testCancelPushState(Method m) {
    final String key = k(m);
    final String value = v(m);

    assertAllCachesEmpty();
    setSitesStatus(false);
    assertSiteStatusInAllCaches(XSiteAdminOperations.OFFLINE);

    putInAllCache(key, value);
    assertValueInAllCachesInPrimarySite(key, value);

    // check the value is not in the backups
    assertCacheEmpty(1, 0, null);
    assertCacheEmpty(1, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name());
    assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name());
    assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_2.name());

    List<BlockingTransport> blockingTransportList = getBlockingTransport(0, true);
    blockingTransportList.forEach(BlockingTransport::blockCommands);

    extractGlobalComponent(site(0).cacheManagers().get(0), GlobalXSiteAdminOperations.class)
        .pushState(siteName(1));

    extractGlobalComponent(site(0).cacheManagers().get(0), GlobalXSiteAdminOperations.class)
        .cancelPushState(siteName(1));

    // check state and data
    assertSiteStatus(0, 0, null, 1, XSiteAdminOperations.ONLINE);
    assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name(), 1, XSiteAdminOperations.ONLINE);
    assertSiteStatus(
        0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name(), 2, XSiteAdminOperations.OFFLINE);
    assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_2.name(), 2, XSiteAdminOperations.OFFLINE);

    assertCacheEmpty(1, 0, null);
    assertCacheEmpty(1, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name());
    assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name());
    assertCacheEmpty(2, 0, CacheType.BACKUP_TO_SITE_2.name());

    blockingTransportList.forEach(BlockingTransport::unblockCommands);
  }
 private void assertSiteStatusInAllCaches(String status) {
   assertSiteStatus(0, 0, null, 1, status);
   assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name(), 1, status);
   assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name(), 2, status);
   assertSiteStatus(0, 0, CacheType.BACKUP_TO_SITE_2.name(), 2, status);
 }
 private void assertValueInAllCachesInPrimarySite(String key, String value) {
   assertValueInCache(0, null, key, value);
   assertValueInCache(0, CacheType.BACKUP_TO_SITE_1_AND_2.name(), key, value);
   assertValueInCache(0, CacheType.BACKUP_TO_SITE_2.name(), key, value);
   assertValueInCache(0, CacheType.NO_BACKUP.name(), key, value);
 }
 private void putInAllCache(String key, String value) {
   cache(0, 0, null).put(key, value);
   cache(0, 0, CacheType.BACKUP_TO_SITE_1_AND_2.name()).put(key, value);
   cache(0, 0, CacheType.BACKUP_TO_SITE_2.name()).put(key, value);
   cache(0, 0, CacheType.NO_BACKUP.name()).put(key, value);
 }
 private void awaitXSiteStateTransfer() {
   awaitXSiteStateTransferFor(null);
   awaitXSiteStateTransferFor(CacheType.BACKUP_TO_SITE_1_AND_2.name());
   awaitXSiteStateTransferFor(CacheType.BACKUP_TO_SITE_2.name());
 }