/** Tests scenarios B-1 and B-2 in the [#16655] SR. */
  @Test
  public void testUpgrade() throws Exception {

    open();

    /* Master writes and reads v0 entities. */
    masterApp.writeData(0);
    masterApp.writeDataA(1);
    masterApp.writeDataB(2);
    masterApp.readData(0);
    masterApp.readDataA(1);
    masterApp.readDataB(2);

    /* Replicas read v0 entities. */
    replicaApp1.readData(0);
    replicaApp1.readDataA(1);
    replicaApp1.readDataB(2);
    replicaApp2.readData(0);
    replicaApp2.readDataA(1);
    replicaApp2.readDataB(2);

    /* Replica1 is upgraded to v1, upgrades metadata in memory. */
    bounceReplica1(1);

    /* Upgraded replica1 reads v0 entities, can't get new index. */
    try {
      replicaApp1.readData(0);
      fail();
    } catch (IndexNotAvailableException e) {
    }
    try {
      replicaApp1.readDataB(2);
      fail();
    } catch (IndexNotAvailableException e) {
    }

    /* Upgraded replica1 can't get index for new entity NewData2. */
    try {
      replicaApp1.readData2(14);
      fail();
    } catch (IndexNotAvailableException e) {
    }

    /* Replica1 can read v0 DataA, because it has no new indexes. */
    replicaApp1.readDataA(1);

    /* Replica2 (not yet upgraded) reads v0 entities without errors. */
    replicaApp2.readData(0);
    replicaApp2.readDataA(1);
    replicaApp2.readDataB(2);

    /* Replica2 is upgraded to v1, upgrades metadata in memory. */
    bounceReplica2(1);

    /* Upgraded replicas read v0 entities, can't get new index. */
    try {
      replicaApp1.readData(0);
      fail();
    } catch (IndexNotAvailableException e) {
    }
    try {
      replicaApp1.readDataB(2);
      fail();
    } catch (IndexNotAvailableException e) {
    }
    try {
      replicaApp2.readData(0);
      fail();
    } catch (IndexNotAvailableException e) {
    }
    try {
      replicaApp2.readDataB(2);
      fail();
    } catch (IndexNotAvailableException e) {
    }

    /* Upgraded replicas can't get index for new entity NewData2. */
    try {
      replicaApp1.readData2(14);
      fail();
    } catch (IndexNotAvailableException e) {
    }
    try {
      replicaApp2.readData2(14);
      fail();
    } catch (IndexNotAvailableException e) {
    }

    /* Upgraded replicas can read v0 DataA, it has no new indexes. */
    replicaApp1.readDataA(1);
    replicaApp2.readDataA(1);

    /* Read again on master for good measure. */
    masterApp.readData(0);
    masterApp.readDataA(1);
    masterApp.readDataB(2);

    /* Master is upgraded to v1, replica1 switches roles with master. */
    bounceMaster(1);

    /* Metadata is refreshed when new indexes are requested. */
    try {
      masterApp.readData(0);
      fail();
    } catch (DatabasePreemptedException expected) {
      masterApp.close();
      masterApp.open(masterEnv);
      masterApp.readData(0);
    }
    masterApp.readDataA(1);
    masterApp.readDataB(2);
    try {
      replicaApp1.readData(0);
      fail();
    } catch (DatabasePreemptedException expected) {
      replicaApp1.close();
      replicaApp1.open(replicaEnv1);
      replicaApp1.readData(0);
    }
    replicaApp1.readDataA(1);
    replicaApp1.readDataB(2);
    try {
      replicaApp2.readData(0);
      fail();
    } catch (DatabasePreemptedException expected) {
      replicaApp2.close();
      replicaApp2.open(replicaEnv2);
      replicaApp2.readData(0);
    }
    replicaApp2.readDataA(1);
    replicaApp2.readDataB(2);

    /* Master writes v1 entities. */
    masterApp.writeData(10);
    masterApp.writeDataA(11);
    masterApp.writeDataB(12);

    /* Master reads v0 and v1 entities. */
    masterApp.readData(0);
    masterApp.readData(10);
    masterApp.readDataA(1);
    masterApp.readDataA(11);
    masterApp.readDataB(2);
    masterApp.readDataB(12);

    /* Replicas read v0 and v1 entities. */
    replicaApp1.readData(0);
    replicaApp1.readData(10);
    replicaApp1.readDataA(1);
    replicaApp1.readDataA(11);
    replicaApp1.readDataB(2);
    replicaApp1.readDataB(12);
    replicaApp2.readData(0);
    replicaApp2.readData(10);
    replicaApp2.readDataA(1);
    replicaApp2.readDataA(11);
    replicaApp2.readDataB(2);
    replicaApp2.readDataB(12);

    /* Master writes new NewDataC subclass, all can read. */
    masterApp.writeDataC(13);
    masterApp.readDataC(13);
    replicaApp1.readDataC(13);
    replicaApp2.readDataC(13);

    /* Master writes new NewData2 entity class, all can read. */
    masterApp.writeData2(14);
    masterApp.readData2(14);
    replicaApp1.readData2(14);
    replicaApp2.readData2(14);

    close();
  }