Пример #1
0
  public static void main(String[] args) throws Exception {
    VoltDB.setDefaultTimezone();

    config = new SchemaChangeConfig();
    config.parse("SchemaChangeClient", args);

    ClientConfig clientConfig = new ClientConfig();
    clientConfig.setProcedureCallTimeout(30 * 60 * 1000); // 30 min
    client = ClientFactory.createClient(clientConfig);
    String[] servers = config.servers.split(",");
    for (String server : servers) {
      server = server.trim();
      client.createConnection(server);
    }

    // get the topo
    topo = getCluterTopology(client);

    // kick this off with a random schema
    VoltTable t = catalogChange(null, true);

    for (int i = 0; i < 50; i++) {
      // make sure the table is full and mess around with it
      loadTable(t);

      for (int j = 0; j < 50; j++) {
        String tableName = TableHelper.getTableName(t);

        // deterministically sample some rows
        VoltTable preT = sample(t);
        // System.out.printf("First sample:\n%s\n", preT.toFormattedString());

        // move to an entirely new table or migrated schema
        t = catalogChange(t, (j == 0) && (rand.nextInt(5) == 0));

        // if the table has been migrated, check the data
        if (TableHelper.getTableName(t).equals(tableName)) {
          VoltTable guessT = t.clone(4096 * 1024);
          // System.out.printf("Empty clone:\n%s\n", guessT.toFormattedString());

          TableHelper.migrateTable(preT, guessT);
          // System.out.printf("Java migration:\n%s\n", guessT.toFormattedString());

          // deterministically sample the same rows
          VoltTable postT = sample(t);
          // System.out.printf("Second sample:\n%s\n", postT.toFormattedString());

          postT.resetRowPosition();
          preT.resetRowPosition();
          StringBuilder sb = new StringBuilder();
          if (!TableHelper.deepEqualsWithErrorMsg(postT, guessT, sb)) {
            System.err.println(sb.toString());
            assert (false);
          }
        }
      }
    }

    client.close();
  }
Пример #2
0
  private boolean checkConstraints(String procName, ClientResponse response) {
    boolean isSatisfied = true;
    int orig_position = -1;

    // Check if all the tables in the result set satisfy the constraints.
    for (int i = 0; isSatisfied && i < response.getResults().length; i++) {
      Pair<String, Integer> key = Pair.of(procName, i);
      if (!m_constraints.containsKey(key)) continue;

      VoltTable table = response.getResults()[i];
      orig_position = table.getActiveRowIndex();
      table.resetRowPosition();

      // Iterate through all rows and check if they satisfy the
      // constraints.
      while (isSatisfied && table.advanceRow()) {
        isSatisfied = Verification.checkRow(m_constraints.get(key), table);
      }

      // Have to reset the position to its original position.
      if (orig_position < 0) table.resetRowPosition();
      else table.advanceToRow(orig_position);
    }

    if (!isSatisfied) System.err.println("Transaction " + procName + " failed check");

    return isSatisfied;
  }
Пример #3
0
  private VoltTable evictData(Table catalog_tbl) throws Exception {
    VoltTable results[] = this.ee.getStats(SysProcSelector.TABLE, this.locators, false, 0L);
    assertEquals(1, results.length);
    // System.err.println(VoltTableUtil.format(results));
    while (results[0].advanceRow()) {
      if (results[0].getString("TABLE_NAME").equalsIgnoreCase(catalog_tbl.getName()) == false)
        continue;
      for (String col : statsFields) {
        int idx = results[0].getColumnIndex(col);
        assertEquals(0, results[0].getLong(idx));
      } // FOR
    } // WHILE

    // Now force the EE to evict our boys out
    // We'll tell it to remove 1MB, which is guaranteed to include all of our tuples
    VoltTable evictResult = this.ee.antiCacheEvictBlock(catalog_tbl, 1024 * 1024, 1);

    //        System.err.println("-------------------------------");
    //        System.err.println(VoltTableUtil.format(evictResult));
    assertNotNull(evictResult);
    assertEquals(1, evictResult.getRowCount());
    assertNotSame(results[0].getColumnCount(), evictResult.getColumnCount());
    evictResult.resetRowPosition();
    boolean adv = evictResult.advanceRow();
    assertTrue(adv);
    return (evictResult);
  }
Пример #4
0
 protected void validateRowSeenAtAllPartitions(
     VoltTable result, String columnName, String targetValue, boolean enforceUnique) {
   result.resetRowPosition();
   Set<Integer> partsSeen = new HashSet<>();
   while (result.advanceRow()) {
     String colValFromRow = result.getString(columnName);
     if (targetValue.equalsIgnoreCase(colValFromRow)) {
       int thisPartId = (int) result.getLong("PARTITION_ID");
       if (enforceUnique) {
         assertFalse(
             "PARTITION_ID: "
                 + thisPartId
                 + " seen twice in table looking for "
                 + targetValue
                 + " in column "
                 + columnName,
             partsSeen.contains(thisPartId));
       }
       partsSeen.add(thisPartId);
     }
   }
   // Remove the MPI in case it's in there
   partsSeen.remove(MpInitiator.MP_INIT_PID);
   assertEquals(PARTITIONS, partsSeen.size());
 }
Пример #5
0
 protected boolean validateRowSeenAtAllSites(
     VoltTable result, String columnName, String targetValue, boolean enforceUnique) {
   result.resetRowPosition();
   Set<Long> sitesSeen = new HashSet<Long>();
   while (result.advanceRow()) {
     String colValFromRow = result.getString(columnName);
     if (targetValue.equalsIgnoreCase(colValFromRow)) {
       long hostId = result.getLong("HOST_ID");
       long thisSiteId = result.getLong("SITE_ID");
       thisSiteId |= hostId << 32;
       if (enforceUnique) {
         assertFalse(
             "SITE_ID: "
                 + thisSiteId
                 + " seen twice in table looking for "
                 + targetValue
                 + " in column "
                 + columnName,
             sitesSeen.contains(thisSiteId));
       }
       sitesSeen.add(thisSiteId);
     }
   }
   return (HOSTS * SITES) == sitesSeen.size();
 }
Пример #6
0
  /**
   * Check the result of a query that has only an OFFSET and no LIMIT clause. This is done by
   * executing the query with and without the offset clause, and then skipping past the offset rows
   * in the expected table here on the client side.
   */
  private static void doOffsetAndCheck(Client client, String stmt)
      throws IOException, InterruptedException, ProcCallException {
    String stmtNoOffset = stmt.substring(0, stmt.indexOf("OFFSET"));
    VoltTable expectedTable = client.callProcedure("@AdHoc", stmtNoOffset).getResults()[0];
    int rowCountBeforeOffset = expectedTable.getRowCount();

    int[] offsets = {0, 1, 5, 10, 11, 15};
    for (int offset : offsets) {
      VoltTable actualTable = client.callProcedure("@AdHoc", stmt, offset).getResults()[0];
      int expectedRowCount = Math.max(rowCountBeforeOffset - offset, 0);
      assertEquals(
          "Actual table has wrong number of rows: ", expectedRowCount, actualTable.getRowCount());
      if (actualTable.getRowCount() == 0) continue;

      // non-empty result.
      // Advance expected table past offset
      // then compare what's left.
      actualTable.resetRowPosition();
      for (int i = 0; i < offset; ++i) expectedTable.advanceRow();

      while (actualTable.advanceRow() && expectedTable.advanceRow()) {
        assertEquals(expectedTable.getLong(0), actualTable.getLong(0));
        assertEquals(expectedTable.getLong(1), actualTable.getLong(1));
      }
    }
  }
Пример #7
0
  private int countHostsProvidingRows(
      VoltTable result, Map<String, String> columnTargets, boolean enforceUnique) {
    result.resetRowPosition();
    Set<Long> hostsSeen = new HashSet<Long>();
    while (result.advanceRow()) {
      if (checkRowForMultipleTargets(result, columnTargets)) {
        Long thisHostId = result.getLong("HOST_ID");
        if (enforceUnique) {
          StringBuilder message = new StringBuilder();
          message.append("HOST_ID: " + thisHostId + " seen twice in table looking for ");
          for (Entry<String, String> entry : columnTargets.entrySet()) {
            message.append(entry.getValue() + " in column " + entry.getKey() + ";");
          }
          assertFalse(message.toString(), hostsSeen.contains(thisHostId));
        }
        hostsSeen.add(thisHostId);
      }
    }

    // * Enable this to force a failure with diagnostics */ hostsSeen.add(123456789L);

    // Before possibly failing an assert, prepare to report details of the non-conforming result.
    m_recentAnalysis = null;
    if (HOSTS != hostsSeen.size()) {
      m_recentAnalysis = new StringBuilder();
      m_recentAnalysis.append("Failure follows from these results:\n");
      Set<Long> seenAgain = new HashSet<Long>();
      result.resetRowPosition();
      while (result.advanceRow()) {
        Long thisHostId = result.getLong("HOST_ID");
        String rowStatus = "Found a non-match";
        if (checkRowForMultipleTargets(result, columnTargets)) {
          if (seenAgain.add(thisHostId)) {
            rowStatus = "Added a match";
          } else {
            rowStatus = "Duplicated a match";
          }
        }
        m_recentAnalysis.append(rowStatus + " at host " + thisHostId + " for ");
        for (String key : columnTargets.keySet()) {
          m_recentAnalysis.append(key + " " + result.getString(key) + ";");
        }
        m_recentAnalysis.append("\n");
      }
    }
    return hostsSeen.size();
  }
Пример #8
0
  /** Grab some random rows that aren't on the first EE page for the table. */
  public static VoltTable sample(VoltTable t) throws Exception {
    VoltTable t2 = t.clone(4096 * 1024);

    ClientResponse cr =
        client.callProcedure(
            "@AdHoc",
            String.format(
                "select * from %s where pkey >= 100000 order by pkey limit 100;",
                TableHelper.getTableName(t)));
    assert (cr.getStatus() == ClientResponse.SUCCESS);
    VoltTable result = cr.getResults()[0];
    result.resetRowPosition();
    while (result.advanceRow()) {
      t2.add(result);
    }

    return t2;
  }
Пример #9
0
  static Topology getCluterTopology(Client client) throws Exception {
    int hosts = -1;
    int sitesPerHost = -1;
    int k = -1;

    VoltTable result = client.callProcedure("@SystemInformation", "DEPLOYMENT").getResults()[0];
    result.resetRowPosition();
    while (result.advanceRow()) {
      String key = result.getString(0);
      String value = result.getString(1);
      if (key.equals("hostcount")) {
        hosts = Integer.parseInt(value);
      }
      if (key.equals("sitesperhost")) {
        sitesPerHost = Integer.parseInt(value);
      }
      if (key.equals("kfactor")) {
        k = Integer.parseInt(value);
      }
    }

    return new Topology(hosts, hosts * sitesPerHost, (hosts * sitesPerHost) / (k + 1));
  }
Пример #10
0
  /**
   * Cache the current statistics.
   *
   * @param time
   */
  private void statsTick(long time) {
    /*
     * grab the table statistics from ee and put it into the statistics
     * agent.
     */
    if (m_tableStats != null) {
      CatalogMap<Table> tables = m_context.database.getTables();
      int[] tableIds = new int[tables.size()];
      int i = 0;
      for (Table table : tables) {
        tableIds[i++] = table.getRelativeIndex();
      }

      // data to aggregate
      long tupleCount = 0;
      int tupleDataMem = 0;
      int tupleAllocatedMem = 0;
      int indexMem = 0;
      int stringMem = 0;

      // update table stats
      final VoltTable[] s1 = m_ee.getStats(SysProcSelector.TABLE, tableIds, false, time);
      if (s1 != null) {
        VoltTable stats = s1[0];
        assert (stats != null);

        // rollup the table memory stats for this site
        while (stats.advanceRow()) {
          tupleCount += stats.getLong(7);
          tupleAllocatedMem += (int) stats.getLong(8);
          tupleDataMem += (int) stats.getLong(9);
          stringMem += (int) stats.getLong(10);
        }
        stats.resetRowPosition();

        m_tableStats.setStatsTable(stats);
      }

      // update index stats
      final VoltTable[] s2 = m_ee.getStats(SysProcSelector.INDEX, tableIds, false, time);
      if ((s2 != null) && (s2.length > 0)) {
        VoltTable stats = s2[0];
        assert (stats != null);

        // rollup the index memory stats for this site
        while (stats.advanceRow()) {
          indexMem += stats.getLong(10);
        }
        stats.resetRowPosition();

        m_indexStats.setStatsTable(stats);
      }

      // update the rolled up memory statistics
      if (m_memStats != null) {
        m_memStats.eeUpdateMemStats(
            m_siteId,
            tupleCount,
            tupleDataMem,
            tupleAllocatedMem,
            indexMem,
            stringMem,
            m_ee.getThreadLocalPoolAllocations());
      }
    }
  }