/** Save the profile information into the database */
  protected final void saveProfile(BenchmarkComponent baseClient) {

    // CONFIG_PROFILE
    Table catalog_tbl =
        catalogContext.database.getTables().get(SEATSConstants.TABLENAME_CONFIG_PROFILE);
    VoltTable vt = CatalogUtil.getVoltTable(catalog_tbl);
    assert (vt != null);
    vt.addRow(
        this.scale_factor, // CFP_SCALE_FACTOR
        this.airport_max_customer_id.toJSONString(), // CFP_AIPORT_MAX_CUSTOMER
        this.flight_start_date, // CFP_FLIGHT_START
        this.flight_upcoming_date, // CFP_FLIGHT_UPCOMING
        this.flight_past_days, // CFP_FLIGHT_PAST_DAYS
        this.flight_future_days, // CFP_FLIGHT_FUTURE_DAYS
        this.num_flights, // CFP_NUM_FLIGHTS
        this.num_customers, // CFP_NUM_CUSTOMERS
        this.num_reservations, // CFP_NUM_RESERVATIONS
        JSONUtil.toJSONString(this.code_id_xref) // CFP_CODE_ID_XREF
        );
    if (debug.val)
      LOG.debug(String.format("Saving profile information into %s\n%s", catalog_tbl, this));
    baseClient.loadVoltTable(catalog_tbl.getName(), vt);

    // CONFIG_HISTOGRAMS
    catalog_tbl =
        catalogContext.database.getTables().get(SEATSConstants.TABLENAME_CONFIG_HISTOGRAMS);
    vt = CatalogUtil.getVoltTable(catalog_tbl);
    assert (vt != null);

    for (Entry<String, Histogram<String>> e : this.airport_histograms.entrySet()) {
      vt.addRow(
          e.getKey(), // CFH_NAME
          e.getValue().toJSONString(), // CFH_DATA
          1 // CFH_IS_AIRPORT
          );
    } // FOR
    if (debug.val) LOG.debug("Saving airport histogram information into " + catalog_tbl);
    baseClient.loadVoltTable(catalog_tbl.getName(), vt);

    for (Entry<String, Histogram<String>> e : this.histograms.entrySet()) {
      vt.addRow(
          e.getKey(), // CFH_NAME
          e.getValue().toJSONString(), // CFH_DATA
          0 // CFH_IS_AIRPORT
          );
    } // FOR
    if (debug.val) LOG.debug("Saving benchmark histogram information into " + catalog_tbl);
    baseClient.loadVoltTable(catalog_tbl.getName(), vt);

    return;
  }
  /** testEvictTuples */
  @Test
  public void testEvictTuples() throws Exception {
    // First load some stuff into the database
    for (String tableName : TARGET_TABLES) {
      this.loadData(this.getTable(tableName));
    }

    // Then make sure that we can evict from each of them
    for (String tableName : TARGET_TABLES) {
      Table catalog_tbl = this.getTable(tableName);
      VoltTable evictResult = this.evictData(catalog_tbl);
      evictResult.advanceRow();

      // Our stats should now come back with at least one block evicted
      VoltTable results[] = this.ee.getStats(SysProcSelector.TABLE, this.locators, false, 0L);
      assertEquals(1, results.length);
      // System.err.println("-------------------------------");
      // System.err.println(VoltTableUtil.format(results));

      while (results[0].advanceRow()) {
        if (results[0].getString("TABLE_NAME").equalsIgnoreCase(catalog_tbl.getName()) == false)
          continue;
        for (String col : statsFields) {
          assertEquals(col, evictResult.getLong(col), results[0].getLong(col));
          if (col == "ANTICACHE_BLOCKS_EVICTED") {
            assertEquals(col, 1, results[0].getLong(col));
          } else {
            assertNotSame(col, 0, results[0].getLong(col));
          }
        } // FOR
      } // WHILE
    } // FOR
  }
  private VoltTable evictData(Table catalog_tbl) throws Exception {
    VoltTable results[] = this.ee.getStats(SysProcSelector.TABLE, this.locators, false, 0L);
    assertEquals(1, results.length);
    // System.err.println(VoltTableUtil.format(results));
    while (results[0].advanceRow()) {
      if (results[0].getString("TABLE_NAME").equalsIgnoreCase(catalog_tbl.getName()) == false)
        continue;
      for (String col : statsFields) {
        int idx = results[0].getColumnIndex(col);
        assertEquals(0, results[0].getLong(idx));
      } // FOR
    } // WHILE

    // Now force the EE to evict our boys out
    // We'll tell it to remove 1MB, which is guaranteed to include all of our tuples
    VoltTable evictResult = this.ee.antiCacheEvictBlock(catalog_tbl, 1024 * 1024, 1);

    //        System.err.println("-------------------------------");
    //        System.err.println(VoltTableUtil.format(evictResult));
    assertNotNull(evictResult);
    assertEquals(1, evictResult.getRowCount());
    assertNotSame(results[0].getColumnCount(), evictResult.getColumnCount());
    evictResult.resetRowPosition();
    boolean adv = evictResult.advanceRow();
    assertTrue(adv);
    return (evictResult);
  }
Beispiel #4
0
 protected Column getColumn(Database catalog_db, Table catalog_tbl, String col_name) {
   assertNotNull(catalog_db);
   assertNotNull(catalog_tbl);
   Column catalog_col = catalog_tbl.getColumns().getIgnoreCase(col_name);
   assert (catalog_col != null)
       : "Failed to retrieve Column '" + col_name + "' from Table '" + catalog_tbl.getName() + "'";
   return (catalog_col);
 }
Beispiel #5
0
  /**
   * Return the name of the vertical partition for the given table name
   *
   * @param tableName
   * @return
   */
  private static String getNextVerticalPartitionName(
      Table catalog_tbl, Collection<Column> catalog_cols) {
    Database catalog_db = ((Database) catalog_tbl.getParent());

    Collection<String> colNames = new HashSet<String>();
    for (Column catalog_col : catalog_cols) {
      colNames.add(catalog_col.getName());
    }

    // Figure out how many vertical partition tables already exist for this table
    int next = 0;
    String prefix = "SYS_VP_" + catalog_tbl.getName() + "_";
    Pattern p = Pattern.compile(Pattern.quote(prefix) + "[\\d]+");
    for (Table otherTable : CatalogUtil.getSysTables(catalog_db)) {
      if (debug.get())
        LOG.debug(String.format("Checking whether '%s' matches prefix '%s'", otherTable, prefix));
      Matcher m = p.matcher(otherTable.getName());
      if (m.matches() == false) continue;

      // Check to make sure it's not the same vertical partition
      Collection<Column> otherColumns = otherTable.getColumns();
      if (debug.get())
        LOG.debug(
            String.format(
                "%s.%s <-> %s.%s",
                catalog_tbl.getName(), catalog_cols, otherTable.getName(), otherColumns));
      if (otherColumns.size() != colNames.size()) continue;
      boolean fail = false;
      for (Column otherCol : otherColumns) {
        if (colNames.contains(otherCol.getName()) == false) {
          fail = true;
          break;
        }
      }
      if (fail) continue;

      next++;
    } // FOR
    String viewName = String.format("%s%02d", prefix, next);
    assert (catalog_tbl.getViews().contains(viewName) == false);

    if (debug.get())
      LOG.debug(String.format("Next VerticalPartition name '%s' for %s", viewName, catalog_tbl));
    return (viewName);
  }
 /**
  * For a given table catalog object, return an Iterable that will iterate through tuples in the
  * generated flat files.
  *
  * @param catalog_tbl
  * @return
  * @throws Exception
  */
 public Iterable<Object[]> getTable(final Table catalog_tbl) throws Exception {
   File file =
       new File(
           EGenLoader.this.output_path
               + File.separator
               + catalog_tbl.getName().toUpperCase()
               + ".txt");
   return (new TableDataIterable(catalog_tbl, file));
 }
Beispiel #7
0
  public void addTablePartitionInfo(Table catalog_tbl, Column catalog_col) {
    assert (catalog_col != null) : "Unexpected null partition column for " + catalog_tbl;

    // TODO: Support special columns
    if (catalog_col instanceof VerticalPartitionColumn) {
      catalog_col = ((VerticalPartitionColumn) catalog_col).getHorizontalColumn();
    }
    if (catalog_col instanceof MultiColumn) {
      catalog_col = ((MultiColumn) catalog_col).get(0);
    }
    this.addTablePartitionInfo(catalog_tbl.getName(), catalog_col.getName());
  }
Beispiel #8
0
 protected Column getColumn(Table catalog_tbl, int col_idx) {
   int num_columns = catalog_tbl.getColumns().size();
   if (col_idx < 0) col_idx = num_columns + col_idx; // Python!
   assert (col_idx >= 0 && col_idx < num_columns)
       : "Invalid column index for " + catalog_tbl + ": " + col_idx;
   Column catalog_col = catalog_tbl.getColumns().get(col_idx);
   assert (catalog_col != null)
       : "Failed to retrieve Column at '"
           + col_idx
           + "' from Table '"
           + catalog_tbl.getName()
           + "'";
   return (catalog_col);
 }
  private void loadData(Table catalog_tbl) throws Exception {
    // Load in a bunch of dummy data for this table
    VoltTable vt = CatalogUtil.getVoltTable(catalog_tbl);
    assertNotNull(vt);
    for (int i = 0; i < NUM_TUPLES; i++) {
      Object row[] = VoltTableUtil.getRandomRow(catalog_tbl);

      if (catalog_tbl.getName().equalsIgnoreCase(TPCCConstants.TABLENAME_ORDER_LINE)) {
        row[0] = i; // OL_O_ID
        row[1] = (byte) i; // OL_D_ID
        row[2] = (short) i; // OL_W_ID
      }
      vt.addRow(row);
    } // FOR
    this.executor.loadTable(1000l, catalog_tbl, vt, false);

    int statsLocators[] = {catalog_tbl.getRelativeIndex()};
    VoltTable stats[] = this.ee.getStats(SysProcSelector.TABLE, statsLocators, false, 0L);
    assertEquals(1, stats.length);
    // System.err.println(VoltTableUtil.format(stats));
  }
Beispiel #10
0
  public static MaterializedViewInfo addVerticalPartition(
      final Table catalog_tbl, final Collection<Column> catalog_cols, final boolean createIndex)
      throws Exception {
    assert (catalog_cols.isEmpty() == false);
    Database catalog_db = ((Database) catalog_tbl.getParent());

    String viewName = getNextVerticalPartitionName(catalog_tbl, catalog_cols);
    if (debug.get())
      LOG.debug(
          String.format(
              "Adding Vertical Partition %s for %s: %s", viewName, catalog_tbl, catalog_cols));

    // Create a new virtual table
    Table virtual_tbl = catalog_db.getTables().get(viewName);
    if (virtual_tbl == null) {
      virtual_tbl = catalog_db.getTables().add(viewName);
    }
    virtual_tbl.setIsreplicated(true);
    virtual_tbl.setMaterializer(catalog_tbl);
    virtual_tbl.setSystable(true);
    virtual_tbl.getColumns().clear();

    // Create MaterializedView and link it to the virtual table
    MaterializedViewInfo catalog_view = catalog_tbl.getViews().add(viewName);
    catalog_view.setVerticalpartition(true);
    catalog_view.setDest(virtual_tbl);
    List<Column> indexColumns = new ArrayList<Column>();

    Column partition_col = catalog_tbl.getPartitioncolumn();
    if (partition_col instanceof VerticalPartitionColumn) {
      partition_col = ((VerticalPartitionColumn) partition_col).getHorizontalColumn();
    }
    if (debug.get()) LOG.debug(catalog_tbl.getName() + " Partition Column: " + partition_col);

    int i = 0;
    assert (catalog_cols != null);
    assert (catalog_cols.isEmpty() == false)
        : "No vertical partitioning columns for " + catalog_view.fullName();
    for (Column catalog_col : catalog_cols) {
      // MaterializedView ColumnRef
      ColumnRef catalog_ref = catalog_view.getGroupbycols().add(catalog_col.getName());
      catalog_ref.setColumn(catalog_col);
      catalog_ref.setIndex(i++);

      // VirtualTable Column
      Column virtual_col = virtual_tbl.getColumns().add(catalog_col.getName());
      virtual_col.setDefaulttype(catalog_col.getDefaulttype());
      virtual_col.setDefaultvalue(catalog_col.getDefaultvalue());
      virtual_col.setIndex(catalog_col.getIndex());
      virtual_col.setNullable(catalog_col.getNullable());
      virtual_col.setSize(catalog_col.getSize());
      virtual_col.setType(catalog_col.getType());
      if (debug.get())
        LOG.debug(String.format("Added VerticalPartition column %s", virtual_col.fullName()));

      // If they want an index, then we'll make one based on every column except for the column
      // that the table is partitioned on
      if (createIndex) {
        boolean include = true;
        if (partition_col instanceof MultiColumn) {
          include = (((MultiColumn) partition_col).contains(catalog_col) == false);
        } else if (catalog_col.equals(partition_col)) {
          include = false;
        }
        if (include) indexColumns.add(virtual_col);
      }
    } // FOR

    if (createIndex) {
      if (indexColumns.isEmpty()) {
        Map<String, Object> m = new ListOrderedMap<String, Object>();
        m.put("Partition Column", partition_col);
        m.put("VP Table Columns", virtual_tbl.getColumns());
        m.put("Passed-in Columns", CatalogUtil.debug(catalog_cols));
        LOG.error("Failed to find index columns\n" + StringUtil.formatMaps(m));
        throw new Exception(String.format("No columns selected for index on %s", viewName));
      }
      String idxName = "SYS_IDX_" + viewName;
      Index virtual_idx = virtual_tbl.getIndexes().get(idxName);
      if (virtual_idx == null) {
        virtual_idx = virtual_tbl.getIndexes().add(idxName);
      }
      virtual_idx.getColumns().clear();

      IndexType idxType =
          (indexColumns.size() == 1 ? IndexType.HASH_TABLE : IndexType.BALANCED_TREE);
      virtual_idx.setType(idxType.getValue());
      i = 0;
      for (Column catalog_col : indexColumns) {
        ColumnRef cref = virtual_idx.getColumns().add(catalog_col.getTypeName());
        cref.setColumn(catalog_col);
        cref.setIndex(i++);
      } // FOR

      if (debug.get())
        LOG.debug(
            String.format(
                "Created %s index '%s' for vertical partition '%s'", idxType, idxName, viewName));
    }
    return (catalog_view);
  }