예제 #1
0
  private List<Future> getBatches(DataLoadThreadTime dataLoadThreadTime, Scenario scenario)
      throws Exception {
    RowCalculator rowCalculator = new RowCalculator(getThreadPoolSize(), scenario.getRowCount());
    List<Future> writeBatches = new ArrayList<>();

    for (int i = 0; i < getThreadPoolSize(); i++) {
      List<Column> phxMetaCols =
          pUtil.getColumnsFromPhoenix(
              scenario.getSchemaName(),
              scenario.getTableNameWithoutSchemaName(),
              pUtil.getConnection());
      int threadRowCount = rowCalculator.getNext();
      logger.info("Kick off thread (#" + i + ")for upsert with (" + threadRowCount + ") rows.");
      Future<Info> write =
          upsertData(
              scenario, phxMetaCols, scenario.getTableName(), threadRowCount, dataLoadThreadTime);
      writeBatches.add(write);
    }
    if (writeBatches.isEmpty()) {
      throw new PherfException(
          "Holy shit snacks! Throwing up hands in disbelief and exiting. Could not write data for some unknown reason.");
    }

    return writeBatches;
  }
  /**
   * Timed query execution
   *
   * @throws Exception
   */
  private void timedQuery() throws Exception {
    boolean isSelectCountStatement =
        query.getStatement().toUpperCase().trim().contains("COUNT(*)") ? true : false;

    Connection conn = null;
    PreparedStatement statement = null;
    ResultSet rs = null;
    Long start = System.currentTimeMillis();
    Date startDate = Calendar.getInstance().getTime();
    String exception = null;
    long resultRowCount = 0;

    try {
      conn = pUtil.getConnection(query.getTenantId());
      statement = conn.prepareStatement(query.getStatement());
      boolean isQuery = statement.execute();
      if (isQuery) {
        rs = statement.getResultSet();
        while (rs.next()) {
          if (null != query.getExpectedAggregateRowCount()) {
            if (rs.getLong(1) != query.getExpectedAggregateRowCount())
              throw new RuntimeException(
                  "Aggregate count "
                      + rs.getLong(1)
                      + " does not match expected "
                      + query.getExpectedAggregateRowCount());
          }

          if (isSelectCountStatement) {
            resultRowCount = rs.getLong(1);
          } else {
            resultRowCount++;
          }
        }
      } else {
        conn.commit();
      }
    } catch (Exception e) {
      e.printStackTrace();
      exception = e.getMessage();
    } finally {
      getThreadTime()
          .getRunTimesInMs()
          .add(
              new RunTime(
                  exception,
                  startDate,
                  resultRowCount,
                  (int) (System.currentTimeMillis() - start)));

      if (rs != null) rs.close();
      if (statement != null) statement.close();
      if (conn != null) conn.close();
    }
  }
예제 #3
0
 public WriteWorkload(XMLConfigParser parser) throws Exception {
   this(PhoenixUtil.create(), parser);
 }
예제 #4
0
 /**
  * TODO Move this method to PhoenixUtil Update Phoenix table stats
  *
  * @param tableName
  * @throws Exception
  */
 public void updatePhoenixStats(String tableName) throws Exception {
   logger.info("Updating stats for " + tableName);
   pUtil.executeStatement("UPDATE STATISTICS " + tableName);
 }
class MultiThreadedRunner implements Runnable {
  private static final Logger logger = LoggerFactory.getLogger(MultiThreadedRunner.class);
  private Query query;
  private ThreadTime threadTime;
  private PhoenixUtil pUtil = PhoenixUtil.create();
  private String threadName;
  private DataModelResult dataModelResult;
  private long numberOfExecutions;
  private long executionDurationInMs;
  private static long lastResultWritten = System.currentTimeMillis() - 1000;
  private final ResultManager resultManager;

  /**
   * MultiThreadedRunner
   *
   * @param threadName
   * @param query
   * @param dataModelResult
   * @param threadTime
   * @param numberOfExecutions
   * @param executionDurationInMs
   */
  MultiThreadedRunner(
      String threadName,
      Query query,
      DataModelResult dataModelResult,
      ThreadTime threadTime,
      long numberOfExecutions,
      long executionDurationInMs,
      boolean writeRuntimeResults) {
    this.query = query;
    this.threadName = threadName;
    this.threadTime = threadTime;
    this.dataModelResult = dataModelResult;
    this.numberOfExecutions = numberOfExecutions;
    this.executionDurationInMs = executionDurationInMs;
    this.resultManager = new ResultManager(dataModelResult.getName(), writeRuntimeResults);
  }

  /** Executes run for a minimum of number of execution or execution duration */
  @Override
  public void run() {
    logger.info(
        "\n\nThread Starting "
            + threadName
            + " ; "
            + query.getStatement()
            + " for "
            + numberOfExecutions
            + "times\n\n");
    Long start = System.currentTimeMillis();
    for (long i = numberOfExecutions;
        (i > 0 && ((System.currentTimeMillis() - start) < executionDurationInMs));
        i--) {
      try {
        synchronized (resultManager) {
          timedQuery();
          if ((System.currentTimeMillis() - lastResultWritten) > 1000) {
            resultManager.write(dataModelResult);
            lastResultWritten = System.currentTimeMillis();
          }
        }
      } catch (Exception e) {
        e.printStackTrace();
      }
    }
    logger.info("\n\nThread exiting." + threadName + "\n\n");
  }

  private synchronized ThreadTime getThreadTime() {
    return threadTime;
  }

  /**
   * Timed query execution
   *
   * @throws Exception
   */
  private void timedQuery() throws Exception {
    boolean isSelectCountStatement =
        query.getStatement().toUpperCase().trim().contains("COUNT(*)") ? true : false;

    Connection conn = null;
    PreparedStatement statement = null;
    ResultSet rs = null;
    Long start = System.currentTimeMillis();
    Date startDate = Calendar.getInstance().getTime();
    String exception = null;
    long resultRowCount = 0;

    try {
      conn = pUtil.getConnection(query.getTenantId());
      statement = conn.prepareStatement(query.getStatement());
      boolean isQuery = statement.execute();
      if (isQuery) {
        rs = statement.getResultSet();
        while (rs.next()) {
          if (null != query.getExpectedAggregateRowCount()) {
            if (rs.getLong(1) != query.getExpectedAggregateRowCount())
              throw new RuntimeException(
                  "Aggregate count "
                      + rs.getLong(1)
                      + " does not match expected "
                      + query.getExpectedAggregateRowCount());
          }

          if (isSelectCountStatement) {
            resultRowCount = rs.getLong(1);
          } else {
            resultRowCount++;
          }
        }
      } else {
        conn.commit();
      }
    } catch (Exception e) {
      e.printStackTrace();
      exception = e.getMessage();
    } finally {
      getThreadTime()
          .getRunTimesInMs()
          .add(
              new RunTime(
                  exception,
                  startDate,
                  resultRowCount,
                  (int) (System.currentTimeMillis() - start)));

      if (rs != null) rs.close();
      if (statement != null) statement.close();
      if (conn != null) conn.close();
    }
  }
}