Example #1
0
  private String toStringTrueFalse() {
    StringBuilder sb = new StringBuilder();

    final int COLUMN_WIDTH = 14;
    String formatString = "%" + COLUMN_WIDTH + "s";

    // make header.
    sb.append("True-False: \n-----------\n");
    sb.append(String.format(formatString, "Index"));
    sb.append(String.format(formatString, "Trials"));

    for (AlgorithmType t : AlgorithmType.values()) {
      sb.append(String.format(formatString, t));
    }
    sb.append(String.format(formatString, "Agent"));
    sb.append(String.format(formatString, "Description"));
    sb.append("\n");
    for (int i = 0; i < AlgorithmType.values().length + 4; i++)
      sb.append(String.format(formatString, "-------------"));
    sb.append("\n");

    int totalCount = 0; // for displaying the total count at the bottom.
    double weightedAgentAccuracy =
        0; // for calculating weighted accuracy rate across all question profiles.

    // display values.
    for (int i = 0; i < NUM_FEATURE_COMBOS; i++) {
      // 2016/01/05 - version 2.0.0 - added condition for
      // trial counts > 0 in order to remove the "zero" rows,
      // i.e., rows showing profile indices where there are no
      // trials.
      // Also, added the hasFeature(truefalse) condition.
      if (trialCounts[i] > 0 && Profile.hasFeature(i, FeatureType.TRUE_FALSE.ordinal())) {
        sb.append(String.format(formatString, i));
        sb.append(String.format(formatString, trialCounts[i]));

        for (int j = 0; j < AlgorithmType.values().length; j++) {
          sb.append(String.format("%" + COLUMN_WIDTH + ".3f", successProbs[i][j]));
        }
        double agentAccuracy = maxSuccessProb(successProbs[i]);
        sb.append(String.format("%" + COLUMN_WIDTH + ".3f", agentAccuracy));
        sb.append(Profile.getDescription(i));
        sb.append("\n");
        totalCount += trialCounts[i];
        weightedAgentAccuracy += trialCounts[i] * agentAccuracy;
      }
    }
    weightedAgentAccuracy /= totalCount;

    // display total trial count
    //		sb.append(String.format("%" + COLUMN_WIDTH + "s%" + COLUMN_WIDTH
    //				+ "d\n", "Total Count:", totalCount));
    sb.append(
        String.format(
            "%"
                + COLUMN_WIDTH
                + "s%"
                + COLUMN_WIDTH
                + "d%"
                + COLUMN_WIDTH * 9
                + "s%"
                + COLUMN_WIDTH
                + ".3f\n",
            "Total Count:",
            totalCount,
            " ",
            weightedAgentAccuracy));

    return new String(sb);
  }
Example #2
0
/**
 * ProfileData acts as a data store facility for the results of the test performed by the Profile
 * object. This data includes the number of trials and number of successes among those trials (i.e.,
 * the number of times the correct answer was selected) broken down by algorithm type and question
 * profile (i.e., by combination of features).
 *
 * @author jjohnson346
 */
public class ProfileData {

  private int[][] successCounts; // stores the number of successes for each
  // question profile,
  // for each algo.

  public final int[] trialCounts; // stores the number of trials for each
  // question profile.

  public final double[][] successProbs; // the prob of success for each algo,
  // for each question profile
  // IMPORTANT: note that it is PUBLIC

  private final int NUM_ALGOS = AlgorithmType.values().length; // the number
  // of algos
  // to store
  // data for

  private final int NUM_FEATURES = FeatureType.values().length; // the number
  // of
  // features
  // identified

  private final int NUM_FEATURE_COMBOS = (int) Math.pow(2.0, NUM_FEATURES); // the
  // number
  // of
  // combos
  // of
  // features,
  // assume
  // true/false
  // value
  // for
  // each.

  /**
   * constructor initializes the successCounts, trialCounts and successProbs arrays according to the
   * number of algos, number of features, and the consequent number of feature combinations.
   */
  public ProfileData() {
    successCounts = new int[NUM_FEATURE_COMBOS][NUM_ALGOS];
    trialCounts = new int[NUM_FEATURE_COMBOS];
    successProbs = new double[NUM_FEATURE_COMBOS][NUM_ALGOS];
  }

  /**
   * inserts the results data for a question into the data store. That is, this function inserts for
   * each algorithm, the results of applying that algorithm on the question, as given by the array,
   * results, passed in as an input argument. This data is inserted the location of the data store
   * corresponding to the profile for the given question.
   *
   * @param question the question for which to insert the data, the profile is the pertinent info,
   *     here.
   * @param results an array of booleans giving whether the corresponding algo was successful on the
   *     question.
   */
  public void insert(CFEExamQuestion question, boolean[] results) {
    // get the profile for the question passed in as an input parm.
    int profileIndex = question.getProfile().getProfileIndex();

    // increment the number of trials for the index corresponding to the
    // question's profile.
    trialCounts[profileIndex]++;

    // increment the success count in the successCounts array for those
    // algos that were successful.
    for (int j = 0; j < successCounts[profileIndex].length; j++) {
      if (results[j]) successCounts[profileIndex][j]++;
    }
  }

  /**
   * calculates the probability of success for a each algorithm on each question profile and stores
   * the results in a public array, successProbs, which is used by the CFEExamAgent to determine on
   * each question it confronts which algo to use. That is, it picks the algo with highest
   * probability of success given the profile of the current question.
   */
  public void calculate() {
    for (int i = 0; i < NUM_FEATURE_COMBOS; i++) {
      for (int j = 0; j < NUM_ALGOS; j++) {
        if (trialCounts[i] != 0) successProbs[i][j] = (double) successCounts[i][j] / trialCounts[i];
        else successProbs[i][j] = 0.0;
      }
    }
  }

  /**
   * loads profile data from file, profile data.txt, and stores the contents in the public array,
   * successProbs. successProbs is the critical array used by the CFEExamAgent for selecting the
   * algo to use on each question it confronts.
   */
  public void load() throws FileNotFoundException {
    // change backslashes to forward slashes for mac version.
    // Scanner scanner = new Scanner(new
    // File("profile data\\profile data.txt"));
    Scanner scanner = new Scanner(new File("profile data//profile data.txt"));

    // skip first 2 header lines.
    scanner.nextLine();
    scanner.nextLine();

    for (int i = 0; i < NUM_FEATURE_COMBOS; i++) {

      // skip first column showing the index.
      scanner.nextInt();

      trialCounts[i] = scanner.nextInt();

      for (int j = 0; j < NUM_ALGOS; j++) {
        successProbs[i][j] = scanner.nextDouble();
      }
    }
    scanner.close();
  }

  /**
   * returns a pretty formatted version of the contents of number of trials and successProb for each
   * question profile.
   */
  @Override
  public String toString() {
    StringBuilder sb = new StringBuilder();
    sb.append(toStringTrueFalse());
    sb.append("\n\n");
    sb.append(toStringMultipleChoice());
    return new String(sb);
    //		StringBuilder sb = new StringBuilder();
    //
    //		final int COLUMN_WIDTH = 14;
    //		String formatString = "%" + COLUMN_WIDTH + "s";
    //
    //		// make header.
    //		sb.append(String.format(formatString, "Index"));
    //		sb.append(String.format(formatString, "Trials"));
    //
    //		for (AlgorithmType t : AlgorithmType.values()) {
    //			sb.append(String.format(formatString, t));
    //		}
    //		sb.append(String.format(formatString, "Agent"));
    //		sb.append(String.format(formatString, "Description"));
    //		sb.append("\n");
    //		for (int i = 0; i < AlgorithmType.values().length + 4; i++)
    //			sb.append(String.format(formatString, "-------------"));
    //		sb.append("\n");
    //
    //		// display values.
    //		for (int i = 0; i < NUM_FEATURE_COMBOS; i++) {
    //			// 2016/01/05 - version 2.0.0 - added condition for
    //			// trial counts > 0 in order to remove the "zero" rows,
    //			// i.e., rows showing profile indices where there are no
    //			// trials.
    //			if (trialCounts[i] > 0) {
    //				sb.append(String.format(formatString, i));
    //				sb.append(String.format(formatString, trialCounts[i]));
    //
    //				for (int j = 0; j < AlgorithmType.values().length; j++) {
    //					sb.append(String.format("%" + COLUMN_WIDTH + ".3f", successProbs[i][j]));
    //				}
    //				sb.append(String.format("%" + COLUMN_WIDTH + ".3f", maxSuccessProb(successProbs[i])));
    //				sb.append(Profile.getDescription(i));
    //				sb.append("\n");
    //			}
    //		}
    //
    //		// display total trial count
    //		int totalCount = 0;
    //		for (int i = 0; i < trialCounts.length; i++) {
    //			totalCount += trialCounts[i];
    //		}
    //		sb.append(String.format("%" + COLUMN_WIDTH + "s%" + COLUMN_WIDTH
    //				+ "d\n", "Total Count:", totalCount));
    //
    //		return new String(sb);
  }

  private String toStringMultipleChoice() {
    StringBuilder sb = new StringBuilder();

    final int COLUMN_WIDTH = 14;
    String formatString = "%" + COLUMN_WIDTH + "s";

    // make header.
    sb.append("Multiple Choice:\n----------------\n");
    sb.append(String.format(formatString, "Index"));
    sb.append(String.format(formatString, "Trials"));

    for (AlgorithmType t : AlgorithmType.values()) {
      sb.append(String.format(formatString, t));
    }
    sb.append(String.format(formatString, "Agent"));
    sb.append(String.format(formatString, "Description"));
    sb.append("\n");
    for (int i = 0; i < AlgorithmType.values().length + 4; i++)
      sb.append(String.format(formatString, "-------------"));
    sb.append("\n");

    int totalCount = 0; // for displaying the total count at the bottom.
    double weightedAgentAccuracy =
        0; // for calculating weighted accuracy rate across all question profiles.

    // display values.
    for (int i = 0; i < NUM_FEATURE_COMBOS; i++) {
      // 2016/01/05 - version 2.0.0 - added condition for
      // trial counts > 0 in order to remove the "zero" rows,
      // i.e., rows showing profile indices where there are no
      // trials.
      // Also, added the !hasFeature(truefalse) condition.
      if (trialCounts[i] > 0 && !Profile.hasFeature(i, FeatureType.TRUE_FALSE.ordinal())) {
        sb.append(String.format(formatString, i));
        sb.append(String.format(formatString, trialCounts[i]));

        for (int j = 0; j < AlgorithmType.values().length; j++) {
          sb.append(String.format("%" + COLUMN_WIDTH + ".3f", successProbs[i][j]));
        }
        double agentAccuracy = maxSuccessProb(successProbs[i]);
        sb.append(String.format("%" + COLUMN_WIDTH + ".3f", maxSuccessProb(successProbs[i])));
        sb.append(Profile.getDescription(i));
        sb.append("\n");
        totalCount += trialCounts[i];
        weightedAgentAccuracy += trialCounts[i] * agentAccuracy;
      }
    }
    weightedAgentAccuracy /= totalCount;

    // display total trial count
    sb.append(
        String.format(
            "%"
                + COLUMN_WIDTH
                + "s%"
                + COLUMN_WIDTH
                + "d%"
                + COLUMN_WIDTH * 9
                + "s%"
                + COLUMN_WIDTH
                + ".3f\n",
            "Total Count:",
            totalCount,
            " ",
            weightedAgentAccuracy));

    return new String(sb);
  }

  private String toStringTrueFalse() {
    StringBuilder sb = new StringBuilder();

    final int COLUMN_WIDTH = 14;
    String formatString = "%" + COLUMN_WIDTH + "s";

    // make header.
    sb.append("True-False: \n-----------\n");
    sb.append(String.format(formatString, "Index"));
    sb.append(String.format(formatString, "Trials"));

    for (AlgorithmType t : AlgorithmType.values()) {
      sb.append(String.format(formatString, t));
    }
    sb.append(String.format(formatString, "Agent"));
    sb.append(String.format(formatString, "Description"));
    sb.append("\n");
    for (int i = 0; i < AlgorithmType.values().length + 4; i++)
      sb.append(String.format(formatString, "-------------"));
    sb.append("\n");

    int totalCount = 0; // for displaying the total count at the bottom.
    double weightedAgentAccuracy =
        0; // for calculating weighted accuracy rate across all question profiles.

    // display values.
    for (int i = 0; i < NUM_FEATURE_COMBOS; i++) {
      // 2016/01/05 - version 2.0.0 - added condition for
      // trial counts > 0 in order to remove the "zero" rows,
      // i.e., rows showing profile indices where there are no
      // trials.
      // Also, added the hasFeature(truefalse) condition.
      if (trialCounts[i] > 0 && Profile.hasFeature(i, FeatureType.TRUE_FALSE.ordinal())) {
        sb.append(String.format(formatString, i));
        sb.append(String.format(formatString, trialCounts[i]));

        for (int j = 0; j < AlgorithmType.values().length; j++) {
          sb.append(String.format("%" + COLUMN_WIDTH + ".3f", successProbs[i][j]));
        }
        double agentAccuracy = maxSuccessProb(successProbs[i]);
        sb.append(String.format("%" + COLUMN_WIDTH + ".3f", agentAccuracy));
        sb.append(Profile.getDescription(i));
        sb.append("\n");
        totalCount += trialCounts[i];
        weightedAgentAccuracy += trialCounts[i] * agentAccuracy;
      }
    }
    weightedAgentAccuracy /= totalCount;

    // display total trial count
    //		sb.append(String.format("%" + COLUMN_WIDTH + "s%" + COLUMN_WIDTH
    //				+ "d\n", "Total Count:", totalCount));
    sb.append(
        String.format(
            "%"
                + COLUMN_WIDTH
                + "s%"
                + COLUMN_WIDTH
                + "d%"
                + COLUMN_WIDTH * 9
                + "s%"
                + COLUMN_WIDTH
                + ".3f\n",
            "Total Count:",
            totalCount,
            " ",
            weightedAgentAccuracy));

    return new String(sb);
  }

  private double maxSuccessProb(double[] successProbs) {
    double max = successProbs[0];
    for (int i = 1; i < successProbs.length; i++) {
      if (successProbs[i] > max) max = successProbs[i];
    }
    return max;
  }
}