@Test
  public void test9680_WhichHasAOneThousanthFraction() throws Exception {
    String TEST_REACH_CLIENT_ID = "9680";
    Long TEST_REACH_SYSTEM_ID = 9680L;
    Long ONLY_REACH_UPSTREAM_OF_TEST_REACH_ID = 9681L;

    // Create terminal reaches and put in cache
    List<String> targetList = new ArrayList<String>();
    targetList.add(TEST_REACH_CLIENT_ID);
    TerminalReaches targets = new TerminalReaches(TEST_MODEL_ID, targetList);
    ConfiguredCache.TerminalReaches.put(targets.getId(), targets);

    // Load predict data and model incremental areas - used for comparison
    PredictData pd = SharedApplication.getInstance().getPredictData(TEST_MODEL_ID);
    DataTable incrementalReachAreas =
        SharedApplication.getInstance()
            .getCatchmentAreas(new UnitAreaRequest(TEST_MODEL_ID, AreaType.INCREMENTAL));

    // Stats on the test reach
    int testReachRowNumber = pd.getRowForReachID(TEST_REACH_SYSTEM_ID);
    Double testReachFrac = pd.getTopo().getDouble(testReachRowNumber, PredictData.TOPO_FRAC_COL);
    Double testReachFractionedWatershedArea =
        new CalcFractionedWatershedArea(
                new FractionedWatershedAreaRequest(
                    new ReachID(TEST_MODEL_ID, TEST_REACH_SYSTEM_ID)))
            .run();
    Double testReachUnfractionedWatershedArea =
        new CalcFractionedWatershedArea(
                new FractionedWatershedAreaRequest(
                    new ReachID(TEST_MODEL_ID, TEST_REACH_SYSTEM_ID), false, false, true))
            .run();
    Double testReachIncrementalArea = incrementalReachAreas.getDouble(testReachRowNumber, 1);

    // load stats on the only reach immediately upstream of the test reach
    Double upstreamReachFractionedWatershedArea =
        new CalcFractionedWatershedArea(
                new FractionedWatershedAreaRequest(
                    new ReachID(TEST_MODEL_ID, ONLY_REACH_UPSTREAM_OF_TEST_REACH_ID)))
            .run();

    // Test the assumptions about the basic (non-table form) of the area data
    assertTrue(
        Math.abs(testReachFractionedWatershedArea - testReachUnfractionedWatershedArea)
            > 1d); // These should be different values
    assertTrue(Math.abs(testReachFractionedWatershedArea - testReachIncrementalArea) > 1d);
    assertEquals(
        testReachFrac * upstreamReachFractionedWatershedArea + testReachIncrementalArea,
        testReachFractionedWatershedArea,
        .0001d);

    // Do the same calculation via the Table version of the action
    CalcFractionedWatershedAreaTable action = new CalcFractionedWatershedAreaTable(targets.getId());
    ColumnData data = action.run();

    assertEquals(testReachFractionedWatershedArea, data.getDouble(testReachRowNumber), .0001d);
  }
  @Before
  public void setup() {

    // Set up contexts
    AdjustmentGroups adjustments = new AdjustmentGroups(TEST_MODEL_ID);

    predictData = SharedApplication.getInstance().getPredictData(TEST_MODEL_ID);
    predictResult = SharedApplication.getInstance().getPredictResult(adjustments);
    UnitAreaRequest watershedAreaReq =
        new UnitAreaRequest(TEST_MODEL_ID, AreaType.TOTAL_CONTRIBUTING);
    DataTable watershedAreaTable =
        SharedApplication.getInstance().getCatchmentAreas(watershedAreaReq);
    watershedAreaColumn = new ColumnDataFromTable(watershedAreaTable, 1);
  }
  /**
   * Runs QA checks against the data.
   *
   * @param modelId
   * @return
   * @throws Exception
   */
  public boolean testSingleModelDataQuality(Long modelId) throws Exception {
    Connection conn = SharedApplication.getInstance().getROConnection();

    // Get list of queries in properties file
    Properties props = new Properties();

    String path = this.getClass().getName().replace('.', '/') + ".properties";
    InputStream ins = Thread.currentThread().getContextClassLoader().getResourceAsStream(path);
    if (ins == null) {
      ins = Action.class.getResourceAsStream(path);
    }
    props.load(ins);

    Enumeration<?> elements = props.propertyNames();
    boolean passed = true;

    try {
      while (elements.hasMoreElements()) {
        String queryName = elements.nextElement().toString();
        log.debug("Running data validation test " + queryName + "'...");
        passed = passed & testSingleModelDataQuality(modelId, queryName, conn);
      }
    } finally {
      SharedApplication.closeConnection(conn, null);
    }

    if (passed) {
      log.debug("++++++++ Model #" + modelId + " PASSED all data validation tests ++++++++");
    } else {
      // The fail message is printed in the deligated method, so no need to reprint here.
    }
    return passed;
  }
  @Override
  public ReachRowValueMap createEntry(Object terminalReaches) throws Exception {
    TerminalReaches targets = (TerminalReaches) terminalReaches;

    PredictData predictData =
        SharedApplication.getInstance().getPredictData(new Long(targets.getModelID()));

    Set<String> clientTargetReachIdSet = targets.getReachIdsAsSet();

    List<Long> targetReachIdList =
        SharedApplication.getInstance()
            .getReachFullIdAsLong(targets.getModelID(), clientTargetReachIdSet);
    HashSet<Long> targetReachIdSet = new HashSet<Long>();
    targetReachIdSet.addAll(targetReachIdList);

    CalcDeliveryFractionMap action = new CalcDeliveryFractionMap(predictData, targetReachIdSet);

    ReachRowValueMap delFrac = action.run();

    return delFrac;
  }
  @Override
  protected void initFields() throws Exception {
    File dataDir = getDataDirectory();

    if (!dataDir.exists()) {
      Files.createDirectories(dataDir.toPath());
    }

    dataColumn = context.getDataColumn().getColumnData();
    columnIndex =
        SharedApplication.getInstance().getPredictData(context.getModelID()).getTopo().getIndex();
    outputFile =
        new File(
            dataDir,
            NamingConventions.convertContextIdToXMLSafeName(
                    context.getModelID().intValue(), context.getId())
                + ".dbf");
    outputFile.createNewFile();

    DataSeriesType type = context.getAnalysis().getDataSeries();

    // grab the delivery fraction map if this is a delivery data series.
    // This is used to weed out the reaches that are not upstream of the
    // user selected terminal reaches.
    if (type.isDeliveryRequired()) {

      TerminalReaches tReaches = context.getTerminalReaches();

      assert (tReaches != null) : "client should not submit a delivery request without reaches";

      reachRowValueMap = SharedApplication.getInstance().getDeliveryFractionMap(tReaches);

      if (reachRowValueMap == null) {
        throw new Exception("Unable to find or calculate the delivery fraction map");
      }
    } else {
      reachRowValueMap = null;
    }
  }
  /**
   * Tests all the source / total / incremental combinations for reasonable std error estimate
   * values.
   *
   * @param modelId
   * @return
   * @throws Exception
   */
  public boolean testSingleModelErrorEstimates(Long modelId) throws Exception {
    PredictData pd = SharedApplication.getInstance().getPredictData(modelId);
    int failCount = 0;

    for (int srcIndex = 0; srcIndex < pd.getSrcMetadata().getRowCount(); srcIndex++) {
      int srcId = pd.getSourceIdForSourceIndex(srcIndex);
      failCount += testSingleModelErrorEstimates(modelId, srcId, false);
      failCount += testSingleModelErrorEstimates(modelId, srcId, true);
    }

    // Add the total values (ie, all sources togeteher
    failCount += testSingleModelErrorEstimates(modelId, null, false);
    failCount += testSingleModelErrorEstimates(modelId, null, true);

    System.out.println("Grand Total Error est failures for model " + modelId + ": " + failCount);
    return failCount == 0;
  }
  protected File getDataDirectory() {
    File dDir;

    if (this.dataDirectory != null) {
      dDir = this.dataDirectory;
    } else {
      DynamicReadOnlyProperties props = SharedApplication.getInstance().getConfiguration();
      String fallbackDataDirectory =
          System.getProperty("user.home")
              + File.separatorChar
              + "sparrow"
              + File.separatorChar
              + "data";
      String sparrowDataDirectory = props.getProperty(DATA_EXPORT_DIRECTORY, fallbackDataDirectory);
      dDir = new File(sparrowDataDirectory);
      this.dataDirectory = dDir;
    }

    return dDir;
  }
  @Test
  public void testTargetWith4UpstreamReaches() throws Exception {
    Long TEST_ROW_SYSTEM_ID = 9687L;
    String TEST_ROW_CLIENT_ID = "9687";

    List<String> targetList = new ArrayList<String>();
    targetList.add(TEST_ROW_CLIENT_ID);
    TerminalReaches targets = new TerminalReaches(TEST_MODEL_ID, targetList);
    ConfiguredCache.TerminalReaches.put(targets.getId(), targets);

    PredictData pd = SharedApplication.getInstance().getPredictData(TEST_MODEL_ID);
    Double watershedArea =
        new CalcFractionedWatershedArea(
                new FractionedWatershedAreaRequest(new ReachID(TEST_MODEL_ID, TEST_ROW_SYSTEM_ID)))
            .run();

    int rowCount = pd.getTopo().getRowCount();
    int rowNumber = pd.getRowForReachID(TEST_ROW_SYSTEM_ID);

    CalcFractionedWatershedAreaTable action = new CalcFractionedWatershedAreaTable(targets.getId());
    ColumnData data = action.run();

    assertEquals(rowCount, data.getRowCount().intValue());
    assertNotNull(data.getDouble(rowNumber));

    for (int row = 0; row < rowCount; row++) {
      if (row == rowNumber) {
        assertEquals(
            "Row " + row + " should be " + watershedArea,
            watershedArea,
            data.getDouble(row),
            .000000001d);
      } else {
        assertNull("Row " + row + " should be null", data.getDouble(row));
      }
    }
  }
 /**
  * Returns a sorted list of identifiers for reaches that fall within the bounding box defined by
  * the specified arguments. A reach is considered to fall within the bounding box if any part of
  * its geometry falls within the bounding box.
  *
  * @return A list of identifiers for reaches that fall within the defined bounding box.
  * @throws Exception
  */
 private Long[] getResults(Long modelId, String bounds) throws Exception {
   ModelBBox modelBBox = new ModelBBox(modelId, bounds);
   Long[] ids = SharedApplication.getInstance().getReachesInBBox(modelBBox);
   return ids;
 }
  /**
   * Tests a single model / source / total/incremental combination for a reasonalbe std error
   * estimate.
   *
   * @param modelId
   * @param sourceId
   * @param total
   * @return
   * @throws Exception
   */
  public int testSingleModelErrorEstimates(Long modelId, Integer sourceId, boolean total)
      throws Exception {

    AdjustmentGroups noAdjustments = new AdjustmentGroups(modelId);
    int failCount = 0;
    String incTot = (total) ? "Total" : "Incremental";

    // Build the error data series
    DataSeriesType errSeries =
        (total)
            ? DataSeriesType.total_std_error_estimate
            : DataSeriesType.incremental_std_error_estimate;
    BasicAnalysis errAnalysis = new BasicAnalysis(errSeries, sourceId, null, null);
    PredictionContext errContext =
        new PredictionContext(modelId, noAdjustments, errAnalysis, null, null, null);
    SparrowColumnSpecifier errColumn =
        SharedApplication.getInstance().getAnalysisResult(errContext);

    ///
    // Build the related standard series
    DataSeriesType stdSeries = (total) ? DataSeriesType.total : DataSeriesType.incremental;
    BasicAnalysis stdAnalysis = new BasicAnalysis(stdSeries, sourceId, null, null);
    PredictionContext stdContext =
        new PredictionContext(modelId, noAdjustments, stdAnalysis, null, null, null);
    SparrowColumnSpecifier stdColumn =
        SharedApplication.getInstance().getAnalysisResult(stdContext);

    ///
    // Do the comparison
    for (int row = 0; row < stdColumn.getRowCount(); row++) {
      Double std = stdColumn.getDouble(row);
      Double err = errColumn.getDouble(row);

      if (err == null && std < 1d) {
        // I think its OK to have no err estimate if the predicted value is very small
        // System.out.println("Predict: " + std + " Err: null");
      } else if (err != null && std <= 1d && err <= 20) {
        // An error of 10 is pretty small, though it may be many times larger than the predicted
        // value
        // System.out.println("Predict: " + std + " Err: " + err);
      } else if (err != null && err <= (std * 20)) {
        // System.out.println("Predict: " + std + " Err: " + err);
      } else {
        System.out.println(
            "Model "
                + modelId
                + ", Src:"
                + sourceId
                + " "
                + incTot
                + ": Suspicious Std Error.  Value: "
                + std
                + " Err: "
                + err);
        failCount++;
      }
    }

    System.out.println(
        "Model "
            + modelId
            + ", Src:"
            + sourceId
            + " "
            + incTot
            + ": Total Error est failures: "
            + failCount);

    return failCount;
  }
  public boolean testSingleMmodel(URL predictFile, Long modelId, boolean logWithDetails)
      throws Exception {
    beforeEachTest();
    boolean pass = false;

    DataTable t = loadTextPredict(predictFile);

    if (t != null) {
      AdjustmentGroups ags = new AdjustmentGroups(modelId);
      PredictResult prs = SharedApplication.getInstance().getPredictResult(ags);
      PredictData pd = SharedApplication.getInstance().getPredictData(modelId);

      log.setLevel(Level.FATAL); // Turn off logging
      int noDecayFailures = testComparison(t, prs, pd, false, modelId, logWithDetails);
      int decayFailures = testComparison(t, prs, pd, true, modelId, logWithDetails);
      log.setLevel(Level.DEBUG); // Turn back on

      if (decayFailures < noDecayFailures) {
        if (decayFailures == 0) {
          log.debug(
              "++++++++ Model #" + modelId + " PASSED using DECAYED incremental values +++++++++");
          pass = true;
        } else {
          log.debug(
              "-------- Model #"
                  + modelId
                  + " FAILED using DECAYED incremental values.  Details: --------");
          testComparison(t, prs, pd, true, modelId, logWithDetails);
        }
      } else if (noDecayFailures < decayFailures) {
        if (noDecayFailures == 0) {
          log.debug(
              "++++++++ Model #"
                  + modelId
                  + " PASSED using NON-DECAYED incremental values +++++++++");
          pass = true;
        } else {
          log.debug(
              "-------- Model #"
                  + modelId
                  + " FAILED using NON-DECAYED incremental values.  Details: --------");
          testComparison(t, prs, pd, false, modelId, logWithDetails);
        }
      } else if (noDecayFailures == decayFailures) {
        // Equal failures mean there is a row count or other type of error
        log.debug(
            "Hey, found these no decay fails: "
                + noDecayFailures
                + " and these decay fails: "
                + decayFailures);
        log.debug(
            "-------- Model #"
                + modelId
                + " FAILED.  MAJOR ISSUE (no decay matches decay) - SEE DETAILS: --------");
        testComparison(t, prs, pd, false, modelId, logWithDetails);
      }
    } else {
      log.debug(
          "-------- Model #"
              + modelId
              + " FAILED.  MAJOR ISSUE (couldn't load file) - SEE DETAILS (above) --------");
    }

    return pass;
  }
  public void runTheTests() throws Exception {

    int failCount = 0;

    try {
      Connection conn = SharedApplication.getInstance().getROConnection();
      conn.close();
    } catch (Exception e) {
      throw new Exception("Oops, a bad pwd, or lack of network access to the db?", e);
    }

    if (singleModelPath != null) {
      File file = new File(singleModelPath);
      if (file.exists()) {

        String idString = singleModelPath.substring(0, singleModelPath.lastIndexOf('.'));
        idString = idString.substring(idString.lastIndexOf(File.separatorChar) + 1);
        Long id = null;

        try {
          id = Long.parseLong(idString);
        } catch (Exception e) {
          idString =
              prompt(
                  "Couldn't figure out the model number (expecting it as the file name).  What is the model number?");
          id = Long.parseLong(idString);
        }

        boolean pass = true;
        pass =
            pass & testSingleModelDataQuality(id); // Test 1 (already split off as failable tests)
        pass = pass & testSingleMmodel(file.toURL(), id, logShouldIncludeDetails); // Test 2
        pass = pass & testSingleModelErrorEstimates(id); // Test 3
        if (!pass) failCount++;
      } else {
        throw new Exception("The specified model path does not exist!");
      }

    } else {

      int modelCount = 0;
      for (long id = firstModelId; id <= lastModelId; id++) {

        String filePath = activeModelDirectory + id + ".txt";
        File file = new File(filePath);

        if (file.exists()) {
          modelCount++;

          boolean pass = true;
          pass = pass & testSingleModelDataQuality(id);
          pass = pass & testSingleMmodel(file.toURL(), id, logShouldIncludeDetails);

          if (!pass) failCount++;
        }
      }

      if (modelCount == 0) {
        throw new Exception(
            "The specified directory does not contain any models within the first & last model IDs!");
      }
    }

    if (failCount == 0) {
      log.debug("+ + + + + EVERYTHING LOOKS GREAT! + + + + +");
    } else {
      log.error(
          "- - - - - AT LEAST ONE MODEL FAILED VALIDATION.  PLEASE REVIEW THE LOG MESSAGES TO FIND THE VALIDATION ERRORS. + + + + +");
    }
  }
 public void beforeEachTest() {
   SharedApplication.getInstance().clearAllCaches();
 }