/**
* Runs QA checks against the data.
*
* @param modelId
* @return
* @throws Exception
*/
public boolean testSingleModelDataQuality(Long modelId) throws Exception {
Connection conn = SharedApplication.getInstance().getROConnection();
// Get list of queries in properties file
Properties props = new Properties();
String path = this.getClass().getName().replace('.', '/') + ".properties";
InputStream ins = Thread.currentThread().getContextClassLoader().getResourceAsStream(path);
if (ins == null) {
ins = Action.class.getResourceAsStream(path);
}
props.load(ins);
Enumeration<?> elements = props.propertyNames();
boolean passed = true;
try {
while (elements.hasMoreElements()) {
String queryName = elements.nextElement().toString();
log.debug("Running data validation test " + queryName + "'...");
passed = passed & testSingleModelDataQuality(modelId, queryName, conn);
}
} finally {
SharedApplication.closeConnection(conn, null);
}
if (passed) {
log.debug("++++++++ Model #" + modelId + " PASSED all data validation tests ++++++++");
} else {
// The fail message is printed in the deligated method, so no need to reprint here.
}
return passed;
}
@Test
public void test9680_WhichHasAOneThousanthFraction() throws Exception {
String TEST_REACH_CLIENT_ID = "9680";
Long TEST_REACH_SYSTEM_ID = 9680L;
Long ONLY_REACH_UPSTREAM_OF_TEST_REACH_ID = 9681L;
// Create terminal reaches and put in cache
List<String> targetList = new ArrayList<String>();
targetList.add(TEST_REACH_CLIENT_ID);
TerminalReaches targets = new TerminalReaches(TEST_MODEL_ID, targetList);
ConfiguredCache.TerminalReaches.put(targets.getId(), targets);
// Load predict data and model incremental areas - used for comparison
PredictData pd = SharedApplication.getInstance().getPredictData(TEST_MODEL_ID);
DataTable incrementalReachAreas =
SharedApplication.getInstance()
.getCatchmentAreas(new UnitAreaRequest(TEST_MODEL_ID, AreaType.INCREMENTAL));
// Stats on the test reach
int testReachRowNumber = pd.getRowForReachID(TEST_REACH_SYSTEM_ID);
Double testReachFrac = pd.getTopo().getDouble(testReachRowNumber, PredictData.TOPO_FRAC_COL);
Double testReachFractionedWatershedArea =
new CalcFractionedWatershedArea(
new FractionedWatershedAreaRequest(
new ReachID(TEST_MODEL_ID, TEST_REACH_SYSTEM_ID)))
.run();
Double testReachUnfractionedWatershedArea =
new CalcFractionedWatershedArea(
new FractionedWatershedAreaRequest(
new ReachID(TEST_MODEL_ID, TEST_REACH_SYSTEM_ID), false, false, true))
.run();
Double testReachIncrementalArea = incrementalReachAreas.getDouble(testReachRowNumber, 1);
// load stats on the only reach immediately upstream of the test reach
Double upstreamReachFractionedWatershedArea =
new CalcFractionedWatershedArea(
new FractionedWatershedAreaRequest(
new ReachID(TEST_MODEL_ID, ONLY_REACH_UPSTREAM_OF_TEST_REACH_ID)))
.run();
// Test the assumptions about the basic (non-table form) of the area data
assertTrue(
Math.abs(testReachFractionedWatershedArea - testReachUnfractionedWatershedArea)
> 1d); // These should be different values
assertTrue(Math.abs(testReachFractionedWatershedArea - testReachIncrementalArea) > 1d);
assertEquals(
testReachFrac * upstreamReachFractionedWatershedArea + testReachIncrementalArea,
testReachFractionedWatershedArea,
.0001d);
// Do the same calculation via the Table version of the action
CalcFractionedWatershedAreaTable action = new CalcFractionedWatershedAreaTable(targets.getId());
ColumnData data = action.run();
assertEquals(testReachFractionedWatershedArea, data.getDouble(testReachRowNumber), .0001d);
}
@Before
public void setup() {
// Set up contexts
AdjustmentGroups adjustments = new AdjustmentGroups(TEST_MODEL_ID);
predictData = SharedApplication.getInstance().getPredictData(TEST_MODEL_ID);
predictResult = SharedApplication.getInstance().getPredictResult(adjustments);
UnitAreaRequest watershedAreaReq =
new UnitAreaRequest(TEST_MODEL_ID, AreaType.TOTAL_CONTRIBUTING);
DataTable watershedAreaTable =
SharedApplication.getInstance().getCatchmentAreas(watershedAreaReq);
watershedAreaColumn = new ColumnDataFromTable(watershedAreaTable, 1);
}
@Override
public ReachRowValueMap createEntry(Object terminalReaches) throws Exception {
TerminalReaches targets = (TerminalReaches) terminalReaches;
PredictData predictData =
SharedApplication.getInstance().getPredictData(new Long(targets.getModelID()));
Set<String> clientTargetReachIdSet = targets.getReachIdsAsSet();
List<Long> targetReachIdList =
SharedApplication.getInstance()
.getReachFullIdAsLong(targets.getModelID(), clientTargetReachIdSet);
HashSet<Long> targetReachIdSet = new HashSet<Long>();
targetReachIdSet.addAll(targetReachIdList);
CalcDeliveryFractionMap action = new CalcDeliveryFractionMap(predictData, targetReachIdSet);
ReachRowValueMap delFrac = action.run();
return delFrac;
}
@Override
protected void initFields() throws Exception {
File dataDir = getDataDirectory();
if (!dataDir.exists()) {
Files.createDirectories(dataDir.toPath());
}
dataColumn = context.getDataColumn().getColumnData();
columnIndex =
SharedApplication.getInstance().getPredictData(context.getModelID()).getTopo().getIndex();
outputFile =
new File(
dataDir,
NamingConventions.convertContextIdToXMLSafeName(
context.getModelID().intValue(), context.getId())
+ ".dbf");
outputFile.createNewFile();
DataSeriesType type = context.getAnalysis().getDataSeries();
// grab the delivery fraction map if this is a delivery data series.
// This is used to weed out the reaches that are not upstream of the
// user selected terminal reaches.
if (type.isDeliveryRequired()) {
TerminalReaches tReaches = context.getTerminalReaches();
assert (tReaches != null) : "client should not submit a delivery request without reaches";
reachRowValueMap = SharedApplication.getInstance().getDeliveryFractionMap(tReaches);
if (reachRowValueMap == null) {
throw new Exception("Unable to find or calculate the delivery fraction map");
}
} else {
reachRowValueMap = null;
}
}
/**
* Tests all the source / total / incremental combinations for reasonable std error estimate
* values.
*
* @param modelId
* @return
* @throws Exception
*/
public boolean testSingleModelErrorEstimates(Long modelId) throws Exception {
PredictData pd = SharedApplication.getInstance().getPredictData(modelId);
int failCount = 0;
for (int srcIndex = 0; srcIndex < pd.getSrcMetadata().getRowCount(); srcIndex++) {
int srcId = pd.getSourceIdForSourceIndex(srcIndex);
failCount += testSingleModelErrorEstimates(modelId, srcId, false);
failCount += testSingleModelErrorEstimates(modelId, srcId, true);
}
// Add the total values (ie, all sources togeteher
failCount += testSingleModelErrorEstimates(modelId, null, false);
failCount += testSingleModelErrorEstimates(modelId, null, true);
System.out.println("Grand Total Error est failures for model " + modelId + ": " + failCount);
return failCount == 0;
}
protected File getDataDirectory() {
File dDir;
if (this.dataDirectory != null) {
dDir = this.dataDirectory;
} else {
DynamicReadOnlyProperties props = SharedApplication.getInstance().getConfiguration();
String fallbackDataDirectory =
System.getProperty("user.home")
+ File.separatorChar
+ "sparrow"
+ File.separatorChar
+ "data";
String sparrowDataDirectory = props.getProperty(DATA_EXPORT_DIRECTORY, fallbackDataDirectory);
dDir = new File(sparrowDataDirectory);
this.dataDirectory = dDir;
}
return dDir;
}
@Test
public void testTargetWith4UpstreamReaches() throws Exception {
Long TEST_ROW_SYSTEM_ID = 9687L;
String TEST_ROW_CLIENT_ID = "9687";
List<String> targetList = new ArrayList<String>();
targetList.add(TEST_ROW_CLIENT_ID);
TerminalReaches targets = new TerminalReaches(TEST_MODEL_ID, targetList);
ConfiguredCache.TerminalReaches.put(targets.getId(), targets);
PredictData pd = SharedApplication.getInstance().getPredictData(TEST_MODEL_ID);
Double watershedArea =
new CalcFractionedWatershedArea(
new FractionedWatershedAreaRequest(new ReachID(TEST_MODEL_ID, TEST_ROW_SYSTEM_ID)))
.run();
int rowCount = pd.getTopo().getRowCount();
int rowNumber = pd.getRowForReachID(TEST_ROW_SYSTEM_ID);
CalcFractionedWatershedAreaTable action = new CalcFractionedWatershedAreaTable(targets.getId());
ColumnData data = action.run();
assertEquals(rowCount, data.getRowCount().intValue());
assertNotNull(data.getDouble(rowNumber));
for (int row = 0; row < rowCount; row++) {
if (row == rowNumber) {
assertEquals(
"Row " + row + " should be " + watershedArea,
watershedArea,
data.getDouble(row),
.000000001d);
} else {
assertNull("Row " + row + " should be null", data.getDouble(row));
}
}
}
/**
* Returns a sorted list of identifiers for reaches that fall within the bounding box defined by
* the specified arguments. A reach is considered to fall within the bounding box if any part of
* its geometry falls within the bounding box.
*
* @return A list of identifiers for reaches that fall within the defined bounding box.
* @throws Exception
*/
private Long[] getResults(Long modelId, String bounds) throws Exception {
ModelBBox modelBBox = new ModelBBox(modelId, bounds);
Long[] ids = SharedApplication.getInstance().getReachesInBBox(modelBBox);
return ids;
}
/**
* Tests a single model / source / total/incremental combination for a reasonalbe std error
* estimate.
*
* @param modelId
* @param sourceId
* @param total
* @return
* @throws Exception
*/
public int testSingleModelErrorEstimates(Long modelId, Integer sourceId, boolean total)
throws Exception {
AdjustmentGroups noAdjustments = new AdjustmentGroups(modelId);
int failCount = 0;
String incTot = (total) ? "Total" : "Incremental";
// Build the error data series
DataSeriesType errSeries =
(total)
? DataSeriesType.total_std_error_estimate
: DataSeriesType.incremental_std_error_estimate;
BasicAnalysis errAnalysis = new BasicAnalysis(errSeries, sourceId, null, null);
PredictionContext errContext =
new PredictionContext(modelId, noAdjustments, errAnalysis, null, null, null);
SparrowColumnSpecifier errColumn =
SharedApplication.getInstance().getAnalysisResult(errContext);
///
// Build the related standard series
DataSeriesType stdSeries = (total) ? DataSeriesType.total : DataSeriesType.incremental;
BasicAnalysis stdAnalysis = new BasicAnalysis(stdSeries, sourceId, null, null);
PredictionContext stdContext =
new PredictionContext(modelId, noAdjustments, stdAnalysis, null, null, null);
SparrowColumnSpecifier stdColumn =
SharedApplication.getInstance().getAnalysisResult(stdContext);
///
// Do the comparison
for (int row = 0; row < stdColumn.getRowCount(); row++) {
Double std = stdColumn.getDouble(row);
Double err = errColumn.getDouble(row);
if (err == null && std < 1d) {
// I think its OK to have no err estimate if the predicted value is very small
// System.out.println("Predict: " + std + " Err: null");
} else if (err != null && std <= 1d && err <= 20) {
// An error of 10 is pretty small, though it may be many times larger than the predicted
// value
// System.out.println("Predict: " + std + " Err: " + err);
} else if (err != null && err <= (std * 20)) {
// System.out.println("Predict: " + std + " Err: " + err);
} else {
System.out.println(
"Model "
+ modelId
+ ", Src:"
+ sourceId
+ " "
+ incTot
+ ": Suspicious Std Error. Value: "
+ std
+ " Err: "
+ err);
failCount++;
}
}
System.out.println(
"Model "
+ modelId
+ ", Src:"
+ sourceId
+ " "
+ incTot
+ ": Total Error est failures: "
+ failCount);
return failCount;
}
public boolean testSingleMmodel(URL predictFile, Long modelId, boolean logWithDetails)
throws Exception {
beforeEachTest();
boolean pass = false;
DataTable t = loadTextPredict(predictFile);
if (t != null) {
AdjustmentGroups ags = new AdjustmentGroups(modelId);
PredictResult prs = SharedApplication.getInstance().getPredictResult(ags);
PredictData pd = SharedApplication.getInstance().getPredictData(modelId);
log.setLevel(Level.FATAL); // Turn off logging
int noDecayFailures = testComparison(t, prs, pd, false, modelId, logWithDetails);
int decayFailures = testComparison(t, prs, pd, true, modelId, logWithDetails);
log.setLevel(Level.DEBUG); // Turn back on
if (decayFailures < noDecayFailures) {
if (decayFailures == 0) {
log.debug(
"++++++++ Model #" + modelId + " PASSED using DECAYED incremental values +++++++++");
pass = true;
} else {
log.debug(
"-------- Model #"
+ modelId
+ " FAILED using DECAYED incremental values. Details: --------");
testComparison(t, prs, pd, true, modelId, logWithDetails);
}
} else if (noDecayFailures < decayFailures) {
if (noDecayFailures == 0) {
log.debug(
"++++++++ Model #"
+ modelId
+ " PASSED using NON-DECAYED incremental values +++++++++");
pass = true;
} else {
log.debug(
"-------- Model #"
+ modelId
+ " FAILED using NON-DECAYED incremental values. Details: --------");
testComparison(t, prs, pd, false, modelId, logWithDetails);
}
} else if (noDecayFailures == decayFailures) {
// Equal failures mean there is a row count or other type of error
log.debug(
"Hey, found these no decay fails: "
+ noDecayFailures
+ " and these decay fails: "
+ decayFailures);
log.debug(
"-------- Model #"
+ modelId
+ " FAILED. MAJOR ISSUE (no decay matches decay) - SEE DETAILS: --------");
testComparison(t, prs, pd, false, modelId, logWithDetails);
}
} else {
log.debug(
"-------- Model #"
+ modelId
+ " FAILED. MAJOR ISSUE (couldn't load file) - SEE DETAILS (above) --------");
}
return pass;
}
public void runTheTests() throws Exception {
int failCount = 0;
try {
Connection conn = SharedApplication.getInstance().getROConnection();
conn.close();
} catch (Exception e) {
throw new Exception("Oops, a bad pwd, or lack of network access to the db?", e);
}
if (singleModelPath != null) {
File file = new File(singleModelPath);
if (file.exists()) {
String idString = singleModelPath.substring(0, singleModelPath.lastIndexOf('.'));
idString = idString.substring(idString.lastIndexOf(File.separatorChar) + 1);
Long id = null;
try {
id = Long.parseLong(idString);
} catch (Exception e) {
idString =
prompt(
"Couldn't figure out the model number (expecting it as the file name). What is the model number?");
id = Long.parseLong(idString);
}
boolean pass = true;
pass =
pass & testSingleModelDataQuality(id); // Test 1 (already split off as failable tests)
pass = pass & testSingleMmodel(file.toURL(), id, logShouldIncludeDetails); // Test 2
pass = pass & testSingleModelErrorEstimates(id); // Test 3
if (!pass) failCount++;
} else {
throw new Exception("The specified model path does not exist!");
}
} else {
int modelCount = 0;
for (long id = firstModelId; id <= lastModelId; id++) {
String filePath = activeModelDirectory + id + ".txt";
File file = new File(filePath);
if (file.exists()) {
modelCount++;
boolean pass = true;
pass = pass & testSingleModelDataQuality(id);
pass = pass & testSingleMmodel(file.toURL(), id, logShouldIncludeDetails);
if (!pass) failCount++;
}
}
if (modelCount == 0) {
throw new Exception(
"The specified directory does not contain any models within the first & last model IDs!");
}
}
if (failCount == 0) {
log.debug("+ + + + + EVERYTHING LOOKS GREAT! + + + + +");
} else {
log.error(
"- - - - - AT LEAST ONE MODEL FAILED VALIDATION. PLEASE REVIEW THE LOG MESSAGES TO FIND THE VALIDATION ERRORS. + + + + +");
}
}
public void beforeEachTest() {
SharedApplication.getInstance().clearAllCaches();
}
