public Integer getRunId(boolean random) throws Exception { String sql = "SELECT `rid`,`start_time`,`processed`,`error` " + "FROM `run` WHERE `processed` IS NULL AND `error` IS NULL AND error_message IS NULL " + "ORDER BY `start_time` ASC LIMIT 0, 100; "; JSONArray runJson = (JSONArray) apiconnector.freeQuery(sql).get("data"); int randomint = 0; if (random) { Random r = new Random(); randomint = Math.abs(r.nextInt()); } if (runJson.length() > 0) { int run_id = ((JSONArray) runJson.get(randomint % runJson.length())).getInt(0); return run_id; } else { return null; } }
public void evaluate(int run_id) throws Exception { Conversion.log("OK", "Process Run", "Start processing run: " + run_id); final Map<String, Integer> file_ids = new HashMap<String, Integer>(); final Task task; final DataSetDescription dataset; PredictionEvaluator predictionEvaluator; RunEvaluation runevaluation = new RunEvaluation(run_id); RunTrace trace = null; JSONArray runJson = (JSONArray) apiconnector .freeQuery("SELECT `task_id` FROM `run` WHERE `rid` = " + run_id) .get("data"); JSONArray filesJson = (JSONArray) apiconnector .freeQuery("SELECT `field`,`file_id` FROM `runfile` WHERE `source` = " + run_id) .get("data"); try { int task_id = ((JSONArray) runJson.get(0)).getInt(0); task = apiconnector.taskGet(task_id); Data_set source_data = TaskInformation.getSourceData(task); Estimation_procedure estimationprocedure = null; try { estimationprocedure = TaskInformation.getEstimationProcedure(task); } catch (Exception e) { } Integer dataset_id = source_data.getLabeled_data_set_id() != null ? source_data.getLabeled_data_set_id() : source_data.getData_set_id(); for (int i = 0; i < filesJson.length(); ++i) { String field = ((JSONArray) filesJson.get(i)).getString(0); int file_index = ((JSONArray) filesJson.get(i)).getInt(1); file_ids.put(field, file_index); } if (file_ids.get("description") == null) { runevaluation.setError("Run description file not present. "); File evaluationFile = Conversion.stringToTempFile( xstream.toXML(runevaluation), "run_" + run_id + "evaluations", "xml"); RunEvaluate re = apiconnector.runEvaluate(evaluationFile); Conversion.log("Error", "Process Run", "Run processed, but with error: " + re.getRun_id()); return; } if (file_ids.get("predictions") == null && file_ids.get("subgroups") == null) { // TODO: this is currently true, but later on we might have tasks that do // not require evaluations! runevaluation.setError("Required output files not present (e.g., arff predictions). "); File evaluationFile = Conversion.stringToTempFile( xstream.toXML(runevaluation), "run_" + run_id + "evaluations", "xml"); RunEvaluate re = apiconnector.runEvaluate(evaluationFile); Conversion.log("Error", "Process Run", "Run processed, but with error: " + re.getRun_id()); return; } if (file_ids.get("trace") != null) { trace = traceToXML(file_ids.get("trace"), task_id, run_id); } String description = OpenmlConnector.getStringFromUrl( apiconnector .getOpenmlFileUrl( file_ids.get("description"), "Run_" + run_id + "_description.xml") .toString()); Run run_description = (Run) xstream.fromXML(description); dataset = apiconnector.dataGet(dataset_id); Conversion.log("OK", "Process Run", "Start prediction evaluator. "); // TODO! no string comparisons, do something better String filename_prefix = "Run_" + run_id + "_"; if (task.getTask_type().equals("Supervised Data Stream Classification")) { predictionEvaluator = new EvaluateStreamPredictions( apiconnector.getOpenmlFileUrl(dataset.getFile_id(), dataset.getName()), apiconnector.getOpenmlFileUrl( file_ids.get("predictions"), filename_prefix + "predictions.arff"), source_data.getTarget_feature()); } else if (task.getTask_type().equals("Survival Analysis")) { predictionEvaluator = new EvaluateSurvivalAnalysisPredictions( task, apiconnector.getOpenmlFileUrl(dataset.getFile_id(), dataset.getName()), new URL(estimationprocedure.getData_splits_url()), apiconnector.getOpenmlFileUrl( file_ids.get("predictions"), filename_prefix + "predictions.arff")); } else if (task.getTask_type().equals("Subgroup Discovery")) { predictionEvaluator = new EvaluateSubgroups(); } else { predictionEvaluator = new EvaluateBatchPredictions( task, apiconnector.getOpenmlFileUrl(dataset.getFile_id(), dataset.getName()), new URL(estimationprocedure.getData_splits_url()), apiconnector.getOpenmlFileUrl( file_ids.get("predictions"), filename_prefix + "predictions.arff"), estimationprocedure .getType() .equals(EstimationProcedure.estimationProceduresTxt[6])); } runevaluation.addEvaluationMeasures(predictionEvaluator.getEvaluationScores()); if (run_description.getOutputEvaluation() != null) { Conversion.log( "OK", "Process Run", "Start consistency check with user defined measures. (x " + run_description.getOutputEvaluation().length + ")"); // TODO: This can be done so much faster ... String warningMessage = ""; boolean warningFound = false; for (EvaluationScore recorded : run_description.getOutputEvaluation()) { boolean foundSame = false; // important check: because of legacy (implementation_id), the flow id might be missing if (recorded.getFlow() != null && recorded.getFunction() != null) { for (EvaluationScore calculated : runevaluation.getEvaluation_scores()) { if (recorded.isSame(calculated)) { foundSame = true; if (recorded.sameValue(calculated) == false) { String offByStr = ""; try { double diff = Math.abs( Double.parseDouble(recorded.getValue()) - Double.parseDouble(calculated.getValue())); offByStr = " (off by " + diff + ")"; } catch (NumberFormatException nfe) { } warningMessage += "Inconsistent Evaluation score: " + recorded + offByStr; warningFound = true; } } } if (foundSame == false) { // give the record the correct sample size if (recorded.getSample() != null && recorded.getSample_size() == null) { recorded.setSample_size( predictionEvaluator .getPredictionCounter() .getShadowTypeSize( recorded.getRepeat(), recorded.getFold(), recorded.getSample())); } runevaluation.addEvaluationMeasure(recorded); } } } if (warningFound) runevaluation.setWarning(warningMessage); } else { Conversion.log("OK", "Process Run", "No local evaluation measures to compare to. "); } } catch (Exception e) { e.printStackTrace(); Conversion.log( "Warning", "Process Run", "Unexpected error, will proceed with upload process: " + e.getMessage()); runevaluation.setError(e.getMessage()); } Conversion.log("OK", "Process Run", "Start uploading results ... "); try { String runEvaluation = xstream.toXML(runevaluation); // System.out.println(runEvaluation); File evaluationFile = Conversion.stringToTempFile(runEvaluation, "run_" + run_id + "evaluations", "xml"); RunEvaluate re = apiconnector.runEvaluate(evaluationFile); if (trace != null) { String runTrace = xstream.toXML(trace); // System.out.println(runTrace); File traceFile = Conversion.stringToTempFile(runTrace, "run_" + run_id + "trace", "xml"); apiconnector.runTraceUpload(traceFile); } Conversion.log("OK", "Process Run", "Run processed: " + re.getRun_id()); } catch (Exception e) { Conversion.log("ERROR", "Process Run", "An error occured during API call: " + e.getMessage()); } }