// the 'isPrivate' flag should be set to 'true' for information
// used by the scoring application but not to be revealed to participants
// to see 'public' annotations requires READ access in the Evaluation's
// access control list, as the participant has (see setUp(), above). To
// see 'private' annotatations requires READ_PRIVATE_SUBMISSION access,
// which the Evaluation admin has by default
private static void addAnnotation(
SubmissionStatus status, String key, String value, boolean isPrivate) {
if (value.length() > 499) value = value.substring(0, 499);
Annotations annotations = status.getAnnotations();
if (annotations == null) {
annotations = new Annotations();
status.setAnnotations(annotations);
}
List<StringAnnotation> sas = annotations.getStringAnnos();
if (sas == null) {
sas = new ArrayList<StringAnnotation>();
annotations.setStringAnnos(sas);
}
StringAnnotation matchingSa = null;
for (StringAnnotation existingSa : sas) {
if (existingSa.getKey().equals(key)) {
matchingSa = existingSa;
break;
}
}
if (matchingSa == null) {
StringAnnotation sa = new StringAnnotation();
sa.setIsPrivate(isPrivate);
sa.setKey(key);
sa.setValue(value);
sas.add(sa);
} else {
matchingSa.setIsPrivate(isPrivate);
matchingSa.setValue(value);
}
}
/**
* Note: There are two types of scoring, that in which each submission is scored alone and that in
* which the entire set of submissions is rescored whenever a new one arrives.
*
* @throws SynapseException
* @throws JSONObjectAdapterException
*/
public void execute() throws SynapseException, IOException, JSONObjectAdapterException {
long startTime = System.currentTimeMillis();
long total = Integer.MAX_VALUE;
int started = 0;
for (int offset = 0; offset < total; offset += PAGE_SIZE) {
PaginatedResults<SubmissionBundle> submissionPGs = null;
submissionPGs =
synapseAdmin.getAllSubmissionBundlesByStatus(
evaluation.getId(), SubmissionStatusEnum.RECEIVED, offset, PAGE_SIZE);
total = (int) submissionPGs.getTotalNumberOfResults();
List<SubmissionBundle> page = submissionPGs.getResults();
for (int i = 0; i < page.size(); i++) {
SubmissionBundle bundle = page.get(i);
Submission sub = bundle.getSubmission();
FileHandle fileHandle = getFileHandleFromEntityBundle(sub.getEntityBundleJSON());
SubmissionStatus status = bundle.getSubmissionStatus();
// Retrieve and run COUNT_QUERY. If > 0 then execute Docker image
String tableId = readStringAnnotation(status, TABLE_ID_ANNOTATION_NAME);
String countQuery = readStringAnnotation(status, COUNT_QUERY_ANNOTATION_NAME);
if (tableId != null && countQuery != null) {
int count = executeCountQuery(tableId, countQuery);
if (count == 0) {
System.out.println("Count query returned 0: " + countQuery);
continue; // nothing to process, so skip this submission
}
}
// if no tableId or no countQuery then run the image
// it's then the image's job to put this information in the annotations
try {
String dockerReference = getDockerReferenceFromFileHandle(fileHandle);
if (readStringAnnotation(status, DOCKER_REPOSITORY_ANNOTATION_NAME) == null)
addAnnotation(status, DOCKER_REPOSITORY_ANNOTATION_NAME, dockerReference, false);
status.setStatus(SubmissionStatusEnum.EVALUATION_IN_PROGRESS);
status = synapseAdmin.updateSubmissionStatus(status);
// we run the image if either (1) there are new rows to process of (2) it hasn't been run
// before
runOneImage(fileHandle, sub.getId());
// make sure that the Docker reference is in an annotation so it can be displayed
started++;
} catch (Exception e) {
status = synapseAdmin.getSubmissionStatus(sub.getId());
if (status.getStatus() != SubmissionStatusEnum.RECEIVED) {
status.setStatus(SubmissionStatusEnum.RECEIVED);
status = synapseAdmin.updateSubmissionStatus(status);
}
// TODO send failure notification to submitter
e.printStackTrace();
}
}
}
long delta = System.currentTimeMillis() - startTime;
System.out.println("Started " + started + " jobs. Elapsed time: " + formatInterval(delta));
}
private static String readStringAnnotation(SubmissionStatus status, String key) {
Annotations annotations = status.getAnnotations();
if (annotations == null) return null;
List<StringAnnotation> sas = annotations.getStringAnnos();
if (sas == null) return null;
for (StringAnnotation sa : sas) {
if (sa.getKey().equals(key)) return sa.getValue();
}
return null;
}
private static void removeAnnotation(SubmissionStatus status, String key) {
Annotations annotations = status.getAnnotations();
if (annotations == null) return;
List<StringAnnotation> sas = annotations.getStringAnnos();
if (sas == null) return;
for (StringAnnotation existingSa : sas) {
if (existingSa.getKey().equals(key)) {
sas.remove(existingSa);
}
}
}