public static final List<RemoteFormDefinition> retrieveAvailableFormsFromServer(
ServerConnectionInfo serverInfo, TerminationFuture terminationFuture)
throws XmlDocumentFetchException, ParsingException {
AggregateUtils.DocumentFetchResult result = fetchFormList(serverInfo, true, terminationFuture);
List<RemoteFormDefinition> formDefs =
XmlManipulationUtils.parseFormListResponse(result.isOpenRosaResponse, result.doc);
return formDefs;
}
private String downloadManifestAndMediaFiles(File mediaDir, FormStatus fs) {
RemoteFormDefinition fd = (RemoteFormDefinition) fs.getFormDefinition();
if (fd.getManifestUrl() == null) return null;
fs.setStatusString("Fetching form manifest", true);
EventBus.publish(new FormStatusEvent(fs));
List<MediaFile> files = new ArrayList<MediaFile>();
AggregateUtils.DocumentFetchResult result;
try {
DocumentDescription formManifestDescription =
new DocumentDescription(
"Fetch of manifest failed. Detailed reason: ",
"Fetch of manifest failed ",
"form manifest",
terminationFuture);
result =
AggregateUtils.getXmlDocument(
fd.getManifestUrl(), serverInfo, false, formManifestDescription, null);
} catch (XmlDocumentFetchException e) {
return e.getMessage();
}
try {
files = XmlManipulationUtils.parseFormManifestResponse(result.isOpenRosaResponse, result.doc);
} catch (ParsingException e) {
return e.getMessage();
}
// OK we now have the full set of files to download...
logger.info("Downloading " + files.size() + " media files.");
int mCount = 0;
if (files.size() > 0) {
for (MediaFile m : files) {
++mCount;
fs.setStatusString(
String.format(" (getting %1$d of %2$d media files)", mCount, files.size()), true);
EventBus.publish(new FormStatusEvent(fs));
try {
downloadMediaFileIfChanged(mediaDir, m, fs);
} catch (Exception e) {
return e.getLocalizedMessage();
}
}
}
return null;
}
public static DecryptOutcome decryptAndValidateSubmission(
Document doc, PrivateKey rsaPrivateKey, File instanceDir, File unEncryptedDir)
throws ParsingException, FileSystemException, CryptoException {
Element rootElement = doc.getRootElement();
String base64EncryptedSymmetricKey;
String instanceIdMetadata = null;
List<String> mediaNames = new ArrayList<String>();
String encryptedSubmissionFile;
String base64EncryptedElementSignature;
{
Element base64Key = null;
Element base64Signature = null;
Element encryptedXml = null;
for (int i = 0; i < rootElement.getChildCount(); ++i) {
if (rootElement.getType(i) == Node.ELEMENT) {
Element child = rootElement.getElement(i);
String name = child.getName();
if (name.equals("base64EncryptedKey")) {
base64Key = child;
} else if (name.equals("base64EncryptedElementSignature")) {
base64Signature = child;
} else if (name.equals("encryptedXmlFile")) {
encryptedXml = child;
} else if (name.equals("media")) {
Element media = child;
for (int j = 0; j < media.getChildCount(); ++j) {
if (media.getType(j) == Node.ELEMENT) {
Element mediaChild = media.getElement(j);
String mediaFileElementName = mediaChild.getName();
if (mediaFileElementName.equals("file")) {
String mediaName = XFormParser.getXMLText(mediaChild, true);
if (mediaName == null || mediaName.length() == 0) {
mediaNames.add(null);
} else {
mediaNames.add(mediaName);
}
}
}
}
}
}
}
// verify base64Key
if (base64Key == null) {
throw new ParsingException("Missing base64EncryptedKey element in encrypted form.");
}
base64EncryptedSymmetricKey = XFormParser.getXMLText(base64Key, true);
// get instanceID out of OpenRosa meta block
instanceIdMetadata = XmlManipulationUtils.getOpenRosaInstanceId(rootElement);
if (instanceIdMetadata == null) {
throw new ParsingException("Missing instanceID within meta block of encrypted form.");
}
// get submission filename
if (encryptedXml == null) {
throw new ParsingException("Missing encryptedXmlFile element in encrypted form.");
}
encryptedSubmissionFile = XFormParser.getXMLText(encryptedXml, true);
if (base64Signature == null) {
throw new ParsingException(
"Missing base64EncryptedElementSignature element in encrypted form.");
}
base64EncryptedElementSignature = XFormParser.getXMLText(base64Signature, true);
}
if (instanceIdMetadata == null
|| base64EncryptedSymmetricKey == null
|| base64EncryptedElementSignature == null
|| encryptedSubmissionFile == null) {
throw new ParsingException("Missing one or more required elements of encrypted form.");
}
FormInstanceMetadata fim;
try {
fim = XmlManipulationUtils.getFormInstanceMetadata(rootElement);
} catch (ParsingException e) {
e.printStackTrace();
throw new ParsingException(
"Unable to extract form instance medatadata from submission manifest. Cause: "
+ e.toString());
}
if (!instanceIdMetadata.equals(fim.instanceId)) {
throw new ParsingException(
"InstanceID within metadata does not match that on top level element.");
}
boolean isValidated =
FileSystemUtils.decryptSubmissionFiles(
base64EncryptedSymmetricKey,
fim,
mediaNames,
encryptedSubmissionFile,
base64EncryptedElementSignature,
rsaPrivateKey,
instanceDir,
unEncryptedDir);
// and change doc to be the decrypted submission document
File decryptedSubmission = new File(unEncryptedDir, "submission.xml");
doc = XmlManipulationUtils.parseXml(decryptedSubmission);
if (doc == null) {
return null;
}
// verify that the metadata matches between the manifest and the submission
rootElement = doc.getRootElement();
FormInstanceMetadata sim = XmlManipulationUtils.getFormInstanceMetadata(rootElement);
if (!fim.xparam.equals(sim.xparam)) {
throw new ParsingException(
"FormId or version in decrypted submission does not match that in manifest!");
}
if (!fim.instanceId.equals(sim.instanceId)) {
throw new ParsingException(
"InstanceId in decrypted submission does not match that in manifest!");
}
return new DecryptOutcome(doc, isValidated);
}
private static boolean decryptSubmissionFiles(
String base64EncryptedSymmetricKey,
FormInstanceMetadata fim,
List<String> mediaNames,
String encryptedSubmissionFile,
String base64EncryptedElementSignature,
PrivateKey rsaPrivateKey,
File instanceDir,
File unencryptedDir)
throws FileSystemException, CryptoException, ParsingException {
EncryptionInformation ei =
new EncryptionInformation(base64EncryptedSymmetricKey, fim.instanceId, rsaPrivateKey);
byte[] elementDigest;
try {
// construct the base64-encoded RSA-encrypted symmetric key
Cipher pkCipher;
pkCipher = Cipher.getInstance(ASYMMETRIC_ALGORITHM);
// extract digest
pkCipher.init(Cipher.DECRYPT_MODE, rsaPrivateKey);
byte[] encryptedElementSignature = Base64.decodeBase64(base64EncryptedElementSignature);
elementDigest = pkCipher.doFinal(encryptedElementSignature);
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
throw new CryptoException(
"Error decrypting base64EncryptedElementSignature Cause: " + e.toString());
} catch (NoSuchPaddingException e) {
e.printStackTrace();
throw new CryptoException(
"Error decrypting base64EncryptedElementSignature Cause: " + e.toString());
} catch (InvalidKeyException e) {
e.printStackTrace();
throw new CryptoException(
"Error decrypting base64EncryptedElementSignature Cause: " + e.toString());
} catch (IllegalBlockSizeException e) {
e.printStackTrace();
throw new CryptoException(
"Error decrypting base64EncryptedElementSignature Cause: " + e.toString());
} catch (BadPaddingException e) {
e.printStackTrace();
throw new CryptoException(
"Error decrypting base64EncryptedElementSignature Cause: " + e.toString());
}
// NOTE: will decrypt only the files in the media list, plus the encryptedSubmissionFile
File[] allFiles = instanceDir.listFiles();
List<File> filesToProcess = new ArrayList<File>();
for (File f : allFiles) {
if (mediaNames.contains(f.getName())) {
filesToProcess.add(f);
} else if (encryptedSubmissionFile.equals(f.getName())) {
filesToProcess.add(f);
}
}
// should have all media files plus one submission.xml.enc file
if (filesToProcess.size() != mediaNames.size() + 1) {
// figure out what we're missing...
int lostFileCount = 0;
List<String> missing = new ArrayList<String>();
for (String name : mediaNames) {
if (name == null) {
// this was lost due to an pre-ODK Aggregate 1.4.5 mark-as-complete action
++lostFileCount;
continue;
}
File f = new File(instanceDir, name);
if (!filesToProcess.contains(f)) {
missing.add(name);
}
}
StringBuilder b = new StringBuilder();
for (String name : missing) {
b.append(" ").append(name);
}
if (!filesToProcess.contains(new File(instanceDir, encryptedSubmissionFile))) {
b.append(" ").append(encryptedSubmissionFile);
throw new FileSystemException(
"Error decrypting: " + instanceDir.getName() + " Missing files:" + b.toString());
} else {
// ignore the fact that we don't have the lost files
if (filesToProcess.size() + lostFileCount != mediaNames.size() + 1) {
throw new FileSystemException(
"Error decrypting: " + instanceDir.getName() + " Missing files:" + b.toString());
}
}
}
// decrypt the media files IN ORDER.
for (String mediaName : mediaNames) {
String displayedName = (mediaName == null) ? "<missing .enc file>" : mediaName;
File f = (mediaName == null) ? null : new File(instanceDir, mediaName);
try {
decryptFile(ei, f, unencryptedDir);
} catch (InvalidKeyException e) {
e.printStackTrace();
throw new CryptoException("Error decrypting:" + displayedName + " Cause: " + e.toString());
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
throw new CryptoException("Error decrypting:" + displayedName + " Cause: " + e.toString());
} catch (InvalidAlgorithmParameterException e) {
e.printStackTrace();
throw new CryptoException("Error decrypting:" + displayedName + " Cause: " + e.toString());
} catch (NoSuchPaddingException e) {
e.printStackTrace();
throw new CryptoException("Error decrypting:" + displayedName + " Cause: " + e.toString());
} catch (IOException e) {
e.printStackTrace();
throw new FileSystemException(
"Error decrypting:" + displayedName + " Cause: " + e.toString());
}
}
// decrypt the submission file
File f = new File(instanceDir, encryptedSubmissionFile);
try {
decryptFile(ei, f, unencryptedDir);
} catch (InvalidKeyException e) {
e.printStackTrace();
throw new CryptoException("Error decrypting:" + f.getName() + " Cause: " + e.toString());
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
throw new CryptoException("Error decrypting:" + f.getName() + " Cause: " + e.toString());
} catch (InvalidAlgorithmParameterException e) {
e.printStackTrace();
throw new CryptoException("Error decrypting:" + f.getName() + " Cause: " + e.toString());
} catch (NoSuchPaddingException e) {
e.printStackTrace();
throw new CryptoException("Error decrypting:" + f.getName() + " Cause: " + e.toString());
} catch (IOException e) {
e.printStackTrace();
throw new FileSystemException("Error decrypting:" + f.getName() + " Cause: " + e.toString());
}
// get the FIM for the decrypted submission file
File submissionFile =
new File(
unencryptedDir,
encryptedSubmissionFile.substring(0, encryptedSubmissionFile.lastIndexOf(".enc")));
FormInstanceMetadata submissionFim;
try {
Document subDoc = XmlManipulationUtils.parseXml(submissionFile);
submissionFim = XmlManipulationUtils.getFormInstanceMetadata(subDoc.getRootElement());
} catch (ParsingException e) {
e.printStackTrace();
throw new FileSystemException(
"Error decrypting: " + submissionFile.getName() + " Cause: " + e.toString());
} catch (FileSystemException e) {
e.printStackTrace();
throw new FileSystemException(
"Error decrypting: " + submissionFile.getName() + " Cause: " + e.getMessage());
}
boolean same = submissionFim.xparam.formId.equals(fim.xparam.formId);
if (!same) {
throw new FileSystemException(
"Error decrypting:"
+ unencryptedDir.getName()
+ " Cause: form instance metadata differs from that in manifest");
}
// Construct the element signature string
StringBuilder b = new StringBuilder();
appendElementSignatureSource(b, fim.xparam.formId);
if (fim.xparam.modelVersion != null) {
appendElementSignatureSource(b, Long.toString(fim.xparam.modelVersion));
}
appendElementSignatureSource(b, base64EncryptedSymmetricKey);
appendElementSignatureSource(b, fim.instanceId);
boolean missingFile = false;
for (String encFilename : mediaNames) {
if (encFilename == null) {
missingFile = true;
continue;
}
File decryptedFile =
new File(unencryptedDir, encFilename.substring(0, encFilename.lastIndexOf(".enc")));
if (decryptedFile.getName().endsWith(".missing")) {
// this is a missing file -- we will not be able to
// confirm the signature of the submission.
missingFile = true;
continue;
}
String md5 = FileSystemUtils.getMd5Hash(decryptedFile);
appendElementSignatureSource(b, decryptedFile.getName() + "::" + md5);
}
String md5 = FileSystemUtils.getMd5Hash(submissionFile);
appendElementSignatureSource(b, submissionFile.getName() + "::" + md5);
// compute the digest of the element signature string
byte[] messageDigest;
try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(b.toString().getBytes("UTF-8"));
messageDigest = md.digest();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
throw new CryptoException("Error computing xml signature Cause: " + e.toString());
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
throw new CryptoException("Error computing xml signature Cause: " + e.toString());
}
same = true;
for (int i = 0; i < messageDigest.length; ++i) {
if (messageDigest[i] != elementDigest[i]) {
same = false;
break;
}
}
return same;
}
private void downloadSubmission(
File formInstancesDir,
DatabaseUtils formDatabase,
BriefcaseFormDefinition lfd,
FormStatus fs,
String uri)
throws Exception {
if (formDatabase.hasRecordedInstance(uri) != null) {
logger.info("already present - skipping fetch: " + uri);
return;
}
String formId = lfd.getSubmissionKey(uri);
if (isCancelled()) {
fs.setStatusString("aborting fetch of submission...", true);
EventBus.publish(new FormStatusEvent(fs));
throw new SubmissionDownloadException("Transfer cancelled by user.");
}
String baseUrl = serverInfo.getUrl() + "/view/downloadSubmission";
Map<String, String> params = new HashMap<String, String>();
params.put("formId", formId);
String fullUrl = WebUtils.createLinkWithProperties(baseUrl, params);
AggregateUtils.DocumentFetchResult result;
try {
DocumentDescription submissionDescription =
new DocumentDescription(
"Fetch of a submission failed. Detailed error: ",
"Fetch of a submission failed.",
"submission",
terminationFuture);
result =
AggregateUtils.getXmlDocument(fullUrl, serverInfo, false, submissionDescription, null);
} catch (XmlDocumentFetchException e) {
throw new SubmissionDownloadException(e.getMessage());
}
// and parse the document...
SubmissionManifest submissionManifest;
try {
submissionManifest = XmlManipulationUtils.parseDownloadSubmissionResponse(result.doc);
} catch (ParsingException e) {
throw new SubmissionDownloadException(e.getMessage());
}
String msg = "Fetched instanceID=" + submissionManifest.instanceID;
logger.info(msg);
if (FileSystemUtils.hasFormSubmissionDirectory(
formInstancesDir, submissionManifest.instanceID)) {
// create instance directory...
File instanceDir =
FileSystemUtils.assertFormSubmissionDirectory(
formInstancesDir, submissionManifest.instanceID);
// fetch attachments
for (MediaFile m : submissionManifest.attachmentList) {
downloadMediaFileIfChanged(instanceDir, m, fs);
}
// write submission file -- we rely on instanceId being unique...
File submissionFile = new File(instanceDir, "submission.xml");
OutputStreamWriter fo = new OutputStreamWriter(new FileOutputStream(submissionFile), "UTF-8");
fo.write(submissionManifest.submissionXml);
fo.close();
// if we get here and it was a legacy server (0.9.x), we don't
// actually know whether the submission was complete. Otherwise,
// if we get here, we know that this is a completed submission
// (because it was in /view/submissionList) and that we safely
// copied it into the storage area (because we didn't get any
// exceptions).
if (serverInfo.isOpenRosaServer()) {
formDatabase.assertRecordedInstanceDirectory(uri, instanceDir);
}
} else {
// create instance directory...
File instanceDir =
FileSystemUtils.assertFormSubmissionDirectory(
formInstancesDir, submissionManifest.instanceID);
// fetch attachments
for (MediaFile m : submissionManifest.attachmentList) {
downloadMediaFileIfChanged(instanceDir, m, fs);
}
// write submission file
File submissionFile = new File(instanceDir, "submission.xml");
OutputStreamWriter fo = new OutputStreamWriter(new FileOutputStream(submissionFile), "UTF-8");
fo.write(submissionManifest.submissionXml);
fo.close();
// if we get here and it was a legacy server (0.9.x), we don't
// actually know whether the submission was complete. Otherwise,
// if we get here, we know that this is a completed submission
// (because it was in /view/submissionList) and that we safely
// copied it into the storage area (because we didn't get any
// exceptions).
if (serverInfo.isOpenRosaServer()) {
formDatabase.assertRecordedInstanceDirectory(uri, instanceDir);
}
}
}
private boolean downloadAllSubmissionsForForm(
File formInstancesDir,
DatabaseUtils formDatabase,
BriefcaseFormDefinition lfd,
FormStatus fs) {
boolean allSuccessful = true;
RemoteFormDefinition fd = (RemoteFormDefinition) fs.getFormDefinition();
int count = 1;
String baseUrl = serverInfo.getUrl() + "/view/submissionList";
String oldWebsafeCursorString = "not-empty";
String websafeCursorString = "";
for (; !oldWebsafeCursorString.equals(websafeCursorString); ) {
if (isCancelled()) {
fs.setStatusString("aborting fetching submissions...", true);
EventBus.publish(new FormStatusEvent(fs));
return false;
}
fs.setStatusString("retrieving next chunk of instances from server...", true);
EventBus.publish(new FormStatusEvent(fs));
Map<String, String> params = new HashMap<String, String>();
params.put("numEntries", Integer.toString(MAX_ENTRIES));
params.put("formId", fd.getFormId());
params.put("cursor", websafeCursorString);
String fullUrl = WebUtils.createLinkWithProperties(baseUrl, params);
oldWebsafeCursorString = websafeCursorString; // remember what we had...
AggregateUtils.DocumentFetchResult result;
try {
DocumentDescription submissionChunkDescription =
new DocumentDescription(
"Fetch of submission download chunk failed. Detailed error: ",
"Fetch of submission download chunk failed.",
"submission download chunk",
terminationFuture);
result =
AggregateUtils.getXmlDocument(
fullUrl, serverInfo, false, submissionChunkDescription, null);
} catch (XmlDocumentFetchException e) {
fs.setStatusString(
"NOT ALL SUBMISSIONS RETRIEVED: Error fetching list of submissions: " + e.getMessage(),
false);
EventBus.publish(new FormStatusEvent(fs));
return false;
}
SubmissionDownloadChunk chunk;
try {
chunk = XmlManipulationUtils.parseSubmissionDownloadListResponse(result.doc);
} catch (ParsingException e) {
fs.setStatusString(
"NOT ALL SUBMISSIONS RETRIEVED: Error parsing the list of submissions: "
+ e.getMessage(),
false);
EventBus.publish(new FormStatusEvent(fs));
return false;
}
websafeCursorString = chunk.websafeCursorString;
for (String uri : chunk.uriList) {
if (isCancelled()) {
fs.setStatusString("aborting fetching submissions...", true);
EventBus.publish(new FormStatusEvent(fs));
return false;
}
try {
fs.setStatusString("fetching instance " + count++ + " ...", true);
EventBus.publish(new FormStatusEvent(fs));
downloadSubmission(formInstancesDir, formDatabase, lfd, fs, uri);
} catch (Exception e) {
e.printStackTrace();
allSuccessful = false;
fs.setStatusString(
"SUBMISSION NOT RETRIEVED: Error fetching submission uri: "
+ uri
+ " details: "
+ e.getMessage(),
false);
EventBus.publish(new FormStatusEvent(fs));
// but try to get the next one...
}
}
}
return allSuccessful;
}