protected HttpResponse sendHttpRequest(
String method, String url, HttpEntity entity, List<NameValuePair> qparams, CallingContext cc)
throws IOException {
HttpParams httpParams = new BasicHttpParams();
HttpConnectionParams.setConnectionTimeout(
httpParams, SOCKET_ESTABLISHMENT_TIMEOUT_MILLISECONDS);
HttpConnectionParams.setSoTimeout(httpParams, SERVICE_TIMEOUT_MILLISECONDS);
// setup client
HttpClientFactory factory = (HttpClientFactory) cc.getBean(BeanDefs.HTTP_CLIENT_FACTORY);
HttpClient client = factory.createHttpClient(httpParams);
// support redirecting to handle http: => https: transition
HttpClientParams.setRedirecting(httpParams, true);
// support authenticating
HttpClientParams.setAuthenticating(httpParams, true);
// redirect limit is set to some unreasonably high number
// resets of the socket cause a retry up to MAX_REDIRECTS times,
// so we should be careful not to set this too high...
httpParams.setParameter(ClientPNames.MAX_REDIRECTS, 32);
httpParams.setParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS, true);
// context holds authentication state machine, so it cannot be
// shared across independent activities.
HttpContext localContext = new BasicHttpContext();
localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);
localContext.setAttribute(ClientContext.CREDS_PROVIDER, credsProvider);
HttpUriRequest request = null;
if (entity == null && (POST.equals(method) || PATCH.equals(method) || PUT.equals(method))) {
throw new IllegalStateException("No body supplied for POST, PATCH or PUT request");
} else if (entity != null
&& !(POST.equals(method) || PATCH.equals(method) || PUT.equals(method))) {
throw new IllegalStateException("Body was supplied for GET or DELETE request");
}
URI nakedUri;
try {
nakedUri = new URI(url);
} catch (Exception e) {
e.printStackTrace();
throw new IllegalStateException(e);
}
if (qparams == null) {
qparams = new ArrayList<NameValuePair>();
}
URI uri;
try {
uri =
new URI(
nakedUri.getScheme(),
nakedUri.getUserInfo(),
nakedUri.getHost(),
nakedUri.getPort(),
nakedUri.getPath(),
URLEncodedUtils.format(qparams, HtmlConsts.UTF8_ENCODE),
null);
} catch (URISyntaxException e1) {
e1.printStackTrace();
throw new IllegalStateException(e1);
}
System.out.println(uri.toString());
if (GET.equals(method)) {
HttpGet get = new HttpGet(uri);
request = get;
} else if (DELETE.equals(method)) {
HttpDelete delete = new HttpDelete(uri);
request = delete;
} else if (PATCH.equals(method)) {
HttpPatch patch = new HttpPatch(uri);
patch.setEntity(entity);
request = patch;
} else if (POST.equals(method)) {
HttpPost post = new HttpPost(uri);
post.setEntity(entity);
request = post;
} else if (PUT.equals(method)) {
HttpPut put = new HttpPut(uri);
put.setEntity(entity);
request = put;
} else {
throw new IllegalStateException("Unexpected request method");
}
HttpResponse resp = client.execute(request);
return resp;
}
/**
* Helper function for constructors.
*
* @param parameterTableRelation
* @param fsc
* @param cc
* @return
* @throws ODKDatastoreException
*/
protected static final <T extends CommonFieldsBase> T retrieveEntity(
T parameterTableRelation, FormServiceCursor fsc, CallingContext cc)
throws ODKDatastoreException {
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
return ds.getEntity(parameterTableRelation, fsc.getAuriService(), user);
}
@BeforeClass
public static void oneTimeSetUp() throws Exception {
CallingContext cc = TestContextFactory.getCallingContext();
MyRelation rel = MyRelation.assertRelation(cc);
System.out.println("dropping the large dataset");
cc.getDatastore()
.dropRelation(rel, cc.getCurrentUser()); // drop it, in case prior test was messed up...
rel = MyRelation.assertRelation(cc);
System.out.println("writing the large dataset");
// write a lot of data...
for (DataValue value : values) {
value.assertInstance(rel, cc);
}
try {
Thread.sleep(PersistConsts.MAX_SETTLE_MILLISECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("done writing the large dataset");
}
void assertInstance(MyRelation rel, CallingContext cc)
throws ODKEntityPersistException, ODKOverQuotaException {
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
for (int i = 0; i < SET_SIZE; ++i) {
MyRelation element = ds.createEntityUsingRelation(rel, user);
element.setStringField(MyRelation.fieldStr, str);
element.setLongField(MyRelation.fieldInt, Long.valueOf(i));
element.setNumericField(MyRelation.fieldDbl, bd);
element.setDateField(MyRelation.fieldDate, d);
element.setBooleanField(MyRelation.fieldBool, b);
ds.putEntity(element, user);
}
for (int i = 0; i < SET_SIZE; ++i) {
MyRelation element = ds.createEntityUsingRelation(rel, user);
element.setStringField(MyRelation.fieldStr, str);
element.setLongField(MyRelation.fieldInt, Long.valueOf(i));
element.setNumericField(MyRelation.fieldDbl, BigDecimal.valueOf(i));
element.setDateField(MyRelation.fieldDate, d);
element.setBooleanField(MyRelation.fieldBool, b);
ds.putEntity(element, user);
}
}
static final synchronized MyRelation assertRelation(CallingContext cc)
throws ODKDatastoreException {
MyRelation relationPrototype;
Datastore ds = cc.getDatastore();
User user = cc.getUserService().getDaemonAccountUser();
relationPrototype = new MyRelation(ds.getDefaultSchemaName());
ds.assertRelation(relationPrototype, user); // may throw exception...
// at this point, the prototype has become fully populated
return relationPrototype; // set static variable only upon success...
}
protected void postUploadTask(CallingContext cc) throws ODKExternalServiceException {
// upload data to external service
if (!fsc.getExternalServicePublicationOption()
.equals(ExternalServicePublicationOption.STREAM_ONLY)) {
UploadSubmissions uploadTask = (UploadSubmissions) cc.getBean(BeanDefs.UPLOAD_TASK_BEAN);
CallingContext ccDaemon = ContextFactory.duplicateContext(cc);
ccDaemon.setAsDaemon(true);
uploadTask.createFormUploadTask(fsc, true, ccDaemon);
}
}
@Override
public void setUploadCompleted(CallingContext cc)
throws ODKEntityPersistException, ODKOverQuotaException {
fsc.setUploadCompleted(true);
if (fsc.getExternalServicePublicationOption() == ExternalServicePublicationOption.UPLOAD_ONLY) {
fsc.setOperationalStatus(OperationalStatus.COMPLETED);
}
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
ds.putEntity(fsc, user);
}
@AfterClass
public static void afterClass() throws Exception {
CallingContext cc = TestContextFactory.getCallingContext();
MyRelation rel = MyRelation.assertRelation(cc);
System.out.println("dropping the large dataset");
cc.getDatastore()
.dropRelation(rel, cc.getCurrentUser()); // drop it, in case prior test was messed up...
}
@Test
public void testCase1int() throws ODKDatastoreException {
CallingContext cc = TestContextFactory.getCallingContext();
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
MyRelation rel = MyRelation.assertRelation(cc);
Query query = ds.createQuery(rel, "QueryResultTest.testCase1int", user);
List<?> values = query.executeDistinctValueForDataField(MyRelation.fieldInt);
assertEquals(SET_SIZE, values.size());
}
@Override
public void persist(CallingContext cc) throws ODKEntityPersistException, ODKOverQuotaException {
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
CommonFieldsBase serviceEntity = retrieveObjectEntity();
List<? extends CommonFieldsBase> repeats = retrieveRepeatElementEntities();
if (repeats != null) {
ds.putEntities(repeats, user);
}
ds.putEntity(serviceEntity, user);
ds.putEntity(fsc, user);
}
@Test
public void testCase2() throws ODKDatastoreException {
CallingContext cc = TestContextFactory.getCallingContext();
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
MyRelation rel = MyRelation.assertRelation(cc);
Query query = ds.createQuery(rel, "QueryResultTest.testCase2", user);
query.addFilter(MyRelation.fieldInt, FilterOperation.GREATER_THAN, SET_SIZE - 2);
query.addSort(MyRelation.fieldDate, Direction.ASCENDING);
query.addSort(MyRelation.fieldDbl, Direction.DESCENDING);
Set<String> pkSet = new HashSet<String>();
int len = 0;
QueryResult result = query.executeQuery(null, 10);
len += result.getResultList().size();
assertEquals(true, result.hasMoreResults());
assertEquals(10, result.getResultList().size());
for (CommonFieldsBase cb : result.getResultList()) {
assertEquals(false, pkSet.contains(cb.getUri()));
pkSet.add(cb.getUri());
((MyRelation) cb).print();
}
boolean done = false;
QueryResumePoint startCursor = result.getResumeCursor();
while (!done) {
System.out.println("Issuing follow-up query");
result = query.executeQuery(startCursor, 20);
len += result.getResultList().size();
startCursor = result.getResumeCursor();
done = !result.hasMoreResults();
for (CommonFieldsBase cb : result.getResultList()) {
assertEquals(false, pkSet.contains(cb.getUri()));
pkSet.add(cb.getUri());
((MyRelation) cb).print();
}
}
assertEquals(false, result.hasMoreResults());
assertEquals(2 * values.length, len);
System.out.println("done with testCase2");
}
@Override
public void delete(CallingContext cc) throws ODKDatastoreException {
CommonFieldsBase serviceEntity = retrieveObjectEntity();
List<? extends CommonFieldsBase> repeats = retrieveRepeatElementEntities();
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
if (repeats != null) {
List<EntityKey> keys = new ArrayList<EntityKey>();
for (CommonFieldsBase repeat : repeats) {
keys.add(repeat.getEntityKey());
}
ds.deleteEntities(keys, user);
repeats.clear();
}
ds.deleteEntity(serviceEntity.getEntityKey(), user);
ds.deleteEntity(fsc.getEntityKey(), user);
}
/**
* Helper function for constructors.
*
* @param parameterTableRelation
* @param cc
* @return
* @throws ODKDatastoreException
*/
protected static final <T extends CommonFieldsBase> T newEntity(
T parameterTableRelation, CallingContext cc) throws ODKDatastoreException {
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
return ds.createEntityUsingRelation(parameterTableRelation, user);
}
@Test
public void testCase3() throws ODKDatastoreException {
CallingContext cc = TestContextFactory.getCallingContext();
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
MyRelation rel = MyRelation.assertRelation(cc);
System.out.println("start testCase3");
Query query = ds.createQuery(rel, "QueryResultTest.testCase3(1st)", user);
query.addFilter(MyRelation.fieldDbl, FilterOperation.EQUAL, new BigDecimal("0.9"));
query.addFilter(MyRelation.fieldBool, FilterOperation.EQUAL, true);
query.addSort(MyRelation.fieldInt, Direction.ASCENDING);
query.addSort(MyRelation.fieldDate, Direction.DESCENDING);
Query backquery = ds.createQuery(rel, "QueryResultTest.testCase3(2nd)", user);
backquery.addFilter(MyRelation.fieldDbl, FilterOperation.EQUAL, new BigDecimal("0.9"));
backquery.addFilter(MyRelation.fieldBool, FilterOperation.EQUAL, true);
backquery.addSort(MyRelation.fieldInt, Direction.DESCENDING);
backquery.addSort(MyRelation.fieldDate, Direction.ASCENDING);
Set<String> pkTotalSet = new HashSet<String>();
List<String> pkOrdering = new ArrayList<String>();
int TOTAL_SIZE = 3 * SET_SIZE;
int fetchSizes[] = {1, 2, 1021, 303, 101, 2831};
int idxFetch = 4;
int len = 0;
QueryResult result = query.executeQuery(null, fetchSizes[idxFetch]);
len += result.getResultList().size();
assertEquals(true, result.hasMoreResults());
assertEquals(fetchSizes[idxFetch], result.getResultList().size());
System.out.println("Accumulating forward query results");
for (CommonFieldsBase cb : result.getResultList()) {
assertEquals(false, pkOrdering.contains(cb.getUri()));
assertEquals(false, pkTotalSet.contains(cb.getUri()));
pkOrdering.add(cb.getUri());
pkTotalSet.add(cb.getUri());
((MyRelation) cb).print();
}
System.out.println("Verifying initial backward query is empty");
QueryResult backResult = backquery.executeQuery(result.getBackwardCursor(), pkOrdering.size());
assertEquals(false, backResult.hasMoreResults());
assertEquals(0, backResult.getResultList().size());
boolean notFirst = false;
boolean done = false;
QueryResumePoint startCursor = result.getResumeCursor();
while (!done) {
idxFetch = (idxFetch + 1) % fetchSizes.length;
System.out.println("Issuing follow-up query");
result = query.executeQuery(startCursor, fetchSizes[idxFetch]);
len += result.getResultList().size();
startCursor = result.getResumeCursor();
done = !result.hasMoreResults();
System.out.println("Verifying backward query against ordering of earlier result");
backResult = backquery.executeQuery(result.getBackwardCursor(), pkOrdering.size());
assertEquals(notFirst, backResult.hasMoreResults());
notFirst = true;
assertEquals(pkOrdering.size(), backResult.getResultList().size());
for (int i = 0; i < pkOrdering.size(); ++i) {
CommonFieldsBase cb = backResult.getResultList().get(i);
((MyRelation) cb).print();
assertEquals(pkOrdering.get(pkOrdering.size() - i - 1), cb.getUri());
}
System.out.println("Accumulating forward query results");
pkOrdering.clear();
for (CommonFieldsBase cb : result.getResultList()) {
assertEquals(false, pkOrdering.contains(cb.getUri()));
assertEquals(false, pkTotalSet.contains(cb.getUri()));
pkOrdering.add(cb.getUri());
pkTotalSet.add(cb.getUri());
((MyRelation) cb).print();
}
}
idxFetch = (idxFetch + 1) % fetchSizes.length;
System.out.println("Before Issuing (what should be empty) follow-up query");
result = query.executeQuery(startCursor, fetchSizes[idxFetch]);
len += result.getResultList().size();
startCursor = result.getResumeCursor();
done = !result.hasMoreResults();
System.out.println("Verifying backward query against ordering of earlier result");
// backquery should match existing data
backResult = backquery.executeQuery(result.getBackwardCursor(), pkOrdering.size());
assertEquals(true, backResult.hasMoreResults());
assertEquals(pkOrdering.size(), backResult.getResultList().size());
for (int i = 0; i < pkOrdering.size(); ++i) {
CommonFieldsBase cb = backResult.getResultList().get(i);
((MyRelation) cb).print();
assertEquals(pkOrdering.get(pkOrdering.size() - i - 1), cb.getUri());
}
assertEquals(false, result.hasMoreResults());
assertEquals(0, result.getResultList().size());
idxFetch = (idxFetch + 1) % fetchSizes.length;
System.out.println("Before Re-Issuing (what should be empty) follow-up query");
// this should be an empty list
result = query.executeQuery(startCursor, fetchSizes[idxFetch]);
assertEquals(false, result.hasMoreResults());
assertEquals(0, result.getResultList().size());
System.out.println("Verifying backward query (again) against ordering of earlier result");
// backquery should match existing data
backResult = backquery.executeQuery(result.getBackwardCursor(), pkOrdering.size());
assertEquals(true, backResult.hasMoreResults());
assertEquals(pkOrdering.size(), backResult.getResultList().size());
for (int i = 0; i < pkOrdering.size(); ++i) {
CommonFieldsBase cb = backResult.getResultList().get(i);
((MyRelation) cb).print();
assertEquals(pkOrdering.get(pkOrdering.size() - i - 1), cb.getUri());
}
assertEquals(TOTAL_SIZE, len);
}
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
CallingContext cc = ContextFactory.getCallingContext(this, req);
User user = cc.getCurrentUser();
UserService userService = cc.getUserService();
// Check to make sure we are using the canonical server name.
// If not, redirect to that name. This ensures that authentication
// cookies will have the proper realm(s) established for them.
String newUrl = cc.getServerURL() + BasicConsts.FORWARDSLASH + ADDR;
String query = req.getQueryString();
if (query != null && query.length() != 0) {
newUrl += "?" + query;
}
URL url = new URL(newUrl);
if (!url.getHost().equalsIgnoreCase(req.getServerName())) {
// we should redirect over to the proper fully-formed URL.
logger.info(
"Incoming servername: "
+ req.getServerName()
+ " expected: "
+ url.getHost()
+ " -- redirecting.");
resp.sendRedirect(newUrl);
return;
}
// OK. We are using the canonical server name.
boolean isSuperUser = false;
try {
isSuperUser = userService.isSuperUser(cc);
} catch (ODKDatastoreException e) {
e.printStackTrace();
}
// determine if this is the first time the system has not been accessed...
if (isSuperUser) {
// this is the super-user -- examine the isEnabled
// field to determine whether this is the first time
// visiting the site. If it is, force a redirect to
// the site-configuration tab.
boolean directToConfigTab = false;
Datastore ds = cc.getDatastore();
try {
long changeTimestamp = SecurityRevisionsTable.getLastSuperUserIdRevisionDate(ds, user);
long reviewStamp = SecurityRevisionsTable.getLastPermissionsViewRevisionDate(ds, user);
if (reviewStamp < changeTimestamp) {
SecurityRevisionsTable.setLastPermissionsViewRevisionDate(ds, user);
directToConfigTab = true;
}
} catch (ODKDatastoreException e) {
e.printStackTrace();
}
if (directToConfigTab) {
newUrl += "#admin/permission///";
logger.info("Redirect to configuration tab: " + newUrl);
resp.sendRedirect(newUrl);
return;
}
}
resp.setContentType(HtmlConsts.RESP_TYPE_HTML);
resp.setCharacterEncoding(HtmlConsts.UTF8_ENCODE);
PrintWriter out = resp.getWriter();
out.print(PAGE_CONTENTS_FIRST);
String simpleApiKey;
try {
simpleApiKey = ServerPreferencesProperties.getGoogleSimpleApiKey(cc);
if (simpleApiKey != null && simpleApiKey.length() != 0) {
out.print("key=" + encodeParameter(simpleApiKey) + "&");
}
} catch (ODKEntityNotFoundException e) {
e.printStackTrace();
logger.info("Unable to access Map APIKey");
} catch (ODKOverQuotaException e) {
e.printStackTrace();
logger.info("Unable to access Map APIKey");
}
out.print(PAGE_CONTENTS_SECOND);
}
/**
* Handler for HTTP Get request to create xform upload page
*
* @see javax.servlet.http.HttpServlet#doGet(javax.servlet.http.HttpServletRequest,
* javax.servlet.http.HttpServletResponse)
*/
@Override
public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
CallingContext cc = ContextFactory.getCallingContext(this, req);
cc.setAsDaemon(true);
logger.info("GAE servlet for CSV generation begins");
// get parameter
final String formId = getParameter(req, ServletConsts.FORM_ID);
if (formId == null) {
logger.error("Missing " + ServletConsts.FORM_ID + " key");
errorMissingKeyParam(resp);
return;
}
final String persistentResultsString = getParameter(req, ServletConsts.PERSISTENT_RESULTS_KEY);
if (persistentResultsString == null) {
logger.error("Missing " + ServletConsts.PERSISTENT_RESULTS_KEY + " key");
errorBadParam(resp);
return;
}
SubmissionKey persistentResultsKey = new SubmissionKey(persistentResultsString);
final String attemptCountString = getParameter(req, ServletConsts.ATTEMPT_COUNT);
if (attemptCountString == null) {
logger.error("Missing " + ServletConsts.ATTEMPT_COUNT + " key");
errorBadParam(resp);
return;
}
Long attemptCount = 1L;
try {
attemptCount = Long.valueOf(attemptCountString);
} catch (Exception e) {
logger.error(
"Invalid "
+ ServletConsts.ATTEMPT_COUNT
+ " value: "
+ attemptCountString
+ " exception: "
+ e.toString());
errorBadParam(resp);
return;
}
IForm form = null;
try {
form = FormFactory.retrieveFormByFormId(formId, cc);
} catch (ODKFormNotFoundException e) {
logger.error("Unable to retrieve formId: " + formId + " exception: " + e.toString());
e.printStackTrace();
odkIdNotFoundError(resp);
return;
} catch (ODKOverQuotaException e) {
logger.error("Unable to retrieve formId: " + formId + " exception: " + e.toString());
e.printStackTrace();
quotaExceededError(resp);
return;
} catch (ODKDatastoreException e) {
logger.error("Unable to retrieve formId: " + formId + " exception: " + e.toString());
e.printStackTrace();
datastoreError(resp);
return;
}
if (!form.hasValidFormDefinition()) {
logger.error("Unable to retrieve formId: " + formId + " invalid form definition");
errorRetreivingData(resp);
return; // ill-formed definition
}
CsvWorkerImpl impl = new CsvWorkerImpl(form, persistentResultsKey, attemptCount, cc);
impl.generateCsv();
}
