protected HttpResponse sendHttpRequest(
      String method, String url, HttpEntity entity, List<NameValuePair> qparams, CallingContext cc)
      throws IOException {

    HttpParams httpParams = new BasicHttpParams();
    HttpConnectionParams.setConnectionTimeout(
        httpParams, SOCKET_ESTABLISHMENT_TIMEOUT_MILLISECONDS);
    HttpConnectionParams.setSoTimeout(httpParams, SERVICE_TIMEOUT_MILLISECONDS);

    // setup client
    HttpClientFactory factory = (HttpClientFactory) cc.getBean(BeanDefs.HTTP_CLIENT_FACTORY);
    HttpClient client = factory.createHttpClient(httpParams);

    // support redirecting to handle http: => https: transition
    HttpClientParams.setRedirecting(httpParams, true);
    // support authenticating
    HttpClientParams.setAuthenticating(httpParams, true);

    // redirect limit is set to some unreasonably high number
    // resets of the socket cause a retry up to MAX_REDIRECTS times,
    // so we should be careful not to set this too high...
    httpParams.setParameter(ClientPNames.MAX_REDIRECTS, 32);
    httpParams.setParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS, true);

    // context holds authentication state machine, so it cannot be
    // shared across independent activities.
    HttpContext localContext = new BasicHttpContext();

    localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);
    localContext.setAttribute(ClientContext.CREDS_PROVIDER, credsProvider);

    HttpUriRequest request = null;
    if (entity == null && (POST.equals(method) || PATCH.equals(method) || PUT.equals(method))) {
      throw new IllegalStateException("No body supplied for POST, PATCH or PUT request");
    } else if (entity != null
        && !(POST.equals(method) || PATCH.equals(method) || PUT.equals(method))) {
      throw new IllegalStateException("Body was supplied for GET or DELETE request");
    }

    URI nakedUri;
    try {
      nakedUri = new URI(url);
    } catch (Exception e) {
      e.printStackTrace();
      throw new IllegalStateException(e);
    }

    if (qparams == null) {
      qparams = new ArrayList<NameValuePair>();
    }
    URI uri;
    try {
      uri =
          new URI(
              nakedUri.getScheme(),
              nakedUri.getUserInfo(),
              nakedUri.getHost(),
              nakedUri.getPort(),
              nakedUri.getPath(),
              URLEncodedUtils.format(qparams, HtmlConsts.UTF8_ENCODE),
              null);
    } catch (URISyntaxException e1) {
      e1.printStackTrace();
      throw new IllegalStateException(e1);
    }
    System.out.println(uri.toString());

    if (GET.equals(method)) {
      HttpGet get = new HttpGet(uri);
      request = get;
    } else if (DELETE.equals(method)) {
      HttpDelete delete = new HttpDelete(uri);
      request = delete;
    } else if (PATCH.equals(method)) {
      HttpPatch patch = new HttpPatch(uri);
      patch.setEntity(entity);
      request = patch;
    } else if (POST.equals(method)) {
      HttpPost post = new HttpPost(uri);
      post.setEntity(entity);
      request = post;
    } else if (PUT.equals(method)) {
      HttpPut put = new HttpPut(uri);
      put.setEntity(entity);
      request = put;
    } else {
      throw new IllegalStateException("Unexpected request method");
    }

    HttpResponse resp = client.execute(request);
    return resp;
  }
 /**
  * Helper function for constructors.
  *
  * @param parameterTableRelation
  * @param fsc
  * @param cc
  * @return
  * @throws ODKDatastoreException
  */
 protected static final <T extends CommonFieldsBase> T retrieveEntity(
     T parameterTableRelation, FormServiceCursor fsc, CallingContext cc)
     throws ODKDatastoreException {
   Datastore ds = cc.getDatastore();
   User user = cc.getCurrentUser();
   return ds.getEntity(parameterTableRelation, fsc.getAuriService(), user);
 }
  @BeforeClass
  public static void oneTimeSetUp() throws Exception {

    CallingContext cc = TestContextFactory.getCallingContext();

    MyRelation rel = MyRelation.assertRelation(cc);

    System.out.println("dropping the large dataset");
    cc.getDatastore()
        .dropRelation(rel, cc.getCurrentUser()); // drop it, in case prior test was messed up...

    rel = MyRelation.assertRelation(cc);

    System.out.println("writing the large dataset");

    // write a lot of data...
    for (DataValue value : values) {
      value.assertInstance(rel, cc);
    }

    try {
      Thread.sleep(PersistConsts.MAX_SETTLE_MILLISECONDS);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }

    System.out.println("done writing the large dataset");
  }
    void assertInstance(MyRelation rel, CallingContext cc)
        throws ODKEntityPersistException, ODKOverQuotaException {
      Datastore ds = cc.getDatastore();
      User user = cc.getCurrentUser();

      for (int i = 0; i < SET_SIZE; ++i) {
        MyRelation element = ds.createEntityUsingRelation(rel, user);
        element.setStringField(MyRelation.fieldStr, str);
        element.setLongField(MyRelation.fieldInt, Long.valueOf(i));
        element.setNumericField(MyRelation.fieldDbl, bd);
        element.setDateField(MyRelation.fieldDate, d);
        element.setBooleanField(MyRelation.fieldBool, b);

        ds.putEntity(element, user);
      }

      for (int i = 0; i < SET_SIZE; ++i) {
        MyRelation element = ds.createEntityUsingRelation(rel, user);
        element.setStringField(MyRelation.fieldStr, str);
        element.setLongField(MyRelation.fieldInt, Long.valueOf(i));
        element.setNumericField(MyRelation.fieldDbl, BigDecimal.valueOf(i));
        element.setDateField(MyRelation.fieldDate, d);
        element.setBooleanField(MyRelation.fieldBool, b);

        ds.putEntity(element, user);
      }
    }
 static final synchronized MyRelation assertRelation(CallingContext cc)
     throws ODKDatastoreException {
   MyRelation relationPrototype;
   Datastore ds = cc.getDatastore();
   User user = cc.getUserService().getDaemonAccountUser();
   relationPrototype = new MyRelation(ds.getDefaultSchemaName());
   ds.assertRelation(relationPrototype, user); // may throw exception...
   // at this point, the prototype has become fully populated
   return relationPrototype; // set static variable only upon success...
 }
  protected void postUploadTask(CallingContext cc) throws ODKExternalServiceException {
    // upload data to external service
    if (!fsc.getExternalServicePublicationOption()
        .equals(ExternalServicePublicationOption.STREAM_ONLY)) {

      UploadSubmissions uploadTask = (UploadSubmissions) cc.getBean(BeanDefs.UPLOAD_TASK_BEAN);
      CallingContext ccDaemon = ContextFactory.duplicateContext(cc);
      ccDaemon.setAsDaemon(true);
      uploadTask.createFormUploadTask(fsc, true, ccDaemon);
    }
  }
 @Override
 public void setUploadCompleted(CallingContext cc)
     throws ODKEntityPersistException, ODKOverQuotaException {
   fsc.setUploadCompleted(true);
   if (fsc.getExternalServicePublicationOption() == ExternalServicePublicationOption.UPLOAD_ONLY) {
     fsc.setOperationalStatus(OperationalStatus.COMPLETED);
   }
   Datastore ds = cc.getDatastore();
   User user = cc.getCurrentUser();
   ds.putEntity(fsc, user);
 }
  @AfterClass
  public static void afterClass() throws Exception {

    CallingContext cc = TestContextFactory.getCallingContext();

    MyRelation rel = MyRelation.assertRelation(cc);

    System.out.println("dropping the large dataset");
    cc.getDatastore()
        .dropRelation(rel, cc.getCurrentUser()); // drop it, in case prior test was messed up...
  }
  @Test
  public void testCase1int() throws ODKDatastoreException {

    CallingContext cc = TestContextFactory.getCallingContext();
    Datastore ds = cc.getDatastore();
    User user = cc.getCurrentUser();
    MyRelation rel = MyRelation.assertRelation(cc);

    Query query = ds.createQuery(rel, "QueryResultTest.testCase1int", user);

    List<?> values = query.executeDistinctValueForDataField(MyRelation.fieldInt);
    assertEquals(SET_SIZE, values.size());
  }
  @Override
  public void persist(CallingContext cc) throws ODKEntityPersistException, ODKOverQuotaException {
    Datastore ds = cc.getDatastore();
    User user = cc.getCurrentUser();

    CommonFieldsBase serviceEntity = retrieveObjectEntity();
    List<? extends CommonFieldsBase> repeats = retrieveRepeatElementEntities();

    if (repeats != null) {
      ds.putEntities(repeats, user);
    }
    ds.putEntity(serviceEntity, user);
    ds.putEntity(fsc, user);
  }
  @Test
  public void testCase2() throws ODKDatastoreException {

    CallingContext cc = TestContextFactory.getCallingContext();
    Datastore ds = cc.getDatastore();
    User user = cc.getCurrentUser();
    MyRelation rel = MyRelation.assertRelation(cc);

    Query query = ds.createQuery(rel, "QueryResultTest.testCase2", user);
    query.addFilter(MyRelation.fieldInt, FilterOperation.GREATER_THAN, SET_SIZE - 2);
    query.addSort(MyRelation.fieldDate, Direction.ASCENDING);
    query.addSort(MyRelation.fieldDbl, Direction.DESCENDING);

    Set<String> pkSet = new HashSet<String>();

    int len = 0;
    QueryResult result = query.executeQuery(null, 10);
    len += result.getResultList().size();
    assertEquals(true, result.hasMoreResults());
    assertEquals(10, result.getResultList().size());

    for (CommonFieldsBase cb : result.getResultList()) {
      assertEquals(false, pkSet.contains(cb.getUri()));
      pkSet.add(cb.getUri());
      ((MyRelation) cb).print();
    }
    boolean done = false;
    QueryResumePoint startCursor = result.getResumeCursor();
    while (!done) {
      System.out.println("Issuing follow-up query");
      result = query.executeQuery(startCursor, 20);
      len += result.getResultList().size();
      startCursor = result.getResumeCursor();
      done = !result.hasMoreResults();

      for (CommonFieldsBase cb : result.getResultList()) {
        assertEquals(false, pkSet.contains(cb.getUri()));
        pkSet.add(cb.getUri());
        ((MyRelation) cb).print();
      }
    }

    assertEquals(false, result.hasMoreResults());
    assertEquals(2 * values.length, len);
    System.out.println("done with testCase2");
  }
  @Override
  public void delete(CallingContext cc) throws ODKDatastoreException {
    CommonFieldsBase serviceEntity = retrieveObjectEntity();
    List<? extends CommonFieldsBase> repeats = retrieveRepeatElementEntities();

    Datastore ds = cc.getDatastore();
    User user = cc.getCurrentUser();

    if (repeats != null) {
      List<EntityKey> keys = new ArrayList<EntityKey>();
      for (CommonFieldsBase repeat : repeats) {
        keys.add(repeat.getEntityKey());
      }
      ds.deleteEntities(keys, user);
      repeats.clear();
    }

    ds.deleteEntity(serviceEntity.getEntityKey(), user);
    ds.deleteEntity(fsc.getEntityKey(), user);
  }
 /**
  * Helper function for constructors.
  *
  * @param parameterTableRelation
  * @param cc
  * @return
  * @throws ODKDatastoreException
  */
 protected static final <T extends CommonFieldsBase> T newEntity(
     T parameterTableRelation, CallingContext cc) throws ODKDatastoreException {
   Datastore ds = cc.getDatastore();
   User user = cc.getCurrentUser();
   return ds.createEntityUsingRelation(parameterTableRelation, user);
 }
  @Test
  public void testCase3() throws ODKDatastoreException {

    CallingContext cc = TestContextFactory.getCallingContext();
    Datastore ds = cc.getDatastore();
    User user = cc.getCurrentUser();
    MyRelation rel = MyRelation.assertRelation(cc);
    System.out.println("start testCase3");

    Query query = ds.createQuery(rel, "QueryResultTest.testCase3(1st)", user);
    query.addFilter(MyRelation.fieldDbl, FilterOperation.EQUAL, new BigDecimal("0.9"));
    query.addFilter(MyRelation.fieldBool, FilterOperation.EQUAL, true);
    query.addSort(MyRelation.fieldInt, Direction.ASCENDING);
    query.addSort(MyRelation.fieldDate, Direction.DESCENDING);

    Query backquery = ds.createQuery(rel, "QueryResultTest.testCase3(2nd)", user);
    backquery.addFilter(MyRelation.fieldDbl, FilterOperation.EQUAL, new BigDecimal("0.9"));
    backquery.addFilter(MyRelation.fieldBool, FilterOperation.EQUAL, true);
    backquery.addSort(MyRelation.fieldInt, Direction.DESCENDING);
    backquery.addSort(MyRelation.fieldDate, Direction.ASCENDING);

    Set<String> pkTotalSet = new HashSet<String>();

    List<String> pkOrdering = new ArrayList<String>();

    int TOTAL_SIZE = 3 * SET_SIZE;
    int fetchSizes[] = {1, 2, 1021, 303, 101, 2831};
    int idxFetch = 4;
    int len = 0;
    QueryResult result = query.executeQuery(null, fetchSizes[idxFetch]);
    len += result.getResultList().size();
    assertEquals(true, result.hasMoreResults());
    assertEquals(fetchSizes[idxFetch], result.getResultList().size());

    System.out.println("Accumulating forward query results");
    for (CommonFieldsBase cb : result.getResultList()) {
      assertEquals(false, pkOrdering.contains(cb.getUri()));
      assertEquals(false, pkTotalSet.contains(cb.getUri()));
      pkOrdering.add(cb.getUri());
      pkTotalSet.add(cb.getUri());
      ((MyRelation) cb).print();
    }

    System.out.println("Verifying initial backward query is empty");
    QueryResult backResult = backquery.executeQuery(result.getBackwardCursor(), pkOrdering.size());
    assertEquals(false, backResult.hasMoreResults());
    assertEquals(0, backResult.getResultList().size());

    boolean notFirst = false;
    boolean done = false;
    QueryResumePoint startCursor = result.getResumeCursor();
    while (!done) {
      idxFetch = (idxFetch + 1) % fetchSizes.length;
      System.out.println("Issuing follow-up query");
      result = query.executeQuery(startCursor, fetchSizes[idxFetch]);
      len += result.getResultList().size();
      startCursor = result.getResumeCursor();
      done = !result.hasMoreResults();

      System.out.println("Verifying backward query against ordering of earlier result");
      backResult = backquery.executeQuery(result.getBackwardCursor(), pkOrdering.size());
      assertEquals(notFirst, backResult.hasMoreResults());
      notFirst = true;
      assertEquals(pkOrdering.size(), backResult.getResultList().size());
      for (int i = 0; i < pkOrdering.size(); ++i) {
        CommonFieldsBase cb = backResult.getResultList().get(i);
        ((MyRelation) cb).print();
        assertEquals(pkOrdering.get(pkOrdering.size() - i - 1), cb.getUri());
      }

      System.out.println("Accumulating forward query results");
      pkOrdering.clear();
      for (CommonFieldsBase cb : result.getResultList()) {
        assertEquals(false, pkOrdering.contains(cb.getUri()));
        assertEquals(false, pkTotalSet.contains(cb.getUri()));
        pkOrdering.add(cb.getUri());
        pkTotalSet.add(cb.getUri());
        ((MyRelation) cb).print();
      }
    }

    idxFetch = (idxFetch + 1) % fetchSizes.length;
    System.out.println("Before Issuing (what should be empty) follow-up query");
    result = query.executeQuery(startCursor, fetchSizes[idxFetch]);
    len += result.getResultList().size();
    startCursor = result.getResumeCursor();
    done = !result.hasMoreResults();

    System.out.println("Verifying backward query against ordering of earlier result");
    // backquery should match existing data
    backResult = backquery.executeQuery(result.getBackwardCursor(), pkOrdering.size());
    assertEquals(true, backResult.hasMoreResults());
    assertEquals(pkOrdering.size(), backResult.getResultList().size());
    for (int i = 0; i < pkOrdering.size(); ++i) {
      CommonFieldsBase cb = backResult.getResultList().get(i);
      ((MyRelation) cb).print();
      assertEquals(pkOrdering.get(pkOrdering.size() - i - 1), cb.getUri());
    }

    assertEquals(false, result.hasMoreResults());
    assertEquals(0, result.getResultList().size());

    idxFetch = (idxFetch + 1) % fetchSizes.length;
    System.out.println("Before Re-Issuing (what should be empty) follow-up query");
    // this should be an empty list
    result = query.executeQuery(startCursor, fetchSizes[idxFetch]);
    assertEquals(false, result.hasMoreResults());
    assertEquals(0, result.getResultList().size());

    System.out.println("Verifying backward query (again) against ordering of earlier result");
    // backquery should match existing data
    backResult = backquery.executeQuery(result.getBackwardCursor(), pkOrdering.size());
    assertEquals(true, backResult.hasMoreResults());
    assertEquals(pkOrdering.size(), backResult.getResultList().size());
    for (int i = 0; i < pkOrdering.size(); ++i) {
      CommonFieldsBase cb = backResult.getResultList().get(i);
      ((MyRelation) cb).print();
      assertEquals(pkOrdering.get(pkOrdering.size() - i - 1), cb.getUri());
    }

    assertEquals(TOTAL_SIZE, len);
  }
  @Override
  protected void doGet(HttpServletRequest req, HttpServletResponse resp)
      throws ServletException, IOException {
    CallingContext cc = ContextFactory.getCallingContext(this, req);
    User user = cc.getCurrentUser();
    UserService userService = cc.getUserService();

    // Check to make sure we are using the canonical server name.
    // If not, redirect to that name.  This ensures that authentication
    // cookies will have the proper realm(s) established for them.
    String newUrl = cc.getServerURL() + BasicConsts.FORWARDSLASH + ADDR;
    String query = req.getQueryString();
    if (query != null && query.length() != 0) {
      newUrl += "?" + query;
    }
    URL url = new URL(newUrl);
    if (!url.getHost().equalsIgnoreCase(req.getServerName())) {
      // we should redirect over to the proper fully-formed URL.
      logger.info(
          "Incoming servername: "
              + req.getServerName()
              + " expected: "
              + url.getHost()
              + " -- redirecting.");
      resp.sendRedirect(newUrl);
      return;
    }

    // OK. We are using the canonical server name.
    boolean isSuperUser = false;
    try {
      isSuperUser = userService.isSuperUser(cc);
    } catch (ODKDatastoreException e) {
      e.printStackTrace();
    }

    // determine if this is the first time the system has not been accessed...
    if (isSuperUser) {
      // this is the super-user -- examine the isEnabled
      // field to determine whether this is the first time
      // visiting the site. If it is, force a redirect to
      // the site-configuration tab.
      boolean directToConfigTab = false;
      Datastore ds = cc.getDatastore();
      try {
        long changeTimestamp = SecurityRevisionsTable.getLastSuperUserIdRevisionDate(ds, user);
        long reviewStamp = SecurityRevisionsTable.getLastPermissionsViewRevisionDate(ds, user);

        if (reviewStamp < changeTimestamp) {
          SecurityRevisionsTable.setLastPermissionsViewRevisionDate(ds, user);
          directToConfigTab = true;
        }
      } catch (ODKDatastoreException e) {
        e.printStackTrace();
      }
      if (directToConfigTab) {
        newUrl += "#admin/permission///";
        logger.info("Redirect to configuration tab: " + newUrl);
        resp.sendRedirect(newUrl);
        return;
      }
    }

    resp.setContentType(HtmlConsts.RESP_TYPE_HTML);
    resp.setCharacterEncoding(HtmlConsts.UTF8_ENCODE);
    PrintWriter out = resp.getWriter();
    out.print(PAGE_CONTENTS_FIRST);
    String simpleApiKey;
    try {
      simpleApiKey = ServerPreferencesProperties.getGoogleSimpleApiKey(cc);
      if (simpleApiKey != null && simpleApiKey.length() != 0) {
        out.print("key=" + encodeParameter(simpleApiKey) + "&amp;");
      }
    } catch (ODKEntityNotFoundException e) {
      e.printStackTrace();
      logger.info("Unable to access Map APIKey");
    } catch (ODKOverQuotaException e) {
      e.printStackTrace();
      logger.info("Unable to access Map APIKey");
    }
    out.print(PAGE_CONTENTS_SECOND);
  }
  /**
   * Handler for HTTP Get request to create xform upload page
   *
   * @see javax.servlet.http.HttpServlet#doGet(javax.servlet.http.HttpServletRequest,
   *     javax.servlet.http.HttpServletResponse)
   */
  @Override
  public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
    CallingContext cc = ContextFactory.getCallingContext(this, req);
    cc.setAsDaemon(true);

    logger.info("GAE servlet for CSV generation begins");

    // get parameter
    final String formId = getParameter(req, ServletConsts.FORM_ID);
    if (formId == null) {
      logger.error("Missing " + ServletConsts.FORM_ID + " key");
      errorMissingKeyParam(resp);
      return;
    }
    final String persistentResultsString = getParameter(req, ServletConsts.PERSISTENT_RESULTS_KEY);
    if (persistentResultsString == null) {
      logger.error("Missing " + ServletConsts.PERSISTENT_RESULTS_KEY + " key");
      errorBadParam(resp);
      return;
    }
    SubmissionKey persistentResultsKey = new SubmissionKey(persistentResultsString);
    final String attemptCountString = getParameter(req, ServletConsts.ATTEMPT_COUNT);
    if (attemptCountString == null) {
      logger.error("Missing " + ServletConsts.ATTEMPT_COUNT + " key");
      errorBadParam(resp);
      return;
    }
    Long attemptCount = 1L;
    try {
      attemptCount = Long.valueOf(attemptCountString);
    } catch (Exception e) {
      logger.error(
          "Invalid "
              + ServletConsts.ATTEMPT_COUNT
              + " value: "
              + attemptCountString
              + " exception: "
              + e.toString());
      errorBadParam(resp);
      return;
    }

    IForm form = null;
    try {
      form = FormFactory.retrieveFormByFormId(formId, cc);
    } catch (ODKFormNotFoundException e) {
      logger.error("Unable to retrieve formId: " + formId + " exception: " + e.toString());
      e.printStackTrace();
      odkIdNotFoundError(resp);
      return;
    } catch (ODKOverQuotaException e) {
      logger.error("Unable to retrieve formId: " + formId + " exception: " + e.toString());
      e.printStackTrace();
      quotaExceededError(resp);
      return;
    } catch (ODKDatastoreException e) {
      logger.error("Unable to retrieve formId: " + formId + " exception: " + e.toString());
      e.printStackTrace();
      datastoreError(resp);
      return;
    }

    if (!form.hasValidFormDefinition()) {
      logger.error("Unable to retrieve formId: " + formId + " invalid form definition");
      errorRetreivingData(resp);
      return; // ill-formed definition
    }

    CsvWorkerImpl impl = new CsvWorkerImpl(form, persistentResultsKey, attemptCount, cc);

    impl.generateCsv();
  }