Пример #1
0
  private void reviewAssignment(String assignmentId, boolean correct) {

    try {
      if (correct) {
        service.approveAssignment(assignmentId, "Product classified correctly.  Thank you.");
      } else {
        service.approveAssignment(assignmentId, "Product classified.  Thank you.");
      }
    } catch (InternalServiceException e) {
      log.error(
          "InternalServiceException while trying to approve["
              + correct
              + "] assignment "
              + assignmentId
              + ", "
              + e.getMessage());
    } catch (Exception e) {
      log.error(
          "Exception while trying to approve["
              + correct
              + "] assignment "
              + assignmentId
              + ", "
              + e.getMessage());
    }
  }
  /**
   * A Method to create a HIT with the target word and the 2 words on either side along with the
   * sense and part of speech
   *
   * @param Context before the target word
   * @param Target word
   * @param Context after the target word
   * @param Word sense
   * @param Part of speech
   */
  private void createPartialContextGivenSurvey(
      String partialFirst, String target, String partialSecond, String sense, String POS) {
    try {
      HIT hit =
          service.createHIT(
              null,
              partialContextTitle,
              partialContextDescription,
              null,
              partialContextSub(partialFirst, target, partialSecond, sense, POS),
              reward,
              (long) 300,
              (long) 432000,
              (long) 259200,
              numAssignments,
              target,
              requirements,
              null);

      // Print out the HITId and the URL to view the HIT.
      System.out.println("Created HIT: " + hit.getHITId());
      partialContextpr.println(hit.getHITId());
      System.out.println("HIT location: ");
      System.out.println(service.getWebsiteURL() + "/mturk/preview?groupId=" + hit.getHITTypeId());

    } catch (ServiceException e) {
      System.err.println(e.getLocalizedMessage());
    }
  }
  /**
   * A method to create a HIT with context and the target word provided Takes in the target word and
   * the surrounding context in two pieces
   *
   * @param Context before the target word
   * @param Target word
   * @param Context after the target word
   * @throws FileNotFoundException
   */
  public void createContextGivenSurvey(String firstSentence, String word, String secondSentence)
      throws FileNotFoundException {
    try {
      HIT hit =
          service.createHIT(
              null,
              contextGivenTitle,
              contextGivenDescription,
              null,
              contextGivenSub(firstSentence, word, secondSentence),
              reward,
              (long) 300,
              (long) 432000,
              (long) 345600,
              numAssignments,
              word,
              requirements,
              null);

      // Print out the HITId and the URL to view the HIT.
      System.out.println("Created HIT: " + hit.getHITId());
      contextpr.println(hit.getHITId());
      System.out.println("HIT location: ");
      System.out.println(service.getWebsiteURL() + "/mturk/preview?groupId=" + hit.getHITTypeId());

    } catch (ServiceException e) {
      System.err.println(e.getLocalizedMessage());
    }
  }
Пример #4
0
  /**
   * Check to see if there are sufficient funds.
   *
   * @return true if there are sufficient funds. False otherwise.
   */
  public boolean hasEnoughFund() {

    double balance = service.getAccountBalance();
    // System.out.println("Got account balance: "+
    // RequesterService.formatCurrency(balance));
    return balance > 0;
  }
Пример #5
0
  public String createMyHit(
      RequesterService service,
      ArrayList<Object> inputs,
      int numberOfOutputs,
      int numberOfAssignments) {

    /* Some parameters used for initializing MTurk HIT.*/
    String title = "Select best description";
    String description = "Please selec the best description.";
    String keywords = "selection, description";
    Question question = new Question(inputs, numberOfOutputs);
    double reward = 0.01;
    long assignmentDurationInSeconds = 60 * 30; // 30 minutes
    long autoApprovalDelayInSeconds = 60; // 1 minute
    long lifetimeInSeconds = 60 * 60 * 24 * 7; // 1 week
    HIT hit =
        service.createHIT(
            null,
            title,
            description,
            keywords,
            question.getQuestion(),
            reward,
            assignmentDurationInSeconds,
            autoApprovalDelayInSeconds,
            lifetimeInSeconds,
            numberOfAssignments,
            null,
            null,
            null);
    return hit.getHITId();
  }
Пример #6
0
  public void getHITStatus(String hitId) {

    HIT hit = service.getHIT(hitId);

    HITStatus hitStatus = hit.getHITStatus();

    String value = hitStatus.getValue();

    System.out.println("hit value  = " + value);

    Assignment[] assignments = service.getAssignmentsForHIT(hitId, 1);
    if (assignments != null) {
      for (Assignment assignment : assignments) {
        System.out.println("assignment:" + assignment.getAnswer());
      }
    }
  }
  /**
   * Approves all the submissions for the hit whose ID is given
   *
   * @param HIT Id
   * @throws IOException
   */
  public void approveHIT(String hitId) throws IOException {
    try {
      hitId.trim();
      HIT currentHIT = service.getHIT(hitId);
      Assignment[] answers = service.getAssignmentsForHIT(hitId, currentHIT.getMaxAssignments());

      // Print out the HITId and the URL to view the HIT.
      System.out.println("Retrieved HIT: " + hitId);
      for (Assignment answer : answers) {
        answer.setAssignmentStatus(AssignmentStatus.Approved);
        System.out.println("Approved HIT: " + hitId);
        service.approveAssignment(answer.getAssignmentId(), "Accepted");
      }

    } catch (ServiceException e) {
      System.err.println(e.getLocalizedMessage());
    }
  }
  /**
   * Deletes the hit whose ID is given
   *
   * @param HIT Id
   * @throws IOException
   */
  public void deleteHIT(String hitId) throws IOException {
    try {
      hitId.trim();
      service.disableHIT(hitId);

      // Print out the HITId and the URL to view the HIT.
      System.out.println("Removed HIT: " + hitId);

    } catch (ServiceException e) {
      System.err.println(e.getLocalizedMessage());
    }
  }
Пример #9
0
  public ArrayList<Object> getMyHitAnswers(RequesterService service, String hitId) {
    Assignment[] assignments = service.getAllAssignmentsForHIT(hitId);
    ArrayList<Object> rawAnswers = new ArrayList<Object>();

    for (Assignment assignment : assignments) {
      String log =
          assignment.getWorkerId()
              + " had the following"
              + " answers for HIT("
              + assignment.getHITId()
              + "):  "
              + (new Date()).toString()
              + "\n";

      // Interpret the XML and parse answers out.
      String answerXML = assignment.getAnswer();
      QuestionFormAnswers qfa = RequesterService.parseAnswers(answerXML);
      @SuppressWarnings("unchecked")
      ArrayList<QuestionFormAnswersType.AnswerType> answers =
          (ArrayList<QuestionFormAnswersType.AnswerType>) qfa.getAnswer();
      for (QuestionFormAnswersType.AnswerType answer : answers) {
        String assignmentId = assignment.getAssignmentId();
        String answerValues = RequesterService.getAnswerValue(assignmentId, answer);
        String[] rawAnswerValues = null;
        if (answerValues != null) {
          rawAnswerValues = answerValues.split("\\|");
        }
        for (String ans : rawAnswerValues) {
          if (ans.startsWith("desc_identifier:")) {
            rawAnswers.add(ans.substring(16));
            log += ans.substring(16) + "\n";
          }
        }
      }
      log += "---------\n\n";
      LogWriter.writeLog(log, "detail.txt");
    }
    return rawAnswers;
  }
Пример #10
0
  public void deleteHITs(String[] string) {

    service.deleteHITs(string, true, true, null);
  }
Пример #11
0
  public void getHITResult(String hitId) {

    long startTime = System.currentTimeMillis();
    log.info("getting assignment result for hitId " + hitId);

    Assignment[] assignments = service.getAllSubmittedAssignmentsForHIT(hitId);

    Response[] responses = new Response[assignments.length];
    Request request = null;
    // Product product = null;
    if (assignments != null && assignments.length > 0) {
      request = turkDAO.getRequestByHitId(assignments[0].getHITId());
    }

    if (request == null) {
      log.error("request not found for hitId=" + hitId);
      return;
    }

    int i = 0;
    for (Assignment assignment : assignments) {

      log.info(
          "got assignmentID "
              + assignment.getAssignmentId()
              + " from worker "
              + assignment.getWorkerId());

      log.info("answer XML: " + assignment.getAnswer());
      String categoryIdAnswer = answerParser.getAnswer(assignment.getAnswer());
      log.info("answer is: " + categoryIdAnswer);

      Response response = turkDAO.getResponseByAssignmentId(assignment.getAssignmentId());

      if (response != null) {

      } else {
        response = new Response();
        response.setAnswer(categoryIdAnswer);
        response.setCreatedDate(new Date());
        response.setHitId(assignment.getHITId());
        response.setWorkerId(assignment.getWorkerId());
        response.setAssignmentId(assignment.getAssignmentId());
        response.setRequestId(request.getId());
      }

      response.setAnswer(categoryIdAnswer);

      responses[i] = response;
      turkDAO.saveResponse(response);
      i++;
    }

    if (assignments.length == 2
        && responses[0].getAnswer() != null
        && responses[1].getAnswer() != null) {

      if (!AnswerParser.EMPTY_ANSWER.equals(responses[0].getAnswer())
          && responses[0].getAnswer().equals(responses[1].getAnswer())) {

        String unanymousAnswer = responses[0].getAnswer();

        reviewAssignment(assignments[0].getAssignmentId(), true);
        reviewAssignment(assignments[1].getAssignmentId(), true);

        Product product = turkDAO.getProductById(request.getProductId());
        product.setCategoryCode(unanymousAnswer);
        product.setModifiedDate(new Date());
        turkDAO.saveProduct(product);

        for (int j = 0; j < 2; j++) {
          responses[j].setResult(Response.RESULT_APPROVED);

          turkDAO.saveResponse(responses[j]);

          log.info("approved assignmentId " + responses[j].getAssignmentId());
        }

      } else {

        // create adjudication hit

        Product product = turkDAO.getProductById(request.getProductId());

        log.info("answers did not match, creating adjudication hit for product " + product.getId());

        createHIT(product, 2);
      }

    } else if (assignments.length == 1 && responses[0].getAnswer() != null) {
      // check to see if its an adjudication result

      if (request != null && request.getTier() == 2) {
        // this was a tier 2 hit

        // provide feedback for tier one hits
        int productId = request.getProductId();

        // retrieve tier one request
        Request tierOneRequest = turkDAO.getRequestByProductIdAndTier(productId, 1);

        List<Response> tierOneResponses = turkDAO.getResponsesByRequestId(tierOneRequest.getId());

        for (Response tierOneResponse : tierOneResponses) {

          if (tierOneResponse.getAnswer() == null) {
            log.warn(
                "tier2 response received but tier one answer is null for responseId "
                    + tierOneResponse.getId());
          } else if (!AnswerParser.EMPTY_ANSWER.equals(responses[0].getAnswer())
              && tierOneResponse.getAnswer().equals(responses[0].getAnswer())) {

            reviewAssignment(tierOneResponse.getAssignmentId(), true);
            tierOneResponse.setResult(Response.RESULT_APPROVED);

            log.info(
                "approved assigment "
                    + tierOneResponse.getAssignmentId()
                    + ", matched adjudication result");

          } else {
            reviewAssignment(tierOneResponse.getAssignmentId(), false);
            tierOneResponse.setResult(Response.RESULT_REJECTED);

            log.info(
                "rejected assigment "
                    + tierOneResponse.getAssignmentId()
                    + ", did not match adjudication result");
          }
          tierOneResponse.setModifiedDate(new Date());
          turkDAO.saveResponse(tierOneResponse);
        }

        // review tier2 response

        if (!AnswerParser.EMPTY_ANSWER.equals(responses[0].getAnswer())) {

          reviewAssignment(assignments[0].getAssignmentId(), true);
          responses[0].setResult(Response.RESULT_APPROVED);
          log.info("approved adjudication assigment " + assignments[0].getAssignmentId());
        } else {
          reviewAssignment(assignments[0].getAssignmentId(), false);
          responses[0].setResult(Response.RESULT_REJECTED);
          log.info("rejected adjudication assigment " + assignments[0].getAssignmentId());
        }
        responses[0].setModifiedDate(new Date());
        turkDAO.saveResponse(responses[0]);

        Product product = turkDAO.getProductById(request.getProductId());
        product.setCategoryCode(responses[0].getAnswer());
        product.setModifiedDate(new Date());
        turkDAO.saveProduct(product);
      }
    }

    long elapsed = System.currentTimeMillis() - startTime;
    log.info("hitResult processed in " + elapsed + " ms");
  }
Пример #12
0
  public void createHIT(Product product, int tier) {

    String parentCategoryName = turkDAO.getParentCategoryNameForProduct(product.getId());
    log.info("creating tier " + tier + " hit for productId " + product.getId());

    // String hitTypeId = null;
    String title = "Choose the best category for this " + parentCategoryName + " product";
    String description = "Please look at this product and select a categorization for it.";
    String keywords = "shopping,product,merchandise,categorization,shopzilla";

    double reward = 0.0;

    try {
      if (tier == 1) {
        reward = tierOneReward;
      } else {
        reward = tierTwoReward;
      }

    } catch (Exception ignore) {
    }

    int maxAssignments = 3 - tier;
    long assignmentDurationInSeconds = 60 * 60; // one hour
    long autoApprovalDelayInSeconds = 60 * 60 * 24 * 30; // one month
    long lifetimeInSeconds = 60 * 60 * 24; // one day
    String requesterAnnotation = "ShopZilla two plus one HIT";

    String externalURL = "http://" + hostname + "/cloudsort/hit.do?id=" + product.getId();
    if (tier == 2) {
      externalURL += "&amp;tier=2";
    }

    log.info("creating tier " + tier + " hit, url: " + externalURL);

    String question =
        "<?xml version=\"1.0\"?>"
            + "<ExternalQuestion xmlns=\"http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd\">"
            + "	<ExternalURL>"
            + externalURL
            + "</ExternalURL>"
            + "	<FrameHeight>500</FrameHeight>"
            + "</ExternalQuestion>";

    QualificationRequirement[] qualificationRequirements = null;

    Qualification qualification = null;

    try {
      qualification = turkDAO.getQualificationForCategoryId(product.getParentCategoryId());
    } catch (Exception ignore) {
      log.info("no qualification data found for parent category " + product.getParentCategoryId());
    }

    if (qualification != null) {

      if (tier == 2
              && qualification.getQualTypeIdTrusted() != null
              && qualification.getQualTypeIdTrusted().length() > 0
          || tier == 1
              && qualification.getQualTypeIdGeneral() != null
              && qualification.getQualTypeIdGeneral().length() > 0) {

        qualificationRequirements = new QualificationRequirement[1];
        qualificationRequirements[0] = new QualificationRequirement();
        qualificationRequirements[0].setComparator(Comparator.GreaterThanOrEqualTo);
        if (tier == 1) {
          qualificationRequirements[0].setQualificationTypeId(qualification.getQualTypeIdGeneral());
          qualificationRequirements[0].setIntegerValue(qualification.getQualTypeScoreGeneral());
        } else {
          qualificationRequirements[0].setQualificationTypeId(qualification.getQualTypeIdGeneral());
          qualificationRequirements[0].setIntegerValue(qualification.getQualTypeScoreGeneral());
        }
        qualificationRequirements[0].setRequiredToPreview(false);

        log.info(
            "qualification typeID: "
                + qualificationRequirements[0].getQualificationTypeId()
                + ", score: "
                + qualificationRequirements[0].getIntegerValue());
      }
    }

    String responseGroup[] = null;
    String hitTypeId = null;

    long startTime = System.currentTimeMillis();
    HIT hit =
        service.createHIT(
            hitTypeId,
            title,
            description,
            keywords,
            question,
            reward,
            assignmentDurationInSeconds,
            autoApprovalDelayInSeconds,
            lifetimeInSeconds,
            maxAssignments,
            requesterAnnotation,
            qualificationRequirements,
            responseGroup);

    long elapsed = System.currentTimeMillis() - startTime;
    log.info(
        "hit id "
            + hit.getHITId()
            + ", hitType "
            + hit.getHITTypeId()
            + " created in "
            + elapsed
            + " ms");

    Request request = new Request();
    request.setProductId(product.getId());
    request.setCreatedDate(new Date());
    request.setHitId(hit.getHITId());
    request.setParentCategoryId(product.getParentCategoryId());
    request.setTier(tier);
    startTime = System.currentTimeMillis();
    turkDAO.saveRequest(request);
    elapsed = System.currentTimeMillis() - startTime;
    log.info("request saved in " + elapsed + " ms");

    if (hitTypes.contains(hit.getHITTypeId())) {
      log.info("notification already set up for hitTypeId " + hit.getHITTypeId());
    } else {
      log.info("setting up notification for hitTypeId " + hit.getHITTypeId());
      NotificationSpecification notification = new NotificationSpecification();
      notification.setDestination("http://" + hostname + "/cloudsort/hitresult");
      notification.setEventType(
          new EventType[] {EventType.AssignmentSubmitted
            // ,EventType.AssignmentAccepted, EventType.AssignmentReturned,
            // EventType.AssignmentAbandoned
          });
      notification.setTransport(NotificationTransport.REST);
      notification.setVersion("2006-05-05");

      service.setHITTypeNotification(hit.getHITTypeId(), notification, true);

      hitTypes.add(hit.getHITTypeId());
      log.info("notification set up for Hit TypeID = " + hit.getHITTypeId());
    }
  }
Пример #13
0
 public void dumpPastHit(RequesterService service, String hitId) {
   service.disableHIT(hitId);
 }