private void reviewAssignment(String assignmentId, boolean correct) {
try {
if (correct) {
service.approveAssignment(assignmentId, "Product classified correctly. Thank you.");
} else {
service.approveAssignment(assignmentId, "Product classified. Thank you.");
}
} catch (InternalServiceException e) {
log.error(
"InternalServiceException while trying to approve["
+ correct
+ "] assignment "
+ assignmentId
+ ", "
+ e.getMessage());
} catch (Exception e) {
log.error(
"Exception while trying to approve["
+ correct
+ "] assignment "
+ assignmentId
+ ", "
+ e.getMessage());
}
}
/**
* A Method to create a HIT with the target word and the 2 words on either side along with the
* sense and part of speech
*
* @param Context before the target word
* @param Target word
* @param Context after the target word
* @param Word sense
* @param Part of speech
*/
private void createPartialContextGivenSurvey(
String partialFirst, String target, String partialSecond, String sense, String POS) {
try {
HIT hit =
service.createHIT(
null,
partialContextTitle,
partialContextDescription,
null,
partialContextSub(partialFirst, target, partialSecond, sense, POS),
reward,
(long) 300,
(long) 432000,
(long) 259200,
numAssignments,
target,
requirements,
null);
// Print out the HITId and the URL to view the HIT.
System.out.println("Created HIT: " + hit.getHITId());
partialContextpr.println(hit.getHITId());
System.out.println("HIT location: ");
System.out.println(service.getWebsiteURL() + "/mturk/preview?groupId=" + hit.getHITTypeId());
} catch (ServiceException e) {
System.err.println(e.getLocalizedMessage());
}
}
/**
* A method to create a HIT with context and the target word provided Takes in the target word and
* the surrounding context in two pieces
*
* @param Context before the target word
* @param Target word
* @param Context after the target word
* @throws FileNotFoundException
*/
public void createContextGivenSurvey(String firstSentence, String word, String secondSentence)
throws FileNotFoundException {
try {
HIT hit =
service.createHIT(
null,
contextGivenTitle,
contextGivenDescription,
null,
contextGivenSub(firstSentence, word, secondSentence),
reward,
(long) 300,
(long) 432000,
(long) 345600,
numAssignments,
word,
requirements,
null);
// Print out the HITId and the URL to view the HIT.
System.out.println("Created HIT: " + hit.getHITId());
contextpr.println(hit.getHITId());
System.out.println("HIT location: ");
System.out.println(service.getWebsiteURL() + "/mturk/preview?groupId=" + hit.getHITTypeId());
} catch (ServiceException e) {
System.err.println(e.getLocalizedMessage());
}
}
/**
* Check to see if there are sufficient funds.
*
* @return true if there are sufficient funds. False otherwise.
*/
public boolean hasEnoughFund() {
double balance = service.getAccountBalance();
// System.out.println("Got account balance: "+
// RequesterService.formatCurrency(balance));
return balance > 0;
}
public String createMyHit(
RequesterService service,
ArrayList<Object> inputs,
int numberOfOutputs,
int numberOfAssignments) {
/* Some parameters used for initializing MTurk HIT.*/
String title = "Select best description";
String description = "Please selec the best description.";
String keywords = "selection, description";
Question question = new Question(inputs, numberOfOutputs);
double reward = 0.01;
long assignmentDurationInSeconds = 60 * 30; // 30 minutes
long autoApprovalDelayInSeconds = 60; // 1 minute
long lifetimeInSeconds = 60 * 60 * 24 * 7; // 1 week
HIT hit =
service.createHIT(
null,
title,
description,
keywords,
question.getQuestion(),
reward,
assignmentDurationInSeconds,
autoApprovalDelayInSeconds,
lifetimeInSeconds,
numberOfAssignments,
null,
null,
null);
return hit.getHITId();
}
public void getHITStatus(String hitId) {
HIT hit = service.getHIT(hitId);
HITStatus hitStatus = hit.getHITStatus();
String value = hitStatus.getValue();
System.out.println("hit value = " + value);
Assignment[] assignments = service.getAssignmentsForHIT(hitId, 1);
if (assignments != null) {
for (Assignment assignment : assignments) {
System.out.println("assignment:" + assignment.getAnswer());
}
}
}
/**
* Approves all the submissions for the hit whose ID is given
*
* @param HIT Id
* @throws IOException
*/
public void approveHIT(String hitId) throws IOException {
try {
hitId.trim();
HIT currentHIT = service.getHIT(hitId);
Assignment[] answers = service.getAssignmentsForHIT(hitId, currentHIT.getMaxAssignments());
// Print out the HITId and the URL to view the HIT.
System.out.println("Retrieved HIT: " + hitId);
for (Assignment answer : answers) {
answer.setAssignmentStatus(AssignmentStatus.Approved);
System.out.println("Approved HIT: " + hitId);
service.approveAssignment(answer.getAssignmentId(), "Accepted");
}
} catch (ServiceException e) {
System.err.println(e.getLocalizedMessage());
}
}
/**
* Deletes the hit whose ID is given
*
* @param HIT Id
* @throws IOException
*/
public void deleteHIT(String hitId) throws IOException {
try {
hitId.trim();
service.disableHIT(hitId);
// Print out the HITId and the URL to view the HIT.
System.out.println("Removed HIT: " + hitId);
} catch (ServiceException e) {
System.err.println(e.getLocalizedMessage());
}
}
public ArrayList<Object> getMyHitAnswers(RequesterService service, String hitId) {
Assignment[] assignments = service.getAllAssignmentsForHIT(hitId);
ArrayList<Object> rawAnswers = new ArrayList<Object>();
for (Assignment assignment : assignments) {
String log =
assignment.getWorkerId()
+ " had the following"
+ " answers for HIT("
+ assignment.getHITId()
+ "): "
+ (new Date()).toString()
+ "\n";
// Interpret the XML and parse answers out.
String answerXML = assignment.getAnswer();
QuestionFormAnswers qfa = RequesterService.parseAnswers(answerXML);
@SuppressWarnings("unchecked")
ArrayList<QuestionFormAnswersType.AnswerType> answers =
(ArrayList<QuestionFormAnswersType.AnswerType>) qfa.getAnswer();
for (QuestionFormAnswersType.AnswerType answer : answers) {
String assignmentId = assignment.getAssignmentId();
String answerValues = RequesterService.getAnswerValue(assignmentId, answer);
String[] rawAnswerValues = null;
if (answerValues != null) {
rawAnswerValues = answerValues.split("\\|");
}
for (String ans : rawAnswerValues) {
if (ans.startsWith("desc_identifier:")) {
rawAnswers.add(ans.substring(16));
log += ans.substring(16) + "\n";
}
}
}
log += "---------\n\n";
LogWriter.writeLog(log, "detail.txt");
}
return rawAnswers;
}
public void deleteHITs(String[] string) {
service.deleteHITs(string, true, true, null);
}
public void getHITResult(String hitId) {
long startTime = System.currentTimeMillis();
log.info("getting assignment result for hitId " + hitId);
Assignment[] assignments = service.getAllSubmittedAssignmentsForHIT(hitId);
Response[] responses = new Response[assignments.length];
Request request = null;
// Product product = null;
if (assignments != null && assignments.length > 0) {
request = turkDAO.getRequestByHitId(assignments[0].getHITId());
}
if (request == null) {
log.error("request not found for hitId=" + hitId);
return;
}
int i = 0;
for (Assignment assignment : assignments) {
log.info(
"got assignmentID "
+ assignment.getAssignmentId()
+ " from worker "
+ assignment.getWorkerId());
log.info("answer XML: " + assignment.getAnswer());
String categoryIdAnswer = answerParser.getAnswer(assignment.getAnswer());
log.info("answer is: " + categoryIdAnswer);
Response response = turkDAO.getResponseByAssignmentId(assignment.getAssignmentId());
if (response != null) {
} else {
response = new Response();
response.setAnswer(categoryIdAnswer);
response.setCreatedDate(new Date());
response.setHitId(assignment.getHITId());
response.setWorkerId(assignment.getWorkerId());
response.setAssignmentId(assignment.getAssignmentId());
response.setRequestId(request.getId());
}
response.setAnswer(categoryIdAnswer);
responses[i] = response;
turkDAO.saveResponse(response);
i++;
}
if (assignments.length == 2
&& responses[0].getAnswer() != null
&& responses[1].getAnswer() != null) {
if (!AnswerParser.EMPTY_ANSWER.equals(responses[0].getAnswer())
&& responses[0].getAnswer().equals(responses[1].getAnswer())) {
String unanymousAnswer = responses[0].getAnswer();
reviewAssignment(assignments[0].getAssignmentId(), true);
reviewAssignment(assignments[1].getAssignmentId(), true);
Product product = turkDAO.getProductById(request.getProductId());
product.setCategoryCode(unanymousAnswer);
product.setModifiedDate(new Date());
turkDAO.saveProduct(product);
for (int j = 0; j < 2; j++) {
responses[j].setResult(Response.RESULT_APPROVED);
turkDAO.saveResponse(responses[j]);
log.info("approved assignmentId " + responses[j].getAssignmentId());
}
} else {
// create adjudication hit
Product product = turkDAO.getProductById(request.getProductId());
log.info("answers did not match, creating adjudication hit for product " + product.getId());
createHIT(product, 2);
}
} else if (assignments.length == 1 && responses[0].getAnswer() != null) {
// check to see if its an adjudication result
if (request != null && request.getTier() == 2) {
// this was a tier 2 hit
// provide feedback for tier one hits
int productId = request.getProductId();
// retrieve tier one request
Request tierOneRequest = turkDAO.getRequestByProductIdAndTier(productId, 1);
List<Response> tierOneResponses = turkDAO.getResponsesByRequestId(tierOneRequest.getId());
for (Response tierOneResponse : tierOneResponses) {
if (tierOneResponse.getAnswer() == null) {
log.warn(
"tier2 response received but tier one answer is null for responseId "
+ tierOneResponse.getId());
} else if (!AnswerParser.EMPTY_ANSWER.equals(responses[0].getAnswer())
&& tierOneResponse.getAnswer().equals(responses[0].getAnswer())) {
reviewAssignment(tierOneResponse.getAssignmentId(), true);
tierOneResponse.setResult(Response.RESULT_APPROVED);
log.info(
"approved assigment "
+ tierOneResponse.getAssignmentId()
+ ", matched adjudication result");
} else {
reviewAssignment(tierOneResponse.getAssignmentId(), false);
tierOneResponse.setResult(Response.RESULT_REJECTED);
log.info(
"rejected assigment "
+ tierOneResponse.getAssignmentId()
+ ", did not match adjudication result");
}
tierOneResponse.setModifiedDate(new Date());
turkDAO.saveResponse(tierOneResponse);
}
// review tier2 response
if (!AnswerParser.EMPTY_ANSWER.equals(responses[0].getAnswer())) {
reviewAssignment(assignments[0].getAssignmentId(), true);
responses[0].setResult(Response.RESULT_APPROVED);
log.info("approved adjudication assigment " + assignments[0].getAssignmentId());
} else {
reviewAssignment(assignments[0].getAssignmentId(), false);
responses[0].setResult(Response.RESULT_REJECTED);
log.info("rejected adjudication assigment " + assignments[0].getAssignmentId());
}
responses[0].setModifiedDate(new Date());
turkDAO.saveResponse(responses[0]);
Product product = turkDAO.getProductById(request.getProductId());
product.setCategoryCode(responses[0].getAnswer());
product.setModifiedDate(new Date());
turkDAO.saveProduct(product);
}
}
long elapsed = System.currentTimeMillis() - startTime;
log.info("hitResult processed in " + elapsed + " ms");
}
public void createHIT(Product product, int tier) {
String parentCategoryName = turkDAO.getParentCategoryNameForProduct(product.getId());
log.info("creating tier " + tier + " hit for productId " + product.getId());
// String hitTypeId = null;
String title = "Choose the best category for this " + parentCategoryName + " product";
String description = "Please look at this product and select a categorization for it.";
String keywords = "shopping,product,merchandise,categorization,shopzilla";
double reward = 0.0;
try {
if (tier == 1) {
reward = tierOneReward;
} else {
reward = tierTwoReward;
}
} catch (Exception ignore) {
}
int maxAssignments = 3 - tier;
long assignmentDurationInSeconds = 60 * 60; // one hour
long autoApprovalDelayInSeconds = 60 * 60 * 24 * 30; // one month
long lifetimeInSeconds = 60 * 60 * 24; // one day
String requesterAnnotation = "ShopZilla two plus one HIT";
String externalURL = "http://" + hostname + "/cloudsort/hit.do?id=" + product.getId();
if (tier == 2) {
externalURL += "&tier=2";
}
log.info("creating tier " + tier + " hit, url: " + externalURL);
String question =
"<?xml version=\"1.0\"?>"
+ "<ExternalQuestion xmlns=\"http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd\">"
+ " <ExternalURL>"
+ externalURL
+ "</ExternalURL>"
+ " <FrameHeight>500</FrameHeight>"
+ "</ExternalQuestion>";
QualificationRequirement[] qualificationRequirements = null;
Qualification qualification = null;
try {
qualification = turkDAO.getQualificationForCategoryId(product.getParentCategoryId());
} catch (Exception ignore) {
log.info("no qualification data found for parent category " + product.getParentCategoryId());
}
if (qualification != null) {
if (tier == 2
&& qualification.getQualTypeIdTrusted() != null
&& qualification.getQualTypeIdTrusted().length() > 0
|| tier == 1
&& qualification.getQualTypeIdGeneral() != null
&& qualification.getQualTypeIdGeneral().length() > 0) {
qualificationRequirements = new QualificationRequirement[1];
qualificationRequirements[0] = new QualificationRequirement();
qualificationRequirements[0].setComparator(Comparator.GreaterThanOrEqualTo);
if (tier == 1) {
qualificationRequirements[0].setQualificationTypeId(qualification.getQualTypeIdGeneral());
qualificationRequirements[0].setIntegerValue(qualification.getQualTypeScoreGeneral());
} else {
qualificationRequirements[0].setQualificationTypeId(qualification.getQualTypeIdGeneral());
qualificationRequirements[0].setIntegerValue(qualification.getQualTypeScoreGeneral());
}
qualificationRequirements[0].setRequiredToPreview(false);
log.info(
"qualification typeID: "
+ qualificationRequirements[0].getQualificationTypeId()
+ ", score: "
+ qualificationRequirements[0].getIntegerValue());
}
}
String responseGroup[] = null;
String hitTypeId = null;
long startTime = System.currentTimeMillis();
HIT hit =
service.createHIT(
hitTypeId,
title,
description,
keywords,
question,
reward,
assignmentDurationInSeconds,
autoApprovalDelayInSeconds,
lifetimeInSeconds,
maxAssignments,
requesterAnnotation,
qualificationRequirements,
responseGroup);
long elapsed = System.currentTimeMillis() - startTime;
log.info(
"hit id "
+ hit.getHITId()
+ ", hitType "
+ hit.getHITTypeId()
+ " created in "
+ elapsed
+ " ms");
Request request = new Request();
request.setProductId(product.getId());
request.setCreatedDate(new Date());
request.setHitId(hit.getHITId());
request.setParentCategoryId(product.getParentCategoryId());
request.setTier(tier);
startTime = System.currentTimeMillis();
turkDAO.saveRequest(request);
elapsed = System.currentTimeMillis() - startTime;
log.info("request saved in " + elapsed + " ms");
if (hitTypes.contains(hit.getHITTypeId())) {
log.info("notification already set up for hitTypeId " + hit.getHITTypeId());
} else {
log.info("setting up notification for hitTypeId " + hit.getHITTypeId());
NotificationSpecification notification = new NotificationSpecification();
notification.setDestination("http://" + hostname + "/cloudsort/hitresult");
notification.setEventType(
new EventType[] {EventType.AssignmentSubmitted
// ,EventType.AssignmentAccepted, EventType.AssignmentReturned,
// EventType.AssignmentAbandoned
});
notification.setTransport(NotificationTransport.REST);
notification.setVersion("2006-05-05");
service.setHITTypeNotification(hit.getHITTypeId(), notification, true);
hitTypes.add(hit.getHITTypeId());
log.info("notification set up for Hit TypeID = " + hit.getHITTypeId());
}
}
public void dumpPastHit(RequesterService service, String hitId) {
service.disableHIT(hitId);
}