@Override
public void addTask(Task task) {
Queue queue = null;
logger.info("Adding Task[" + task.toString() + "]");
if (task.getQueueName() == null
|| (task.getQueueName() != null && task.getQueueName().trim().length() == 0)) {
queue = QueueFactory.getDefaultQueue();
} else {
queue = QueueFactory.getQueue(task.getQueueName());
}
logger.info("Queue to be used : " + queue.getQueueName());
if (task.getParameterName() != null && task.getParameterValue() != null) {
logger.info("Adding Parameters to process");
queue.add(
TaskOptions.Builder.withUrl(task.getEndpoint())
.param(task.getParameterName(), task.getParameterValue())
.taskName(task.getName()));
} else {
logger.info("excluding Parameters to process");
queue.add(TaskOptions.Builder.withUrl(task.getEndpoint()).taskName(task.getName()));
}
}
public void deleteAudioClip(String audioClipId) throws BadRequestException {
AudioClip audioClip = getAudioClipById(audioClipId);
datastore.delete(KeyFactory.stringToKey(audioClipId));
// task queue to delete the associated audio and image blobs
Queue queue = QueueFactory.getDefaultQueue();
queue.add(
TaskOptions.Builder.withUrl("/rest/users/" + audioClip.getOwnerId() + "/audioClips/audio")
.method(TaskOptions.Method.DELETE)
.param("blobkey", audioClip.getAudioId()));
queue.add(
TaskOptions.Builder.withUrl("/rest/users/" + audioClip.getOwnerId() + "/audioClips/image")
.method(TaskOptions.Method.DELETE)
.param("blobkey", audioClip.getImageId()));
}
/**
* handles the callback from the device indicating that a new data file is available. This method
* will call processFile to retrieve the file and persist the data to the data store it will then
* add access points for each water point in the survey responses.
*
* @param req
*/
@SuppressWarnings("rawtypes")
private void ingestFile(TaskRequest req) {
if (req.getFileName() != null) {
log.info(" Task->processFile");
ArrayList<SurveyInstance> surveyInstances =
processFile(req.getFileName(), req.getPhoneNumber(), req.getChecksum(), req.getOffset());
Map<Long, Survey> surveyMap = new HashMap<Long, Survey>();
SurveyDAO surveyDao = new SurveyDAO();
Queue summQueue = QueueFactory.getQueue("dataSummarization");
Queue defaultQueue = QueueFactory.getDefaultQueue();
for (SurveyInstance instance : surveyInstances) {
Survey s = surveyMap.get(instance.getSurveyId());
if (s == null) {
s = surveyDao.getById(instance.getSurveyId());
surveyMap.put(instance.getSurveyId(), s);
}
if (s != null && s.getRequireApproval() != null && s.getRequireApproval()) {
// if the survey requires approval, don't run any of the
// processors
instance.setApprovedFlag("False");
continue;
} else {
ProcessingAction pa = dispatch(instance.getKey().getId() + "");
TaskOptions options = TaskOptions.Builder.withUrl(pa.getDispatchURL());
Iterator it = pa.getParams().keySet().iterator();
while (it.hasNext()) {
options.param("key", (String) it.next());
}
log.info(
"Received Task Queue calls for surveyInstanceKey: " + instance.getKey().getId() + "");
aph.processSurveyInstance(instance.getKey().getId() + "");
summQueue.add(
TaskOptions.Builder.withUrl("/app_worker/datasummarization")
.param("objectKey", instance.getKey().getId() + "")
.param("type", "SurveyInstance"));
// process the "new" domain structure
defaultQueue.add(
TaskOptions.Builder.withUrl("/app_worker/surveyalservlet")
.param(
SurveyalRestRequest.ACTION_PARAM, SurveyalRestRequest.INGEST_INSTANCE_ACTION)
.param(
SurveyalRestRequest.SURVEY_INSTANCE_PARAM, instance.getKey().getId() + ""));
}
}
}
}
// Run this test twice to demonstrate we're not leaking state across tests.
// If we _are_ leaking state across tests we'll get an exception on the
// second test because there will already be a task with the given name.
private void doTest() throws InterruptedException {
QueueFactory.getDefaultQueue().add(TaskOptions.Builder.withTaskName("task29"));
// Give the task time to execute if tasks are actually enabled (which they
// aren't, but that's part of the test).
Thread.sleep(1000);
LocalTaskQueue ltq = LocalTaskQueueTestConfig.getLocalTaskQueue();
QueueStateInfo qsi = ltq.getQueueStateInfo().get(QueueFactory.getDefaultQueue().getQueueName());
assertEquals(1, qsi.getTaskInfo().size());
assertEquals("task29", qsi.getTaskInfo().get(0).getTaskName());
}
// Executed by user menu click
public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
// Build a task using the TaskOptions Builder pattern from ** above
Queue queue = QueueFactory.getDefaultQueue();
queue.add(TaskOptions.Builder.withUrl("/taskq_demo").method(TaskOptions.Method.POST));
resp.getWriter().println("Task have been added to default queue...");
resp.getWriter().println("Refresh this page to add another count task");
}
public static void createCatalogBackendTask(String queueName, Date date) {
Queue queue = QueueFactory.getQueue(queueName);
TaskOptions taskOptions =
TaskOptions.Builder.withUrl("/tasks/catalog").method(TaskOptions.Method.POST);
if (date != null) {
taskOptions.param(
"date", DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT).format(date));
}
taskOptions.param("backend", Boolean.toString(true));
queue.add(taskOptions);
}
protected void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String key = request.getParameter("keyname");
String keyvalue = request.getParameter("value");
// Add the task to the default queue.
Queue queue = QueueFactory.getDefaultQueue();
queue.add(
TaskOptions.Builder.withUrl("/worker").param("keyname", key).param("value", keyvalue));
response.sendRedirect("/done.html");
}
/**
* puts the summarization request into the queue
*
* @param request
*/
private void invokeSummarizer(DataSummarizationRequest request) {
Queue queue = QueueFactory.getQueue(queueName);
queue.add(
TaskOptions.Builder.withUrl(summarizerPath)
.param(DataSummarizationRequest.ACTION_PARAM, request.getAction())
.param(DataSummarizationRequest.OBJECT_KEY, request.getObjectKey())
.param(DataSummarizationRequest.OBJECT_TYPE, request.getType())
.param(DataSummarizationRequest.OFFSET_KEY, request.getOffset().toString())
.param(
DataSummarizationRequest.CURSOR_PARAM,
request.getCursor() != null ? request.getCursor() : ""));
}
public static void createInitBackendTask(String queueName, Cursor cursor) {
Queue queue = QueueFactory.getQueue(queueName);
TaskOptions taskOptions =
TaskOptions.Builder.withUrl("/tasks/init").method(TaskOptions.Method.POST);
if (cursor != null && cursor.getElement() != null && cursor.getOffset() != null) {
taskOptions
.param("element", cursor.getElement())
.param("offset", cursor.getOffset().toString());
}
if (cursor != null && cursor.backend) {
taskOptions.param("backend", Boolean.toString(cursor.backend));
}
log.info("Added queue " + taskOptions.toString());
queue.add(taskOptions);
}
@Test
public void testNoTaskIterable() {
Transaction tx = DatastoreServiceFactory.getDatastoreService().beginTransaction();
final int beforeNumTasks = getDefaultQueue().fetchStatistics().getNumTasks();
try {
getDefaultQueue().add(tx, Collections.singleton(TaskOptions.Builder.withDefaults()));
} finally {
tx.rollback();
}
if (doIgnore("testNoTaskIterable")) {
return;
}
sync(10000); // Wait for statistics servers to refresh.
Assert.assertEquals(beforeNumTasks, getDefaultQueue().fetchStatistics().getNumTasks());
}
@Test
public void testNoTaskSingleAsync() {
Transaction tx = DatastoreServiceFactory.getDatastoreService().beginTransaction();
final int beforeNumTasks = getDefaultQueue().fetchStatistics().getNumTasks();
try {
waitOnFuture(getDefaultQueue().addAsync(tx, TaskOptions.Builder.withDefaults()));
} finally {
tx.rollback();
}
if (doIgnore("testNoTaskSingleAsync")) {
return;
}
sync(10000);
Assert.assertEquals(
beforeNumTasks, waitOnFuture(getDefaultQueue().fetchStatisticsAsync(2013.0)).getNumTasks());
}
TaskOptionsBuilder(String addr) {
task = TaskOptions.Builder.withUrl(BasicConsts.FORWARDSLASH + addr);
task.method(TaskOptions.Method.GET);
}
private ArrayList<SurveyInstance> processFile(
String fileName, String phoneNumber, String checksum, Integer offset) {
ArrayList<SurveyInstance> surveyInstances = new ArrayList<SurveyInstance>();
try {
DeviceFilesDao dfDao = new DeviceFilesDao();
URL url = new URL(DEVICE_FILE_PATH + fileName);
BufferedInputStream bis = new BufferedInputStream(url.openStream());
ZipInputStream zis = new ZipInputStream(bis);
List<DeviceFiles> dfList = null;
DeviceFiles deviceFile = null;
dfList = dfDao.listByUri(url.toURI().toString());
if (dfList != null && dfList.size() > 0) deviceFile = dfList.get(0);
if (deviceFile == null) {
deviceFile = new DeviceFiles();
}
deviceFile.setProcessDate(getNowDateTimeFormatted());
deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
deviceFile.setURI(url.toURI().toString());
if (phoneNumber == null || phoneNumber.equals("null")) deviceFile.setPhoneNumber(null);
else deviceFile.setPhoneNumber(phoneNumber);
if (checksum == null || checksum.equals("null")) deviceFile.setChecksum(null);
else deviceFile.setChecksum(checksum);
deviceFile.setUploadDateTime(new Date());
Date collectionDate = new Date();
ArrayList<String> unparsedLines = null;
try {
unparsedLines = extractDataFromZip(zis);
} catch (Exception iex) {
// Error unzipping the response file
deviceFile.setProcessedStatus(StatusCode.ERROR_INFLATING_ZIP);
String message =
"Error inflating device zip: " + deviceFile.getURI() + " : " + iex.getMessage();
log.log(Level.SEVERE, message);
deviceFile.addProcessingMessage(message);
MailUtil.sendMail(
FROM_ADDRESS,
"FLOW",
recepientList,
"Device File Processing Error: " + fileName,
message);
}
if (unparsedLines != null && unparsedLines.size() > 0) {
if (REGION_FLAG.equals(unparsedLines.get(0))) {
unparsedLines.remove(0);
GeoRegionHelper grh = new GeoRegionHelper();
grh.processRegionsSurvey(unparsedLines);
} else {
int lineNum = offset;
String curId = null;
while (lineNum < unparsedLines.size()) {
String[] parts = unparsedLines.get(lineNum).split("\t");
if (parts.length < 5) {
parts = unparsedLines.get(lineNum).split(",");
}
if (parts.length >= 2) {
if (curId == null) {
curId = parts[1];
} else {
// if this isn't the first time through and
// we are seeing a new id, break since we'll
// process that in another call
if (!curId.equals(parts[1])) {
break;
}
}
}
lineNum++;
}
Long userID = 1L;
dfDao.save(deviceFile);
SurveyInstance inst =
siDao.save(
collectionDate, deviceFile, userID, unparsedLines.subList(offset, lineNum));
if (inst != null) {
// fire a survey event
SurveyEventHelper.fireEvent(
SurveyEventHelper.SUBMISSION_EVENT, inst.getSurveyId(), inst.getKey().getId());
surveyInstances.add(inst);
// TODO: HACK because we were saving so many duplicate
// device files this way they all get the same status
if (dfList != null) {
for (DeviceFiles dfitem : dfList) {
dfitem.setProcessedStatus(inst.getDeviceFile().getProcessedStatus());
}
}
}
if (lineNum < unparsedLines.size()) {
if (inst != null) {
StatusCode processingStatus = inst.getDeviceFile().getProcessedStatus();
if (processingStatus.equals(StatusCode.PROCESSED_WITH_ERRORS)) {
String message =
"Error in file during first processing step. Continuing to next part";
deviceFile.addProcessingMessage(message);
deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
} else {
deviceFile.addProcessingMessage(
"Processed " + lineNum + " lines spawning queue call");
deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
}
}
// if we haven't processed everything yet, invoke a
// new service
Queue queue = QueueFactory.getDefaultQueue();
queue.add(
TaskOptions.Builder.withUrl("/app_worker/task")
.param("action", "processFile")
.param("fileName", fileName)
.param("offset", lineNum + ""));
} else {
StatusCode status = StatusCode.PROCESSED_NO_ERRORS;
if (deviceFile.getProcessedStatus() != null) {
status = deviceFile.getProcessedStatus();
}
deviceFile.setProcessedStatus(status);
if (dfList != null) {
for (DeviceFiles dfitem : dfList) {
dfitem.setProcessedStatus(status);
}
}
}
}
} else {
deviceFile.setProcessedStatus(StatusCode.PROCESSED_WITH_ERRORS);
String message = "Error empty file: " + deviceFile.getURI();
log.log(Level.SEVERE, message);
deviceFile.addProcessingMessage(message);
MailUtil.sendMail(
FROM_ADDRESS,
"FLOW",
recepientList,
"Device File Processing Error: " + fileName,
DEVICE_FILE_PATH + fileName + "\n" + message);
}
dfDao.save(dfList);
zis.close();
} catch (Exception e) {
log.log(Level.SEVERE, "Could not process data file", e);
MailUtil.sendMail(
FROM_ADDRESS,
"FLOW",
recepientList,
"Device File Processing Error: " + fileName,
DEVICE_FILE_PATH + fileName + "\n" + (e.getMessage() != null ? e.getMessage() : ""));
}
return surveyInstances;
}
public static ArrayList<String> extractDataFromZip(ZipInputStream zis)
throws IOException, SignedDataException {
ArrayList<String> lines = new ArrayList<String>();
String line = null;
String surveyDataOnly = null;
String dataSig = null;
ZipEntry entry;
while ((entry = zis.getNextEntry()) != null) {
log.info("Unzipping: " + entry.getName());
ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] buffer = new byte[2048];
int size;
while ((size = zis.read(buffer, 0, buffer.length)) != -1) {
out.write(buffer, 0, size);
}
line = out.toString("UTF-8");
if (entry.getName().endsWith("txt")) {
if (entry.getName().equals("regions.txt")) {
lines.add("regionFlag=true");
} else {
surveyDataOnly = line;
}
String[] linesSplit = line.split("\n");
for (String s : linesSplit) {
if (s.contains("\u0000")) {
s = s.replaceAll("\u0000", "");
}
lines.add(s);
}
} else if (entry.getName().endsWith(".sig")) {
dataSig = line.trim();
} else {
S3Driver s3 = new S3Driver();
String[] imageParts = entry.getName().split("/");
// comment out while testing locally
try {
// GAEImageAdapter gaeIA = new GAEImageAdapter();
// byte[] resizedImage =
// gaeIA.resizeImage(out.toByteArray(), 500, 500);
// s3.uploadFile("dru-test", imageParts[1], resizedImage);
GAEImageAdapter gaeImg = new GAEImageAdapter();
byte[] newImage = gaeImg.resizeImage(out.toByteArray(), 500, 500);
s3.uploadFile("dru-test", imageParts[1], newImage);
// add queue call to resize
Queue queue = QueueFactory.getDefaultQueue();
queue.add(TaskOptions.Builder.withUrl("imageprocessor").param("imageURL", imageParts[1]));
log.info("submiting image resize for imageURL: " + imageParts[1]);
} catch (Exception ex) {
ex.printStackTrace();
}
out.close();
}
zis.closeEntry();
}
// check the signature if we have it
if (surveyDataOnly != null && dataSig != null) {
try {
MessageDigest sha1Digest = MessageDigest.getInstance("SHA1");
byte[] digest = sha1Digest.digest(surveyDataOnly.getBytes("UTF-8"));
SecretKeySpec signingKey =
new SecretKeySpec(
PropertyUtil.getProperty(SIGNING_KEY).getBytes("UTF-8"), SIGNING_ALGORITHM);
Mac mac = Mac.getInstance(SIGNING_ALGORITHM);
mac.init(signingKey);
byte[] hmac = mac.doFinal(digest);
String encodedHmac = com.google.gdata.util.common.util.Base64.encode(hmac);
if (!encodedHmac.trim().equals(dataSig.trim())) {
String allowUnsigned = PropertyUtil.getProperty(ALLOW_UNSIGNED);
if (allowUnsigned != null && allowUnsigned.trim().equalsIgnoreCase("false")) {
throw new SignedDataException(
"Computed signature does not match the one submitted with the data");
} else {
log.warning("Signatures don't match. Processing anyway since allow unsigned is true");
}
}
} catch (GeneralSecurityException e) {
throw new SignedDataException("Could not calculate signature", e);
}
} else if (surveyDataOnly != null) {
// if there is no signature, check the configuration to see if we
// are allowed to proceed
String allowUnsigned = PropertyUtil.getProperty(ALLOW_UNSIGNED);
if (allowUnsigned != null && allowUnsigned.trim().equalsIgnoreCase("false")) {
throw new SignedDataException("Datafile does not have a signature");
}
}
return lines;
}
/**
* Queues a task for background execution using the specified queue name and the configured or
* default task URL.
*
* <p>If the task URL is not configured via the <code>taskUrl</code> init parameter, uses the
* default task URL, which takes the form:
*
* <blockquote>
*
* <code>/_ah/queue/<i><queue name></i></code>
*
* </blockquote>
*
* @param task The task to be executed.
* @param queueName The name of the queue.
* @throws QueueFailureException If an error occurs serializing the task.
* @return A {@link TaskHandle} for the queued task.
*/
public static TaskHandle defer(Deferrable task, String queueName) {
return defer(
task, queueName, taskUrl != null ? TaskOptions.Builder.withUrl(taskUrl) : withDefaults());
}
protected void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
JSONParser parser = new JSONParser();
String originalRouteJsonString = request.getParameter("originalroutejsontext");
JSONObject originalRouteJsonObject = (JSONObject) parser.parse(originalRouteJsonString);
JSONObject routes = (JSONObject) ((JSONArray) originalRouteJsonObject.get("routes")).get(0);
JSONArray legs = (JSONArray) routes.get("legs");
JSONArray steps = (JSONArray) ((JSONObject) legs.get(0)).get("steps");
String routeID = request.getParameter("routeid");
// System.out.println("Route steps loaded in a JSONArray...size is " + steps.size());
List<Double> stepLats = new ArrayList<Double>();
List<Double> stepLngs = new ArrayList<Double>();
for (int i = 0; i < steps.size(); i++) {
JSONObject temp = (JSONObject) ((JSONObject) steps.get(i)).get("end_location");
// System.out.println("Lat of end_location of step " + i + " " + temp.get("lat"));
stepLats.add(Double.parseDouble(temp.get("lat").toString()));
stepLngs.add(Double.parseDouble(temp.get("lng").toString()));
}
// System.out.println("All steps set with size " + stepLngs.size() + " and " +
// stepLats.size());
// System.out.println("Skipping route boxer...");
// RouteBoxer routeBoxer = new RouteBoxer(stepLats, stepLngs,
// Double.parseDouble(request.getParameter("radius")));
// if(routeBoxer.getFlag())
// throw new RuntimeException("Could not create boxes for the route");
// List<Double> boxLats = routeBoxer.getLats();
// List<Double> boxLngs = routeBoxer.getLngs();
// System.out.println("Calculated boxes with number of lats " + boxLats.size() + " and number
// of lngs " + boxLngs.size());
double r = Double.parseDouble(request.getParameter("radius").toString());
int radius = r > RADIUS_TO_LOOK_FOR_PLACES ? RADIUS_TO_LOOK_FOR_PLACES : (int) r;
String[] types = request.getParameter("keywords").split(",");
System.out.println("Size of types is " + types.length);
JSONObject finalPlacesJSONObject = new JSONObject();
for (int j = 0; j < types.length; j++) {
JSONArray jsonArrayForType = new JSONArray();
for (int i = 0; i < stepLats.size(); i++) {
JSONObject placesAroundLocationJSONObject =
(JSONObject)
parser.parse(
GoogleMap.getPlacesAroundLocation(
stepLats.get(i), stepLngs.get(i), radius, types[j]));
JSONArray placesAroundLocationJSONArray =
(JSONArray) placesAroundLocationJSONObject.get("results");
if (!placesAroundLocationJSONArray.isEmpty()) {
jsonArrayForType.addAll(placesAroundLocationJSONArray);
}
}
finalPlacesJSONObject.put(types[j], jsonArrayForType);
}
List<String> place_ids = new ArrayList<String>();
finalPlacesJSONObject = removeDuplicatePlaces(finalPlacesJSONObject);
finalPlacesJSONObject =
filterPlacesRandomly(finalPlacesJSONObject, FINAL_PLACES_NUMBER_PER_REQUEST, place_ids);
// System.out.println("MAGIC " + place_ids.toString());
DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
MemcacheService syncCache = MemcacheServiceFactory.getMemcacheService();
// add places as a property of original route entity
Entity originalRouteEntity =
datastore.get(KeyFactory.createKey("Route", Long.parseLong(routeID)));
Text placesJsonAsText = new Text(finalPlacesJSONObject.toJSONString());
originalRouteEntity.setProperty("placesJSON", placesJsonAsText);
datastore.put(originalRouteEntity);
// System.out.println("SUCCESS written places to datastore");
// add task for fetching place reviews to queue
QueueFactory.getDefaultQueue()
.add(
TaskOptions.Builder.withUrl("/waypointsreview")
.param("places", place_ids.toString()));
System.out.println("Task to get reviews added to queue");
// We cache the route entity
String cacheKey = "route-" + routeID;
syncCache.put(cacheKey, originalRouteEntity);
} catch (Exception e) {
System.out.println("ERROR " + e.getMessage());
e.printStackTrace();
}
}