Exemplo n.º 1
0
  @Override
  public void addTask(Task task) {
    Queue queue = null;

    logger.info("Adding Task[" + task.toString() + "]");

    if (task.getQueueName() == null
        || (task.getQueueName() != null && task.getQueueName().trim().length() == 0)) {
      queue = QueueFactory.getDefaultQueue();
    } else {
      queue = QueueFactory.getQueue(task.getQueueName());
    }

    logger.info("Queue to be used : " + queue.getQueueName());

    if (task.getParameterName() != null && task.getParameterValue() != null) {
      logger.info("Adding Parameters to process");
      queue.add(
          TaskOptions.Builder.withUrl(task.getEndpoint())
              .param(task.getParameterName(), task.getParameterValue())
              .taskName(task.getName()));
    } else {
      logger.info("excluding Parameters to process");
      queue.add(TaskOptions.Builder.withUrl(task.getEndpoint()).taskName(task.getName()));
    }
  }
  public void deleteAudioClip(String audioClipId) throws BadRequestException {
    AudioClip audioClip = getAudioClipById(audioClipId);
    datastore.delete(KeyFactory.stringToKey(audioClipId));

    // task queue to delete the associated audio and image blobs
    Queue queue = QueueFactory.getDefaultQueue();
    queue.add(
        TaskOptions.Builder.withUrl("/rest/users/" + audioClip.getOwnerId() + "/audioClips/audio")
            .method(TaskOptions.Method.DELETE)
            .param("blobkey", audioClip.getAudioId()));
    queue.add(
        TaskOptions.Builder.withUrl("/rest/users/" + audioClip.getOwnerId() + "/audioClips/image")
            .method(TaskOptions.Method.DELETE)
            .param("blobkey", audioClip.getImageId()));
  }
Exemplo n.º 3
0
  /**
   * handles the callback from the device indicating that a new data file is available. This method
   * will call processFile to retrieve the file and persist the data to the data store it will then
   * add access points for each water point in the survey responses.
   *
   * @param req
   */
  @SuppressWarnings("rawtypes")
  private void ingestFile(TaskRequest req) {
    if (req.getFileName() != null) {
      log.info("	Task->processFile");
      ArrayList<SurveyInstance> surveyInstances =
          processFile(req.getFileName(), req.getPhoneNumber(), req.getChecksum(), req.getOffset());
      Map<Long, Survey> surveyMap = new HashMap<Long, Survey>();
      SurveyDAO surveyDao = new SurveyDAO();
      Queue summQueue = QueueFactory.getQueue("dataSummarization");
      Queue defaultQueue = QueueFactory.getDefaultQueue();
      for (SurveyInstance instance : surveyInstances) {
        Survey s = surveyMap.get(instance.getSurveyId());
        if (s == null) {
          s = surveyDao.getById(instance.getSurveyId());
          surveyMap.put(instance.getSurveyId(), s);
        }
        if (s != null && s.getRequireApproval() != null && s.getRequireApproval()) {
          // if the survey requires approval, don't run any of the
          // processors
          instance.setApprovedFlag("False");
          continue;
        } else {
          ProcessingAction pa = dispatch(instance.getKey().getId() + "");
          TaskOptions options = TaskOptions.Builder.withUrl(pa.getDispatchURL());
          Iterator it = pa.getParams().keySet().iterator();
          while (it.hasNext()) {
            options.param("key", (String) it.next());
          }
          log.info(
              "Received Task Queue calls for surveyInstanceKey: " + instance.getKey().getId() + "");
          aph.processSurveyInstance(instance.getKey().getId() + "");
          summQueue.add(
              TaskOptions.Builder.withUrl("/app_worker/datasummarization")
                  .param("objectKey", instance.getKey().getId() + "")
                  .param("type", "SurveyInstance"));
          // process the "new" domain structure

          defaultQueue.add(
              TaskOptions.Builder.withUrl("/app_worker/surveyalservlet")
                  .param(
                      SurveyalRestRequest.ACTION_PARAM, SurveyalRestRequest.INGEST_INSTANCE_ACTION)
                  .param(
                      SurveyalRestRequest.SURVEY_INSTANCE_PARAM, instance.getKey().getId() + ""));
        }
      }
    }
  }
Exemplo n.º 4
0
  public void enqueue() {
    // these tasks run on the background thread...
    ModulesService modulesApi = ModulesServiceFactory.getModulesService();
    String hostname = modulesApi.getVersionHostname(ServletConsts.BACKEND_GAE_SERVICE, null);
    task.header(ServletConsts.HOST, hostname);

    Queue queue = QueueFactory.getDefaultQueue();
    queue.add(task);
  }
Exemplo n.º 5
0
  // Executed by user menu click
  public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {

    // Build a task using the TaskOptions Builder pattern from ** above
    Queue queue = QueueFactory.getDefaultQueue();
    queue.add(TaskOptions.Builder.withUrl("/taskq_demo").method(TaskOptions.Method.POST));

    resp.getWriter().println("Task have been added to default queue...");

    resp.getWriter().println("Refresh this page to add another count task");
  }
Exemplo n.º 6
0
 public static void createCatalogBackendTask(String queueName, Date date) {
   Queue queue = QueueFactory.getQueue(queueName);
   TaskOptions taskOptions =
       TaskOptions.Builder.withUrl("/tasks/catalog").method(TaskOptions.Method.POST);
   if (date != null) {
     taskOptions.param(
         "date", DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT).format(date));
   }
   taskOptions.param("backend", Boolean.toString(true));
   queue.add(taskOptions);
 }
Exemplo n.º 7
0
  protected void doPost(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    String key = request.getParameter("keyname");
    String keyvalue = request.getParameter("value");

    // Add the task to the default queue.
    Queue queue = QueueFactory.getDefaultQueue();
    queue.add(
        TaskOptions.Builder.withUrl("/worker").param("keyname", key).param("value", keyvalue));

    response.sendRedirect("/done.html");
  }
 /**
  * puts the summarization request into the queue
  *
  * @param request
  */
 private void invokeSummarizer(DataSummarizationRequest request) {
   Queue queue = QueueFactory.getQueue(queueName);
   queue.add(
       TaskOptions.Builder.withUrl(summarizerPath)
           .param(DataSummarizationRequest.ACTION_PARAM, request.getAction())
           .param(DataSummarizationRequest.OBJECT_KEY, request.getObjectKey())
           .param(DataSummarizationRequest.OBJECT_TYPE, request.getType())
           .param(DataSummarizationRequest.OFFSET_KEY, request.getOffset().toString())
           .param(
               DataSummarizationRequest.CURSOR_PARAM,
               request.getCursor() != null ? request.getCursor() : ""));
 }
Exemplo n.º 9
0
 public static void createInitBackendTask(String queueName, Cursor cursor) {
   Queue queue = QueueFactory.getQueue(queueName);
   TaskOptions taskOptions =
       TaskOptions.Builder.withUrl("/tasks/init").method(TaskOptions.Method.POST);
   if (cursor != null && cursor.getElement() != null && cursor.getOffset() != null) {
     taskOptions
         .param("element", cursor.getElement())
         .param("offset", cursor.getOffset().toString());
   }
   if (cursor != null && cursor.backend) {
     taskOptions.param("backend", Boolean.toString(cursor.backend));
   }
   log.info("Added queue " + taskOptions.toString());
   queue.add(taskOptions);
 }
Exemplo n.º 10
0
  @Override
  protected void doGet(HttpServletRequest req, HttpServletResponse res)
      throws ServletException, IOException {

    Queue queue = QueueFactory.getDefaultQueue();

    // ボット定義をとりだし,それぞれの処理をタスクキューで実行
    for (BotDefinition bot : BotDefinition.getBots()) {
      if (!bot.hasAccessToken()) // OAuthの認証に失敗しているものは処理しない
      continue;
      // queue.add(url("/botHandler").param("botId", ""+ bot.getId()));
      queue.add(Builder.withUrl("/botHandler").param("botId", "" + bot.getId()));
      logger.info("submitted a task for botId " + bot.getId());
    }
  }
  @GET
  @Path("/monitoring/check-for-updates")
  public void execute() throws Exception {
    TaskOptions taskOptions;
    Queue queue = QueueFactory.getQueue("monitoring");

    for (Monitoring monitoring : MonitoringManager.findAll()) {
      if (!monitoring.isCompleted()) {
        taskOptions = Builder.withUrl("/admin/monitoring/check-for-updates");
        taskOptions = taskOptions.param("clientId", monitoring.getClientId());
        taskOptions = taskOptions.param("trackId", monitoring.getTrackId());
        taskOptions.method(POST);

        queue.add(taskOptions);
      }
    }
  }
Exemplo n.º 12
0
 public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
   Queue queue = QueueFactory.getQueue("pesky-slow");
   queue.add(withUrl("/cron/basketballscrape"));
 }
Exemplo n.º 13
0
 public void enqueue(String queueName) {
   // the named queue runs on the foreground thread...
   Queue queue = QueueFactory.getQueue(queueName);
   queue.add(task);
 }
Exemplo n.º 14
0
  private ArrayList<SurveyInstance> processFile(
      String fileName, String phoneNumber, String checksum, Integer offset) {
    ArrayList<SurveyInstance> surveyInstances = new ArrayList<SurveyInstance>();

    try {
      DeviceFilesDao dfDao = new DeviceFilesDao();
      URL url = new URL(DEVICE_FILE_PATH + fileName);
      BufferedInputStream bis = new BufferedInputStream(url.openStream());
      ZipInputStream zis = new ZipInputStream(bis);
      List<DeviceFiles> dfList = null;
      DeviceFiles deviceFile = null;
      dfList = dfDao.listByUri(url.toURI().toString());
      if (dfList != null && dfList.size() > 0) deviceFile = dfList.get(0);
      if (deviceFile == null) {
        deviceFile = new DeviceFiles();
      }
      deviceFile.setProcessDate(getNowDateTimeFormatted());
      deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
      deviceFile.setURI(url.toURI().toString());
      if (phoneNumber == null || phoneNumber.equals("null")) deviceFile.setPhoneNumber(null);
      else deviceFile.setPhoneNumber(phoneNumber);
      if (checksum == null || checksum.equals("null")) deviceFile.setChecksum(null);
      else deviceFile.setChecksum(checksum);
      deviceFile.setUploadDateTime(new Date());
      Date collectionDate = new Date();

      ArrayList<String> unparsedLines = null;
      try {
        unparsedLines = extractDataFromZip(zis);
      } catch (Exception iex) {
        // Error unzipping the response file

        deviceFile.setProcessedStatus(StatusCode.ERROR_INFLATING_ZIP);
        String message =
            "Error inflating device zip: " + deviceFile.getURI() + " : " + iex.getMessage();
        log.log(Level.SEVERE, message);
        deviceFile.addProcessingMessage(message);
        MailUtil.sendMail(
            FROM_ADDRESS,
            "FLOW",
            recepientList,
            "Device File Processing Error: " + fileName,
            message);
      }

      if (unparsedLines != null && unparsedLines.size() > 0) {
        if (REGION_FLAG.equals(unparsedLines.get(0))) {
          unparsedLines.remove(0);
          GeoRegionHelper grh = new GeoRegionHelper();
          grh.processRegionsSurvey(unparsedLines);
        } else {

          int lineNum = offset;
          String curId = null;
          while (lineNum < unparsedLines.size()) {
            String[] parts = unparsedLines.get(lineNum).split("\t");
            if (parts.length < 5) {
              parts = unparsedLines.get(lineNum).split(",");
            }
            if (parts.length >= 2) {
              if (curId == null) {
                curId = parts[1];
              } else {
                // if this isn't the first time through and
                // we are seeing a new id, break since we'll
                // process that in another call
                if (!curId.equals(parts[1])) {
                  break;
                }
              }
            }
            lineNum++;
          }

          Long userID = 1L;
          dfDao.save(deviceFile);
          SurveyInstance inst =
              siDao.save(
                  collectionDate, deviceFile, userID, unparsedLines.subList(offset, lineNum));
          if (inst != null) {
            // fire a survey event
            SurveyEventHelper.fireEvent(
                SurveyEventHelper.SUBMISSION_EVENT, inst.getSurveyId(), inst.getKey().getId());
            surveyInstances.add(inst);
            // TODO: HACK because we were saving so many duplicate
            // device files this way they all get the same status
            if (dfList != null) {
              for (DeviceFiles dfitem : dfList) {
                dfitem.setProcessedStatus(inst.getDeviceFile().getProcessedStatus());
              }
            }
          }
          if (lineNum < unparsedLines.size()) {
            if (inst != null) {
              StatusCode processingStatus = inst.getDeviceFile().getProcessedStatus();
              if (processingStatus.equals(StatusCode.PROCESSED_WITH_ERRORS)) {
                String message =
                    "Error in file during first processing step. Continuing to next part";
                deviceFile.addProcessingMessage(message);
                deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
              } else {
                deviceFile.addProcessingMessage(
                    "Processed " + lineNum + " lines spawning queue call");
                deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
              }
            }
            // if we haven't processed everything yet, invoke a
            // new service
            Queue queue = QueueFactory.getDefaultQueue();
            queue.add(
                TaskOptions.Builder.withUrl("/app_worker/task")
                    .param("action", "processFile")
                    .param("fileName", fileName)
                    .param("offset", lineNum + ""));
          } else {
            StatusCode status = StatusCode.PROCESSED_NO_ERRORS;
            if (deviceFile.getProcessedStatus() != null) {
              status = deviceFile.getProcessedStatus();
            }
            deviceFile.setProcessedStatus(status);
            if (dfList != null) {
              for (DeviceFiles dfitem : dfList) {
                dfitem.setProcessedStatus(status);
              }
            }
          }
        }
      } else {
        deviceFile.setProcessedStatus(StatusCode.PROCESSED_WITH_ERRORS);
        String message = "Error empty file: " + deviceFile.getURI();
        log.log(Level.SEVERE, message);
        deviceFile.addProcessingMessage(message);
        MailUtil.sendMail(
            FROM_ADDRESS,
            "FLOW",
            recepientList,
            "Device File Processing Error: " + fileName,
            DEVICE_FILE_PATH + fileName + "\n" + message);
      }

      dfDao.save(dfList);
      zis.close();
    } catch (Exception e) {
      log.log(Level.SEVERE, "Could not process data file", e);
      MailUtil.sendMail(
          FROM_ADDRESS,
          "FLOW",
          recepientList,
          "Device File Processing Error: " + fileName,
          DEVICE_FILE_PATH + fileName + "\n" + (e.getMessage() != null ? e.getMessage() : ""));
    }

    return surveyInstances;
  }
Exemplo n.º 15
0
  public static ArrayList<String> extractDataFromZip(ZipInputStream zis)
      throws IOException, SignedDataException {
    ArrayList<String> lines = new ArrayList<String>();
    String line = null;
    String surveyDataOnly = null;
    String dataSig = null;
    ZipEntry entry;
    while ((entry = zis.getNextEntry()) != null) {
      log.info("Unzipping: " + entry.getName());
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      byte[] buffer = new byte[2048];
      int size;
      while ((size = zis.read(buffer, 0, buffer.length)) != -1) {
        out.write(buffer, 0, size);
      }
      line = out.toString("UTF-8");

      if (entry.getName().endsWith("txt")) {
        if (entry.getName().equals("regions.txt")) {
          lines.add("regionFlag=true");
        } else {
          surveyDataOnly = line;
        }
        String[] linesSplit = line.split("\n");
        for (String s : linesSplit) {
          if (s.contains("\u0000")) {
            s = s.replaceAll("\u0000", "");
          }
          lines.add(s);
        }
      } else if (entry.getName().endsWith(".sig")) {
        dataSig = line.trim();
      } else {
        S3Driver s3 = new S3Driver();
        String[] imageParts = entry.getName().split("/");
        // comment out while testing locally
        try {
          // GAEImageAdapter gaeIA = new GAEImageAdapter();
          // byte[] resizedImage =
          // gaeIA.resizeImage(out.toByteArray(), 500, 500);
          // s3.uploadFile("dru-test", imageParts[1], resizedImage);
          GAEImageAdapter gaeImg = new GAEImageAdapter();
          byte[] newImage = gaeImg.resizeImage(out.toByteArray(), 500, 500);
          s3.uploadFile("dru-test", imageParts[1], newImage);
          // add queue call to resize
          Queue queue = QueueFactory.getDefaultQueue();

          queue.add(TaskOptions.Builder.withUrl("imageprocessor").param("imageURL", imageParts[1]));
          log.info("submiting image resize for imageURL: " + imageParts[1]);
        } catch (Exception ex) {
          ex.printStackTrace();
        }
        out.close();
      }
      zis.closeEntry();
    }
    // check the signature if we have it
    if (surveyDataOnly != null && dataSig != null) {
      try {
        MessageDigest sha1Digest = MessageDigest.getInstance("SHA1");
        byte[] digest = sha1Digest.digest(surveyDataOnly.getBytes("UTF-8"));
        SecretKeySpec signingKey =
            new SecretKeySpec(
                PropertyUtil.getProperty(SIGNING_KEY).getBytes("UTF-8"), SIGNING_ALGORITHM);
        Mac mac = Mac.getInstance(SIGNING_ALGORITHM);
        mac.init(signingKey);
        byte[] hmac = mac.doFinal(digest);

        String encodedHmac = com.google.gdata.util.common.util.Base64.encode(hmac);
        if (!encodedHmac.trim().equals(dataSig.trim())) {
          String allowUnsigned = PropertyUtil.getProperty(ALLOW_UNSIGNED);
          if (allowUnsigned != null && allowUnsigned.trim().equalsIgnoreCase("false")) {
            throw new SignedDataException(
                "Computed signature does not match the one submitted with the data");
          } else {
            log.warning("Signatures don't match. Processing anyway since allow unsigned is true");
          }
        }
      } catch (GeneralSecurityException e) {
        throw new SignedDataException("Could not calculate signature", e);
      }

    } else if (surveyDataOnly != null) {
      // if there is no signature, check the configuration to see if we
      // are allowed to proceed
      String allowUnsigned = PropertyUtil.getProperty(ALLOW_UNSIGNED);
      if (allowUnsigned != null && allowUnsigned.trim().equalsIgnoreCase("false")) {
        throw new SignedDataException("Datafile does not have a signature");
      }
    }

    return lines;
  }