예제 #1
0
  /**
   * handles the callback from the device indicating that a new data file is available. This method
   * will call processFile to retrieve the file and persist the data to the data store it will then
   * add access points for each water point in the survey responses.
   *
   * @param req
   */
  @SuppressWarnings("rawtypes")
  private void ingestFile(TaskRequest req) {
    if (req.getFileName() != null) {
      log.info("	Task->processFile");
      ArrayList<SurveyInstance> surveyInstances =
          processFile(req.getFileName(), req.getPhoneNumber(), req.getChecksum(), req.getOffset());
      Map<Long, Survey> surveyMap = new HashMap<Long, Survey>();
      SurveyDAO surveyDao = new SurveyDAO();
      Queue summQueue = QueueFactory.getQueue("dataSummarization");
      Queue defaultQueue = QueueFactory.getDefaultQueue();
      for (SurveyInstance instance : surveyInstances) {
        Survey s = surveyMap.get(instance.getSurveyId());
        if (s == null) {
          s = surveyDao.getById(instance.getSurveyId());
          surveyMap.put(instance.getSurveyId(), s);
        }
        if (s != null && s.getRequireApproval() != null && s.getRequireApproval()) {
          // if the survey requires approval, don't run any of the
          // processors
          instance.setApprovedFlag("False");
          continue;
        } else {
          ProcessingAction pa = dispatch(instance.getKey().getId() + "");
          TaskOptions options = TaskOptions.Builder.withUrl(pa.getDispatchURL());
          Iterator it = pa.getParams().keySet().iterator();
          while (it.hasNext()) {
            options.param("key", (String) it.next());
          }
          log.info(
              "Received Task Queue calls for surveyInstanceKey: " + instance.getKey().getId() + "");
          aph.processSurveyInstance(instance.getKey().getId() + "");
          summQueue.add(
              TaskOptions.Builder.withUrl("/app_worker/datasummarization")
                  .param("objectKey", instance.getKey().getId() + "")
                  .param("type", "SurveyInstance"));
          // process the "new" domain structure

          defaultQueue.add(
              TaskOptions.Builder.withUrl("/app_worker/surveyalservlet")
                  .param(
                      SurveyalRestRequest.ACTION_PARAM, SurveyalRestRequest.INGEST_INSTANCE_ACTION)
                  .param(
                      SurveyalRestRequest.SURVEY_INSTANCE_PARAM, instance.getKey().getId() + ""));
        }
      }
    }
  }
예제 #2
0
  private ArrayList<SurveyInstance> processFile(
      String fileName, String phoneNumber, String checksum, Integer offset) {
    ArrayList<SurveyInstance> surveyInstances = new ArrayList<SurveyInstance>();

    try {
      DeviceFilesDao dfDao = new DeviceFilesDao();
      URL url = new URL(DEVICE_FILE_PATH + fileName);
      BufferedInputStream bis = new BufferedInputStream(url.openStream());
      ZipInputStream zis = new ZipInputStream(bis);
      List<DeviceFiles> dfList = null;
      DeviceFiles deviceFile = null;
      dfList = dfDao.listByUri(url.toURI().toString());
      if (dfList != null && dfList.size() > 0) deviceFile = dfList.get(0);
      if (deviceFile == null) {
        deviceFile = new DeviceFiles();
      }
      deviceFile.setProcessDate(getNowDateTimeFormatted());
      deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
      deviceFile.setURI(url.toURI().toString());
      if (phoneNumber == null || phoneNumber.equals("null")) deviceFile.setPhoneNumber(null);
      else deviceFile.setPhoneNumber(phoneNumber);
      if (checksum == null || checksum.equals("null")) deviceFile.setChecksum(null);
      else deviceFile.setChecksum(checksum);
      deviceFile.setUploadDateTime(new Date());
      Date collectionDate = new Date();

      ArrayList<String> unparsedLines = null;
      try {
        unparsedLines = extractDataFromZip(zis);
      } catch (Exception iex) {
        // Error unzipping the response file

        deviceFile.setProcessedStatus(StatusCode.ERROR_INFLATING_ZIP);
        String message =
            "Error inflating device zip: " + deviceFile.getURI() + " : " + iex.getMessage();
        log.log(Level.SEVERE, message);
        deviceFile.addProcessingMessage(message);
        MailUtil.sendMail(
            FROM_ADDRESS,
            "FLOW",
            recepientList,
            "Device File Processing Error: " + fileName,
            message);
      }

      if (unparsedLines != null && unparsedLines.size() > 0) {
        if (REGION_FLAG.equals(unparsedLines.get(0))) {
          unparsedLines.remove(0);
          GeoRegionHelper grh = new GeoRegionHelper();
          grh.processRegionsSurvey(unparsedLines);
        } else {

          int lineNum = offset;
          String curId = null;
          while (lineNum < unparsedLines.size()) {
            String[] parts = unparsedLines.get(lineNum).split("\t");
            if (parts.length < 5) {
              parts = unparsedLines.get(lineNum).split(",");
            }
            if (parts.length >= 2) {
              if (curId == null) {
                curId = parts[1];
              } else {
                // if this isn't the first time through and
                // we are seeing a new id, break since we'll
                // process that in another call
                if (!curId.equals(parts[1])) {
                  break;
                }
              }
            }
            lineNum++;
          }

          Long userID = 1L;
          dfDao.save(deviceFile);
          SurveyInstance inst =
              siDao.save(
                  collectionDate, deviceFile, userID, unparsedLines.subList(offset, lineNum));
          if (inst != null) {
            // fire a survey event
            SurveyEventHelper.fireEvent(
                SurveyEventHelper.SUBMISSION_EVENT, inst.getSurveyId(), inst.getKey().getId());
            surveyInstances.add(inst);
            // TODO: HACK because we were saving so many duplicate
            // device files this way they all get the same status
            if (dfList != null) {
              for (DeviceFiles dfitem : dfList) {
                dfitem.setProcessedStatus(inst.getDeviceFile().getProcessedStatus());
              }
            }
          }
          if (lineNum < unparsedLines.size()) {
            if (inst != null) {
              StatusCode processingStatus = inst.getDeviceFile().getProcessedStatus();
              if (processingStatus.equals(StatusCode.PROCESSED_WITH_ERRORS)) {
                String message =
                    "Error in file during first processing step. Continuing to next part";
                deviceFile.addProcessingMessage(message);
                deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
              } else {
                deviceFile.addProcessingMessage(
                    "Processed " + lineNum + " lines spawning queue call");
                deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
              }
            }
            // if we haven't processed everything yet, invoke a
            // new service
            Queue queue = QueueFactory.getDefaultQueue();
            queue.add(
                TaskOptions.Builder.withUrl("/app_worker/task")
                    .param("action", "processFile")
                    .param("fileName", fileName)
                    .param("offset", lineNum + ""));
          } else {
            StatusCode status = StatusCode.PROCESSED_NO_ERRORS;
            if (deviceFile.getProcessedStatus() != null) {
              status = deviceFile.getProcessedStatus();
            }
            deviceFile.setProcessedStatus(status);
            if (dfList != null) {
              for (DeviceFiles dfitem : dfList) {
                dfitem.setProcessedStatus(status);
              }
            }
          }
        }
      } else {
        deviceFile.setProcessedStatus(StatusCode.PROCESSED_WITH_ERRORS);
        String message = "Error empty file: " + deviceFile.getURI();
        log.log(Level.SEVERE, message);
        deviceFile.addProcessingMessage(message);
        MailUtil.sendMail(
            FROM_ADDRESS,
            "FLOW",
            recepientList,
            "Device File Processing Error: " + fileName,
            DEVICE_FILE_PATH + fileName + "\n" + message);
      }

      dfDao.save(dfList);
      zis.close();
    } catch (Exception e) {
      log.log(Level.SEVERE, "Could not process data file", e);
      MailUtil.sendMail(
          FROM_ADDRESS,
          "FLOW",
          recepientList,
          "Device File Processing Error: " + fileName,
          DEVICE_FILE_PATH + fileName + "\n" + (e.getMessage() != null ? e.getMessage() : ""));
    }

    return surveyInstances;
  }