/**
* handles the callback from the device indicating that a new data file is available. This method
* will call processFile to retrieve the file and persist the data to the data store it will then
* add access points for each water point in the survey responses.
*
* @param req
*/
@SuppressWarnings("rawtypes")
private void ingestFile(TaskRequest req) {
if (req.getFileName() != null) {
log.info(" Task->processFile");
ArrayList<SurveyInstance> surveyInstances =
processFile(req.getFileName(), req.getPhoneNumber(), req.getChecksum(), req.getOffset());
Map<Long, Survey> surveyMap = new HashMap<Long, Survey>();
SurveyDAO surveyDao = new SurveyDAO();
Queue summQueue = QueueFactory.getQueue("dataSummarization");
Queue defaultQueue = QueueFactory.getDefaultQueue();
for (SurveyInstance instance : surveyInstances) {
Survey s = surveyMap.get(instance.getSurveyId());
if (s == null) {
s = surveyDao.getById(instance.getSurveyId());
surveyMap.put(instance.getSurveyId(), s);
}
if (s != null && s.getRequireApproval() != null && s.getRequireApproval()) {
// if the survey requires approval, don't run any of the
// processors
instance.setApprovedFlag("False");
continue;
} else {
ProcessingAction pa = dispatch(instance.getKey().getId() + "");
TaskOptions options = TaskOptions.Builder.withUrl(pa.getDispatchURL());
Iterator it = pa.getParams().keySet().iterator();
while (it.hasNext()) {
options.param("key", (String) it.next());
}
log.info(
"Received Task Queue calls for surveyInstanceKey: " + instance.getKey().getId() + "");
aph.processSurveyInstance(instance.getKey().getId() + "");
summQueue.add(
TaskOptions.Builder.withUrl("/app_worker/datasummarization")
.param("objectKey", instance.getKey().getId() + "")
.param("type", "SurveyInstance"));
// process the "new" domain structure
defaultQueue.add(
TaskOptions.Builder.withUrl("/app_worker/surveyalservlet")
.param(
SurveyalRestRequest.ACTION_PARAM, SurveyalRestRequest.INGEST_INSTANCE_ACTION)
.param(
SurveyalRestRequest.SURVEY_INSTANCE_PARAM, instance.getKey().getId() + ""));
}
}
}
}
private ArrayList<SurveyInstance> processFile(
String fileName, String phoneNumber, String checksum, Integer offset) {
ArrayList<SurveyInstance> surveyInstances = new ArrayList<SurveyInstance>();
try {
DeviceFilesDao dfDao = new DeviceFilesDao();
URL url = new URL(DEVICE_FILE_PATH + fileName);
BufferedInputStream bis = new BufferedInputStream(url.openStream());
ZipInputStream zis = new ZipInputStream(bis);
List<DeviceFiles> dfList = null;
DeviceFiles deviceFile = null;
dfList = dfDao.listByUri(url.toURI().toString());
if (dfList != null && dfList.size() > 0) deviceFile = dfList.get(0);
if (deviceFile == null) {
deviceFile = new DeviceFiles();
}
deviceFile.setProcessDate(getNowDateTimeFormatted());
deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
deviceFile.setURI(url.toURI().toString());
if (phoneNumber == null || phoneNumber.equals("null")) deviceFile.setPhoneNumber(null);
else deviceFile.setPhoneNumber(phoneNumber);
if (checksum == null || checksum.equals("null")) deviceFile.setChecksum(null);
else deviceFile.setChecksum(checksum);
deviceFile.setUploadDateTime(new Date());
Date collectionDate = new Date();
ArrayList<String> unparsedLines = null;
try {
unparsedLines = extractDataFromZip(zis);
} catch (Exception iex) {
// Error unzipping the response file
deviceFile.setProcessedStatus(StatusCode.ERROR_INFLATING_ZIP);
String message =
"Error inflating device zip: " + deviceFile.getURI() + " : " + iex.getMessage();
log.log(Level.SEVERE, message);
deviceFile.addProcessingMessage(message);
MailUtil.sendMail(
FROM_ADDRESS,
"FLOW",
recepientList,
"Device File Processing Error: " + fileName,
message);
}
if (unparsedLines != null && unparsedLines.size() > 0) {
if (REGION_FLAG.equals(unparsedLines.get(0))) {
unparsedLines.remove(0);
GeoRegionHelper grh = new GeoRegionHelper();
grh.processRegionsSurvey(unparsedLines);
} else {
int lineNum = offset;
String curId = null;
while (lineNum < unparsedLines.size()) {
String[] parts = unparsedLines.get(lineNum).split("\t");
if (parts.length < 5) {
parts = unparsedLines.get(lineNum).split(",");
}
if (parts.length >= 2) {
if (curId == null) {
curId = parts[1];
} else {
// if this isn't the first time through and
// we are seeing a new id, break since we'll
// process that in another call
if (!curId.equals(parts[1])) {
break;
}
}
}
lineNum++;
}
Long userID = 1L;
dfDao.save(deviceFile);
SurveyInstance inst =
siDao.save(
collectionDate, deviceFile, userID, unparsedLines.subList(offset, lineNum));
if (inst != null) {
// fire a survey event
SurveyEventHelper.fireEvent(
SurveyEventHelper.SUBMISSION_EVENT, inst.getSurveyId(), inst.getKey().getId());
surveyInstances.add(inst);
// TODO: HACK because we were saving so many duplicate
// device files this way they all get the same status
if (dfList != null) {
for (DeviceFiles dfitem : dfList) {
dfitem.setProcessedStatus(inst.getDeviceFile().getProcessedStatus());
}
}
}
if (lineNum < unparsedLines.size()) {
if (inst != null) {
StatusCode processingStatus = inst.getDeviceFile().getProcessedStatus();
if (processingStatus.equals(StatusCode.PROCESSED_WITH_ERRORS)) {
String message =
"Error in file during first processing step. Continuing to next part";
deviceFile.addProcessingMessage(message);
deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
} else {
deviceFile.addProcessingMessage(
"Processed " + lineNum + " lines spawning queue call");
deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
}
}
// if we haven't processed everything yet, invoke a
// new service
Queue queue = QueueFactory.getDefaultQueue();
queue.add(
TaskOptions.Builder.withUrl("/app_worker/task")
.param("action", "processFile")
.param("fileName", fileName)
.param("offset", lineNum + ""));
} else {
StatusCode status = StatusCode.PROCESSED_NO_ERRORS;
if (deviceFile.getProcessedStatus() != null) {
status = deviceFile.getProcessedStatus();
}
deviceFile.setProcessedStatus(status);
if (dfList != null) {
for (DeviceFiles dfitem : dfList) {
dfitem.setProcessedStatus(status);
}
}
}
}
} else {
deviceFile.setProcessedStatus(StatusCode.PROCESSED_WITH_ERRORS);
String message = "Error empty file: " + deviceFile.getURI();
log.log(Level.SEVERE, message);
deviceFile.addProcessingMessage(message);
MailUtil.sendMail(
FROM_ADDRESS,
"FLOW",
recepientList,
"Device File Processing Error: " + fileName,
DEVICE_FILE_PATH + fileName + "\n" + message);
}
dfDao.save(dfList);
zis.close();
} catch (Exception e) {
log.log(Level.SEVERE, "Could not process data file", e);
MailUtil.sendMail(
FROM_ADDRESS,
"FLOW",
recepientList,
"Device File Processing Error: " + fileName,
DEVICE_FILE_PATH + fileName + "\n" + (e.getMessage() != null ? e.getMessage() : ""));
}
return surveyInstances;
}
