public TaskServlet() {
DEVICE_FILE_PATH = com.gallatinsystems.common.util.PropertyUtil.getProperty("deviceZipPath");
FROM_ADDRESS = com.gallatinsystems.common.util.PropertyUtil.getProperty(EMAIL_FROM_ADDRESS_KEY);
aph = new AccessPointHelper();
siDao = new SurveyInstanceDAO();
recepientList = MailUtil.loadRecipientList();
}
private ArrayList<SurveyInstance> processFile(
String fileName, String phoneNumber, String checksum, Integer offset) {
ArrayList<SurveyInstance> surveyInstances = new ArrayList<SurveyInstance>();
try {
DeviceFilesDao dfDao = new DeviceFilesDao();
URL url = new URL(DEVICE_FILE_PATH + fileName);
BufferedInputStream bis = new BufferedInputStream(url.openStream());
ZipInputStream zis = new ZipInputStream(bis);
List<DeviceFiles> dfList = null;
DeviceFiles deviceFile = null;
dfList = dfDao.listByUri(url.toURI().toString());
if (dfList != null && dfList.size() > 0) deviceFile = dfList.get(0);
if (deviceFile == null) {
deviceFile = new DeviceFiles();
}
deviceFile.setProcessDate(getNowDateTimeFormatted());
deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
deviceFile.setURI(url.toURI().toString());
if (phoneNumber == null || phoneNumber.equals("null")) deviceFile.setPhoneNumber(null);
else deviceFile.setPhoneNumber(phoneNumber);
if (checksum == null || checksum.equals("null")) deviceFile.setChecksum(null);
else deviceFile.setChecksum(checksum);
deviceFile.setUploadDateTime(new Date());
Date collectionDate = new Date();
ArrayList<String> unparsedLines = null;
try {
unparsedLines = extractDataFromZip(zis);
} catch (Exception iex) {
// Error unzipping the response file
deviceFile.setProcessedStatus(StatusCode.ERROR_INFLATING_ZIP);
String message =
"Error inflating device zip: " + deviceFile.getURI() + " : " + iex.getMessage();
log.log(Level.SEVERE, message);
deviceFile.addProcessingMessage(message);
MailUtil.sendMail(
FROM_ADDRESS,
"FLOW",
recepientList,
"Device File Processing Error: " + fileName,
message);
}
if (unparsedLines != null && unparsedLines.size() > 0) {
if (REGION_FLAG.equals(unparsedLines.get(0))) {
unparsedLines.remove(0);
GeoRegionHelper grh = new GeoRegionHelper();
grh.processRegionsSurvey(unparsedLines);
} else {
int lineNum = offset;
String curId = null;
while (lineNum < unparsedLines.size()) {
String[] parts = unparsedLines.get(lineNum).split("\t");
if (parts.length < 5) {
parts = unparsedLines.get(lineNum).split(",");
}
if (parts.length >= 2) {
if (curId == null) {
curId = parts[1];
} else {
// if this isn't the first time through and
// we are seeing a new id, break since we'll
// process that in another call
if (!curId.equals(parts[1])) {
break;
}
}
}
lineNum++;
}
Long userID = 1L;
dfDao.save(deviceFile);
SurveyInstance inst =
siDao.save(
collectionDate, deviceFile, userID, unparsedLines.subList(offset, lineNum));
if (inst != null) {
// fire a survey event
SurveyEventHelper.fireEvent(
SurveyEventHelper.SUBMISSION_EVENT, inst.getSurveyId(), inst.getKey().getId());
surveyInstances.add(inst);
// TODO: HACK because we were saving so many duplicate
// device files this way they all get the same status
if (dfList != null) {
for (DeviceFiles dfitem : dfList) {
dfitem.setProcessedStatus(inst.getDeviceFile().getProcessedStatus());
}
}
}
if (lineNum < unparsedLines.size()) {
if (inst != null) {
StatusCode processingStatus = inst.getDeviceFile().getProcessedStatus();
if (processingStatus.equals(StatusCode.PROCESSED_WITH_ERRORS)) {
String message =
"Error in file during first processing step. Continuing to next part";
deviceFile.addProcessingMessage(message);
deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
} else {
deviceFile.addProcessingMessage(
"Processed " + lineNum + " lines spawning queue call");
deviceFile.setProcessedStatus(StatusCode.IN_PROGRESS);
}
}
// if we haven't processed everything yet, invoke a
// new service
Queue queue = QueueFactory.getDefaultQueue();
queue.add(
TaskOptions.Builder.withUrl("/app_worker/task")
.param("action", "processFile")
.param("fileName", fileName)
.param("offset", lineNum + ""));
} else {
StatusCode status = StatusCode.PROCESSED_NO_ERRORS;
if (deviceFile.getProcessedStatus() != null) {
status = deviceFile.getProcessedStatus();
}
deviceFile.setProcessedStatus(status);
if (dfList != null) {
for (DeviceFiles dfitem : dfList) {
dfitem.setProcessedStatus(status);
}
}
}
}
} else {
deviceFile.setProcessedStatus(StatusCode.PROCESSED_WITH_ERRORS);
String message = "Error empty file: " + deviceFile.getURI();
log.log(Level.SEVERE, message);
deviceFile.addProcessingMessage(message);
MailUtil.sendMail(
FROM_ADDRESS,
"FLOW",
recepientList,
"Device File Processing Error: " + fileName,
DEVICE_FILE_PATH + fileName + "\n" + message);
}
dfDao.save(dfList);
zis.close();
} catch (Exception e) {
log.log(Level.SEVERE, "Could not process data file", e);
MailUtil.sendMail(
FROM_ADDRESS,
"FLOW",
recepientList,
"Device File Processing Error: " + fileName,
DEVICE_FILE_PATH + fileName + "\n" + (e.getMessage() != null ? e.getMessage() : ""));
}
return surveyInstances;
}