/**
* Fetches and parses the message envelopes for the supplied messages. The idea is to have this be
* recursive so that we do a series of medium calls instead of one large massive call or a large
* number of smaller calls. Call it a happy balance
*/
private void fetchEnvelope(
List<WebDavMessage> startMessages, MessageRetrievalListener<WebDavMessage> listener)
throws MessagingException {
Map<String, String> headers = new HashMap<String, String>();
String messageBody;
String[] uids;
List<WebDavMessage> messages = new ArrayList<WebDavMessage>(10);
if (startMessages == null || startMessages.isEmpty()) {
return;
}
if (startMessages.size() > 10) {
List<WebDavMessage> newMessages = new ArrayList<WebDavMessage>(startMessages.size() - 10);
for (int i = 0, count = startMessages.size(); i < count; i++) {
if (i < 10) {
messages.add(i, startMessages.get(i));
} else {
newMessages.add(i - 10, startMessages.get(i));
}
}
fetchEnvelope(newMessages, listener);
} else {
messages.addAll(startMessages);
}
uids = new String[messages.size()];
for (int i = 0, count = messages.size(); i < count; i++) {
uids[i] = messages.get(i).getUid();
}
messageBody = store.getMessageEnvelopeXml(uids);
headers.put("Brief", "t");
DataSet dataset = store.processRequest(this.mFolderUrl, "SEARCH", messageBody, headers);
Map<String, ParsedMessageEnvelope> envelopes = dataset.getMessageEnvelopes();
int count = messages.size();
for (int i = messages.size() - 1; i >= 0; i--) {
WebDavMessage message = messages.get(i);
if (listener != null) {
listener.messageStarted(messages.get(i).getUid(), i, count);
}
ParsedMessageEnvelope envelope = envelopes.get(message.getUid());
if (envelope != null) {
message.setNewHeaders(envelope);
message.setFlagInternal(Flag.SEEN, envelope.getReadStatus());
} else {
Log.e(LOG_TAG, "Asked to get metadata for a non-existent message: " + message.getUid());
}
if (listener != null) {
listener.messageFinished(messages.get(i), i, count);
}
}
}
@Override
public List<WebDavMessage> getMessages(
int start, int end, Date earliestDate, MessageRetrievalListener<WebDavMessage> listener)
throws MessagingException {
List<WebDavMessage> messages = new ArrayList<WebDavMessage>();
String[] uids;
Map<String, String> headers = new HashMap<String, String>();
int uidsLength;
String messageBody;
int prevStart = start;
/** Reverse the message range since 0 index is newest */
start = this.mMessageCount - end;
end = start + (end - prevStart);
if (start < 0 || end < 0 || end < start) {
throw new MessagingException(
String.format(Locale.US, "Invalid message set %d %d", start, end));
}
if (start == 0 && end < 10) {
end = 10;
}
/** Verify authentication */
messageBody = store.getMessagesXml();
headers.put("Brief", "t");
headers.put("Range", "rows=" + start + "-" + end);
DataSet dataset = store.processRequest(this.mFolderUrl, "SEARCH", messageBody, headers);
uids = dataset.getUids();
Map<String, String> uidToUrl = dataset.getUidToUrl();
uidsLength = uids.length;
for (int i = 0; i < uidsLength; i++) {
if (listener != null) {
listener.messageStarted(uids[i], i, uidsLength);
}
WebDavMessage message = new WebDavMessage(uids[i], this);
message.setUrl(uidToUrl.get(uids[i]));
messages.add(message);
if (listener != null) {
listener.messageFinished(message, i, uidsLength);
}
}
return messages;
}
/*
* Given a query string, actually do the query for the messages and
* call the MessageRetrievalListener for each one
*/
List<LocalMessage> getMessages(
final MessageRetrievalListener<LocalMessage> listener,
final LocalFolder folder,
final String queryString,
final String[] placeHolders)
throws MessagingException {
final List<LocalMessage> messages = new ArrayList<>();
final int j =
database.execute(
false,
new DbCallback<Integer>() {
@Override
public Integer doDbWork(final SQLiteDatabase db) throws WrappedException {
Cursor cursor = null;
int i = 0;
try {
cursor = db.rawQuery(queryString + " LIMIT 10", placeHolders);
while (cursor.moveToNext()) {
LocalMessage message = new LocalMessage(LocalStore.this, null, folder);
message.populateFromGetMessageCursor(cursor);
messages.add(message);
if (listener != null) {
listener.messageFinished(message, i, -1);
}
i++;
}
cursor.close();
cursor = db.rawQuery(queryString + " LIMIT -1 OFFSET 10", placeHolders);
while (cursor.moveToNext()) {
LocalMessage message = new LocalMessage(LocalStore.this, null, folder);
message.populateFromGetMessageCursor(cursor);
messages.add(message);
if (listener != null) {
listener.messageFinished(message, i, -1);
}
i++;
}
} catch (Exception e) {
Log.d(K9.LOG_TAG, "Got an exception", e);
} finally {
Utility.closeQuietly(cursor);
}
return i;
}
});
if (listener != null) {
listener.messagesFinished(j);
}
return Collections.unmodifiableList(messages);
}
@Override
public List<Pop3Message> getMessages(
int start, int end, Date earliestDate, MessageRetrievalListener<Pop3Message> listener)
throws MessagingException {
if (start < 1 || end < 1 || end < start) {
throw new MessagingException(
String.format(Locale.US, "Invalid message set %d %d", start, end));
}
try {
indexMsgNums(start, end);
} catch (IOException ioe) {
throw new MessagingException("getMessages", ioe);
}
List<Pop3Message> messages = new ArrayList<Pop3Message>();
int i = 0;
for (int msgNum = start; msgNum <= end; msgNum++) {
Pop3Message message = mMsgNumToMsgMap.get(msgNum);
if (message == null) {
/*
* There could be gaps in the message numbers or malformed
* responses which lead to "gaps" in mMsgNumToMsgMap.
*
* See issue 2252
*/
continue;
}
if (listener != null) {
listener.messageStarted(message.getUid(), i++, (end - start) + 1);
}
messages.add(message);
if (listener != null) {
listener.messageFinished(message, i++, (end - start) + 1);
}
}
return messages;
}
/**
* Fetches the full messages or up to {@param lines} lines and passes them to the message parser.
*/
private void fetchMessages(
List<WebDavMessage> messages, MessageRetrievalListener<WebDavMessage> listener, int lines)
throws MessagingException {
WebDavHttpClient httpclient;
httpclient = store.getHttpClient();
/** We can't hand off to processRequest() since we need the stream to parse. */
for (int i = 0, count = messages.size(); i < count; i++) {
WebDavMessage wdMessage = messages.get(i);
int statusCode = 0;
if (listener != null) {
listener.messageStarted(wdMessage.getUid(), i, count);
}
/**
* If fetch is called outside of the initial list (ie, a locally stored message), it may not
* have a URL associated. Verify and fix that
*/
if (wdMessage.getUrl().equals("")) {
wdMessage.setUrl(getMessageUrls(new String[] {wdMessage.getUid()}).get(wdMessage.getUid()));
Log.i(
LOG_TAG,
"Fetching messages with UID = '"
+ wdMessage.getUid()
+ "', URL = '"
+ wdMessage.getUrl()
+ "'");
if (wdMessage.getUrl().equals("")) {
throw new MessagingException("Unable to get URL for message");
}
}
try {
Log.i(
LOG_TAG,
"Fetching message with UID = '"
+ wdMessage.getUid()
+ "', URL = '"
+ wdMessage.getUrl()
+ "'");
HttpGet httpget = new HttpGet(new URI(wdMessage.getUrl()));
HttpResponse response;
HttpEntity entity;
httpget.setHeader("translate", "f");
if (store.getAuthentication() == WebDavConstants.AUTH_TYPE_BASIC) {
httpget.setHeader("Authorization", store.getAuthString());
}
response = httpclient.executeOverride(httpget, store.getContext());
statusCode = response.getStatusLine().getStatusCode();
entity = response.getEntity();
if (statusCode < 200 || statusCode > 300) {
throw new IOException(
"Error during with code "
+ statusCode
+ " during fetch: "
+ response.getStatusLine().toString());
}
if (entity != null) {
InputStream istream = null;
StringBuilder buffer = new StringBuilder();
String tempText;
String resultText;
BufferedReader reader = null;
int currentLines = 0;
try {
istream = WebDavHttpClient.getUngzippedContent(entity);
if (lines != -1) {
// Convert the ungzipped input stream into a StringBuilder
// containing the given line count
reader = new BufferedReader(new InputStreamReader(istream), 8192);
while ((tempText = reader.readLine()) != null && (currentLines < lines)) {
buffer.append(tempText).append("\r\n");
currentLines++;
}
IOUtils.closeQuietly(istream);
resultText = buffer.toString();
istream = new ByteArrayInputStream(resultText.getBytes("UTF-8"));
}
// Parse either the entire message stream, or a stream of the given lines
wdMessage.parse(istream);
} catch (IOException ioe) {
Log.e(
LOG_TAG,
"IOException: "
+ ioe.getMessage()
+ "\nTrace: "
+ WebDavUtils.processException(ioe));
throw new MessagingException("I/O Error", ioe);
} finally {
IOUtils.closeQuietly(reader);
IOUtils.closeQuietly(istream);
}
} else {
Log.v(LOG_TAG, "Empty response");
}
} catch (IllegalArgumentException iae) {
Log.e(
LOG_TAG,
"IllegalArgumentException caught "
+ iae
+ "\nTrace: "
+ WebDavUtils.processException(iae));
throw new MessagingException("IllegalArgumentException caught", iae);
} catch (URISyntaxException use) {
Log.e(
LOG_TAG,
"URISyntaxException caught " + use + "\nTrace: " + WebDavUtils.processException(use));
throw new MessagingException("URISyntaxException caught", use);
} catch (IOException ioe) {
Log.e(
LOG_TAG,
"Non-success response code loading message, response code was "
+ statusCode
+ "\nURL: "
+ wdMessage.getUrl()
+ "\nError: "
+ ioe.getMessage()
+ "\nTrace: "
+ WebDavUtils.processException(ioe));
throw new MessagingException("Failure code " + statusCode, ioe);
}
if (listener != null) {
listener.messageFinished(wdMessage, i, count);
}
}
}
private void fetchEnvelope(
List<Pop3Message> messages, MessageRetrievalListener<Pop3Message> listener)
throws IOException, MessagingException {
int unsizedMessages = 0;
for (Message message : messages) {
if (message.getSize() == -1) {
unsizedMessages++;
}
}
if (unsizedMessages == 0) {
return;
}
if (unsizedMessages < 50 && mMessageCount > 5000) {
/*
* In extreme cases we'll do a command per message instead of a bulk request
* to hopefully save some time and bandwidth.
*/
for (int i = 0, count = messages.size(); i < count; i++) {
Pop3Message message = messages.get(i);
if (listener != null) {
listener.messageStarted(message.getUid(), i, count);
}
String response =
executeSimpleCommand(
String.format(
Locale.US, LIST_COMMAND + " %d", mUidToMsgNumMap.get(message.getUid())));
String[] listParts = response.split(" ");
// int msgNum = Integer.parseInt(listParts[1]);
int msgSize = Integer.parseInt(listParts[2]);
message.setSize(msgSize);
if (listener != null) {
listener.messageFinished(message, i, count);
}
}
} else {
Set<String> msgUidIndex = new HashSet<String>();
for (Message message : messages) {
msgUidIndex.add(message.getUid());
}
int i = 0, count = messages.size();
String response = executeSimpleCommand(LIST_COMMAND);
while ((response = readLine()) != null) {
if (response.equals(".")) {
break;
}
String[] listParts = response.split(" ");
int msgNum = Integer.parseInt(listParts[0]);
int msgSize = Integer.parseInt(listParts[1]);
Pop3Message pop3Message = mMsgNumToMsgMap.get(msgNum);
if (pop3Message != null && msgUidIndex.contains(pop3Message.getUid())) {
if (listener != null) {
listener.messageStarted(pop3Message.getUid(), i, count);
}
pop3Message.setSize(msgSize);
if (listener != null) {
listener.messageFinished(pop3Message, i, count);
}
i++;
}
}
}
}
/**
* Fetch the items contained in the FetchProfile into the given set of Messages in as efficient
* a manner as possible.
*
* @param messages
* @param fp
* @throws MessagingException
*/
@Override
public void fetch(
List<Pop3Message> messages, FetchProfile fp, MessageRetrievalListener<Pop3Message> listener)
throws MessagingException {
if (messages == null || messages.isEmpty()) {
return;
}
List<String> uids = new ArrayList<String>();
for (Message message : messages) {
uids.add(message.getUid());
}
try {
indexUids(uids);
} catch (IOException ioe) {
throw new MessagingException("fetch", ioe);
}
try {
if (fp.contains(FetchProfile.Item.ENVELOPE)) {
/*
* We pass the listener only if there are other things to do in the
* FetchProfile. Since fetchEnvelop works in bulk and eveything else
* works one at a time if we let fetchEnvelope send events the
* event would get sent twice.
*/
fetchEnvelope(messages, fp.size() == 1 ? listener : null);
}
} catch (IOException ioe) {
throw new MessagingException("fetch", ioe);
}
for (int i = 0, count = messages.size(); i < count; i++) {
Pop3Message pop3Message = messages.get(i);
try {
if (listener != null && !fp.contains(FetchProfile.Item.ENVELOPE)) {
listener.messageStarted(pop3Message.getUid(), i, count);
}
if (fp.contains(FetchProfile.Item.BODY)) {
fetchBody(pop3Message, -1);
} else if (fp.contains(FetchProfile.Item.BODY_SANE)) {
/*
* To convert the suggested download size we take the size
* divided by the maximum line size (76).
*/
if (mStoreConfig.getMaximumAutoDownloadMessageSize() > 0) {
fetchBody(pop3Message, (mStoreConfig.getMaximumAutoDownloadMessageSize() / 76));
} else {
fetchBody(pop3Message, -1);
}
} else if (fp.contains(FetchProfile.Item.STRUCTURE)) {
/*
* If the user is requesting STRUCTURE we are required to set the body
* to null since we do not support the function.
*/
pop3Message.setBody(null);
}
if (listener != null && !(fp.contains(FetchProfile.Item.ENVELOPE) && fp.size() == 1)) {
listener.messageFinished(pop3Message, i, count);
}
} catch (IOException ioe) {
throw new MessagingException("Unable to fetch message", ioe);
}
}
}