[9front] upas/fs: handle outlook's quirks

[9front] upas/fs: handle outlook's quirks

[risto.salminen]

Hello all,

Outlook's IMAP server often tells incorrect message
sizes.  This patch makes upas/fs handle that at least
much better than before.

If the length of a message chunk received from the server
is less than the requested chunk length, then the whole
message has been received, which means that the message
was smaller than expected, and upas/fs must adjust its
internal notion of the message length accordingly.
If this is not done, the message body as presented by
upas/fs will contain trailing garbage, that is, previous
memory contents.

The handling of a message larger than expected works
exactly in the same way as the current quirk for Gmail.
When requesting the last chunk of a message, upas/fs
increases the chunk length, so that hopefully the whole
message will be read, and then afterwards also adjusts
its internal notion of the message length accordingly.

To benefit from this patch, the desired mailbox indices
have to be recreated, by manually removing them, so that
upas/fs will then create them automatically.  Otherwise,
things will continue to work as before.

Would this be useful for the broader audience?

Thanks,
Risto

diff a8ad3fb3d00551bad97e7b10a9666821fe4b601a uncommitted
--- a/sys/src/cmd/upas/fs/cache.c
+++ b/sys/src/cmd/upas/fs/cache.c
@@ -199,7 +199,7 @@
 		return -1;
 	}
 	if(m->size - sz0)
-		l += m->size - sz0;	/* awful botch for gmail */
+		l += m->size - sz0;	/* awful botch for gmail and outlook */
 	if(expand){
 		/* grumble.  poor planning. */
 		if(m->badchars > 0)
--- a/sys/src/cmd/upas/fs/imap.c
+++ b/sys/src/cmd/upas/fs/imap.c
@@ -29,6 +29,7 @@
 	Fssl	= 1<<0,
 	Fdebug	= 1<<1,
 	Fgmail	= 1<<2,
+	Foutlook= 1<<3,
 };

 typedef struct {
@@ -331,7 +332,7 @@
 }

 static char*
-fetchrsp(Imap *imap, char *p, Mailbox *, Message *m, int idx)
+fetchrsp(Imap *imap, char *p, Mailbox *, Message *m, int idx, ulong l0)
 {
 	char *f[15], *s, *q;
 	int i, n, a;
@@ -383,10 +384,12 @@
 				return confused;
 			l = xnum(f[i + 1], '{', '}');
 			a = o + l - m->ibadchars - m->size;
-			if(a > 0){
-				assert(imap->flags & Fgmail);
+			if(l < l0 || a > 0){
+				/* message smaller or larger than expected */
+				assert((imap->flags & Fgmail) || (imap->flags & Foutlook));
 				m->size = o + l;
 				msgrealloc(m, m->size);
+				assert(m->ibadchars < m->size);
 				m->size -= m->ibadchars;
 			}
 			if(Bread(&imap->bin, m->start + o, l) != l){
@@ -441,7 +444,7 @@
  *  data or other informational lines mixed in.
  */
 static char*
-imap4resp0(Imap *imap, Mailbox *mb, Message *m)
+imap4resp0(Imap *imap, Mailbox *mb, Message *m, ulong l)
 {
 	char *e, *line, *p, *ep, *op, *q, *verb;
 	int n, idx, unexp;
@@ -508,7 +511,7 @@
 					if(ep[-1] == ')')
 						*--ep = 0;
 				}
-				if(e = fetchrsp(imap, p, mb, m, n - 1))
+				if(e = fetchrsp(imap, p, mb, m, n - 1, l))
 					eprint("imap: fetchrsp: %s\n", e);
 				if(n > 0 && n <= imap->muid && n > imap->nuid)
 					imap->nuid = n;
@@ -553,7 +556,7 @@
 static char*
 imap4resp(Imap *i)
 {
-	return imap4resp0(i, 0, 0);
+	return imap4resp0(i, 0, 0, 0);
 }

 static int
@@ -590,7 +593,7 @@
 	if(p > buf){
 		p[-1] = 0;
 		imap4cmd(imap, "uid store %lud flags (%s)", (ulong)m->imapuid, buf);
-		imap4resp0(imap, mb, m);
+		imap4resp0(imap, mb, m, 0);
 	}
 }

@@ -861,6 +864,16 @@
 	return l;
 }

+/* outlook also lies about message sizes */
+static ulong
+outlookdiscount(Message *m, uvlong o, ulong l)
+{
+	if((m->cstate&Cidx) == 0)
+	if(o + l == m->size)
+		return l + 100 + (o + l)/3;
+	return l;
+}
+
 static int
 imap4fetch(Mailbox *mb, Message *m, uvlong o, ulong l)
 {
@@ -870,9 +883,11 @@
 	imap = mb->aux;
 	if(imap->flags & Fgmail)
 		l = gmaildiscount(m, o, l);
+	if(imap->flags & Foutlook)
+		l = outlookdiscount(m, o, l);
 	idprint(imap, "uid fetch %lud (flags body.peek[]<%llud.%lud>)\n", (ulong)m->imapuid, o, l);
 	imap4cmd(imap, "uid fetch %lud (flags body.peek[]<%llud.%lud>)", (ulong)m->imapuid, o, l);
-	resp = imap4resp0(imap, mb, m);
+	resp = imap4resp0(imap, mb, m, (m->cstate&Cidx) ? 0 : l);
 	if(!isokay(resp)){
 		eprint("imap: imap fetch failed: %s\n", resp);
 		return -1;
@@ -1165,6 +1180,8 @@
 	imap->host = f[2];
 	if(strstr(imap->host, "gmail.com"))
 		imap->flags |= Fgmail;
+	if(strstr(imap->host, "outlook.office365.com"))
+		imap->flags |= Foutlook;
 	imap->refreshtime = 60;
 	if(nf < 4)
 		imap->user = nil;

Re: [9front] upas/fs: handle outlook's quirks

[ori]

Quoth risto.salminen@gmx.com:
> The handling of a message larger than expected works
> exactly in the same way as the current quirk for Gmail.
> When requesting the last chunk of a message, upas/fs
> increases the chunk length, so that hopefully the whole
> message will be read, and then afterwards also adjusts
> its internal notion of the message length accordingly.
> 
> To benefit from this patch, the desired mailbox indices
> have to be recreated, by manually removing them, so that
> upas/fs will then create them automatically.  Otherwise,
> things will continue to work as before.
> 
> Would this be useful for the broader audience?

Yes, though I'm probably going to see if I can
make sense of some other imap implementations and
see how fetching is handled differently there before
I commit this.

Give me a couple of days.

Re: [9front] upas/fs: handle outlook's quirks

[Risto Salminen]

ori wrote:
> Give me a couple of days.

Sure, there is no hurry.

> Yes, though I'm probably going to see if I can
> make sense of some other imap implementations and
> see how fetching is handled differently there before
> I commit this.

I took a quick look at isync and mutt, and both seem to
fetch either the header section or the whole body of
the message at once, without relying on the rfc822.size
attribute that Gmail and Outlook apparently do not tell
correctly.  Therefore they work correctly with any kind
of an IMAP server.

On the other hand, upas/fs fetches the message in
multiple substrings, whose sizes are calculated based
on the rfc822.size attribute, which means that there has
to be special handling for quirky servers that return
incorrect rfc822.size.

The message is fetched in substrings apparently because
then it is possible to fetch only the headers if the
body is not yet needed, which saves bandwidth and memory
and consequently also time.  The same effect should be
achievable by fetching only the header section as the
IMAP specification allows.  Of course, then one would
have to rely on the server implementing that correctly.
In any case, at least fetching the whole message, without
specifying substring limits, at once should work properly
with any IMAP server and indeed return the whole message.

In my opinion, the correct solution for making upas/fs
work with any IMAP server without any special quirks
would be to convert it from the current substring
strategy to fetch either the header section or the whole
body of the message at once.  This would however seem to
require quite extensive changes.  I could try to work on
that in some timeframe though, if that is deemed
appropriate.

Thanks,
Risto