Re: Huge memory consumption on accessing large newsgroup

[Daniel Pittman <daniel@rimspace.net>]

[-- Attachment #1: Type: text/plain, Size: 1051 bytes --]

Daniel Pittman <daniel@rimspace.net> writes:

[...]

> (And because this has been a stupidly annoying couple of week in other
>  areas, and because this is nice simple and essentially stress-free work
>  I am getting tempted to fix it myself.
>
>  So, maybe inversion lists were the way to get the code fixed after all,
>  if not quite so directly as expected. ;) 

...and that turns out to be true.  Here is a patch against current CVS
that addresses the low hanging fruit in the area.  I will tackle the
more complex examples some time soon, or someone else can, and we will
see where we go next.

This has only been very briefly looked at and isn't thoroughly tested at
this stage.  It should all be correct but could really do with a third
party review for correctness before committing to CVS or anything.

        Daniel
-- 
Daniel Pittman <daniel@cybersource.com.au>           Phone: 03 9621 2377
Level 4, 10 Queen St, Melbourne             Web: http://www.cyber.com.au
Cybersource: Australia's Leading Linux and Open Source Solutions Company


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: keep-ranges-compressed-1.patch --]
[-- Type: text/x-diff, Size: 5866 bytes --]

? keep-ranges-compressed-1.patch
Index: lisp/gnus-draft.el
===================================================================
RCS file: /usr/local/cvsroot/gnus/lisp/gnus-draft.el,v
retrieving revision 7.16
diff -u -u -r7.16 gnus-draft.el
--- lisp/gnus-draft.el	24 Mar 2007 19:50:00 -0000	7.16
+++ lisp/gnus-draft.el	2 Oct 2007 13:32:14 -0000
@@ -212,15 +212,14 @@
     (gnus-activate-group "nndraft:queue")
     (save-excursion
       (let* ((articles (nndraft-articles))
-	     (unsendable (gnus-uncompress-range
-			  (cdr (assq 'unsend
-				     (gnus-info-marks
-				      (gnus-get-info "nndraft:queue"))))))
+	     (unsendable (cdr (assq 'unsend
+				    (gnus-info-marks
+				     (gnus-get-info "nndraft:queue"))))))
 	     (gnus-posting-styles nil)
 	     (total (length articles))
 	     article)
 	(while (setq article (pop articles))
-	  (unless (memq article unsendable)
+	  (unless (gnus-member-of-range article unsendable)
 	    (let ((message-sending-message
 		   (format "Sending message %d of %d..."
 			   (- total (length articles)) total)))
Index: lisp/gnus-group.el
===================================================================
RCS file: /usr/local/cvsroot/gnus/lisp/gnus-group.el,v
retrieving revision 7.97
diff -u -u -r7.97 gnus-group.el
--- lisp/gnus-group.el	3 Aug 2007 06:20:58 -0000	7.97
+++ lisp/gnus-group.el	2 Oct 2007 13:32:16 -0000
@@ -1496,10 +1496,10 @@
 	 (active (gnus-active group)))
     (if (not active)
 	0
-      (length (gnus-uncompress-range
-	       (gnus-range-difference
-		(gnus-range-difference (list active) (gnus-info-read info))
-		seen))))))
+      (gnus-range-length
+       (gnus-range-difference
+	(gnus-range-difference (list active) (gnus-info-read info))
+	seen)))))
 
 ;; Moving through the Group buffer (in topic mode) e.g. with C-n doesn't
 ;; update the state (enabled/disabled) of the icon `gnus-group-describe-group'
@@ -4406,9 +4406,8 @@
 		(setcar (nthcdr 3 info)
 			(gnus-delete-alist type (car marked)))
 	      (setcdr m (gnus-compress-sequence articles t)))
-	  (setcdr m (gnus-compress-sequence
-		     (sort (nconc (gnus-uncompress-range (cdr m))
-				  (copy-sequence articles)) '<) t))))))
+	  (setcdr m (gnus-range-add (cdr m)
+				    (sort (copy-sequence articles) '<)))))))
 
 (defun gnus-add-mark (group mark article)
   "Mark ARTICLE in GROUP with MARK, whether the group is displayed or not."
Index: lisp/gnus-move.el
===================================================================
RCS file: /usr/local/cvsroot/gnus/lisp/gnus-move.el,v
retrieving revision 7.9
diff -u -u -r7.9 gnus-move.el
--- lisp/gnus-move.el	24 Jan 2007 07:15:37 -0000	7.9
+++ lisp/gnus-move.el	2 Oct 2007 13:32:18 -0000
@@ -91,8 +91,8 @@
 	;; Then we read the headers from the `from-server'.
 	(when (and (gnus-request-group group nil from-server)
 		   (gnus-active group)
-		   (gnus-uncompress-range
-		    (gnus-active group))
+		   ;; Should this simply test for an empty or nil range?
+		   (gnus-range-length (gnus-active group))
 		   (setq type (gnus-retrieve-headers
 			       (gnus-uncompress-range
 				(gnus-active group))
Index: lisp/gnus-start.el
===================================================================
RCS file: /usr/local/cvsroot/gnus/lisp/gnus-start.el,v
retrieving revision 7.54
diff -u -u -r7.54 gnus-start.el
--- lisp/gnus-start.el	7 Sep 2007 02:52:27 -0000	7.54
+++ lisp/gnus-start.el	2 Oct 2007 13:32:18 -0000
@@ -2376,8 +2376,7 @@
 	   info
 	   (gnus-add-to-range
 	    (gnus-info-read info)
-	    (nconc (gnus-uncompress-range dormant)
-		   (gnus-uncompress-range ticked)))))))))
+	    (gnus-range-add dormant ticked))))))))
 
 (defun gnus-load (file)
   "Load FILE, but in such a way that read errors can be reported."
Index: lisp/gnus-sum.el
===================================================================
RCS file: /usr/local/cvsroot/gnus/lisp/gnus-sum.el,v
retrieving revision 7.201
diff -u -u -r7.201 gnus-sum.el
--- lisp/gnus-sum.el	27 Sep 2007 21:22:18 -0000	7.201
+++ lisp/gnus-sum.el	2 Oct 2007 13:32:24 -0000
@@ -5747,6 +5747,8 @@
       (setq articles (cdr articles)))
     out))
 
+;; REVISIT: Never used and, given the other memory use issues, probably best
+;; if it stays that way.  Should, I think, be removed.  --dp, 2007-10-02
 (defun gnus-uncompress-marks (marks)
   "Uncompress the mark ranges in MARKS."
   (let ((uncompressed '(score bookmark))
@@ -6765,6 +6767,8 @@
 	 (gnus-list-range-difference
 	  (gnus-list-range-difference
 	   (gnus-sorted-complement
+	    ;; REVISIT: This needs `gnus-range-complement' implemented
+	    ;; and tested, then we can drop the expansion of lists here.
 	    (gnus-uncompress-range
 	     (if gnus-newsgroup-maximum-articles
 		 (cons (max (car active)
@@ -6772,7 +6776,7 @@
 			       gnus-newsgroup-maximum-articles
 			       -1))
 		       (cdr active))
-	       active))
+	       active)
 	    (gnus-list-of-unread-articles group))
 	   (cdr (assq 'dormant marked)))
 	  (cdr (assq 'tick marked))))))
Index: lisp/nnimap.el
===================================================================
RCS file: /usr/local/cvsroot/gnus/lisp/nnimap.el,v
retrieving revision 7.36
diff -u -u -r7.36 nnimap.el
--- lisp/nnimap.el	17 Aug 2007 11:09:00 -0000	7.36
+++ lisp/nnimap.el	2 Oct 2007 13:32:27 -0000
@@ -634,14 +634,13 @@
 	(imap-search
 	 (concat "UID "
 		 (imap-range-to-message-set
-		  (gnus-compress-sequence
-		   (append (gnus-uncompress-sequence
-			    (and fetch-old
-				 (cons (if (numberp fetch-old)
-					   (max 1 (- (car articles) fetch-old))
-					 1)
-				       (1- (car articles)))))
-			   articles)))))
+		  (gnus-range-add
+		   (and fetch-old
+			(cons (if (numberp fetch-old)
+				  (max 1 (- (car articles) fetch-old))
+				1)
+			      (1- (car articles))))
+		   articles))))
       (mapcar (lambda (msgid)
 		(imap-search
 		 (format "HEADER Message-Id \"%s\"" msgid)))