Re: CRM114 Mailfilter and spam.el

[asjo@koldfront.dk (Adam Sjøgren)]

[-- Attachment #1: Type: text/plain, Size: 1048 bytes --]

On Thu, 01 Apr 2004 22:54:06 +0200, Adam wrote:

> So, I've begun to take a stab at switcing from spam-stat.el to
> CRM114 Mailfilter and spam.el.

Attached is a patch for spam.el that adds support for CRM114
Mailfilter (until the general external statistical processor interface
is designed/implemented).

It is basically the bogofilter stuff copied, search/replaced and
modified slightly.

(Included is also the two small changes in the patch in
 <87oeq8g1up.fsf@virgil.koldfront.dk>).

To use it, I set this in my .gnus:

 spam-use-crm114 t
 spam-crm114-path "/usr/share/crm114/mailfilter.crm"
 spam-crm114-database-directory "--fileprefix=/home/asjo/.crm114/"

As you can see from the comments, I couldn't figure out how to
automatically add '--fileprefix=' to the database directory when set,
so I took the easy, ugly way out.


  Best regards,

    Adam

-- 
 "Mr. Cotton's... parrot. Same question."                     Adam Sjøgren
                                                         asjo@koldfront.dk

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: crm114-mailfilter.patch --]
[-- Type: text/x-patch, Size: 14514 bytes --]

*** spam.el	2 Apr 2004 19:09:16 -0000	7.27
--- spam.el	3 Apr 2004 23:55:02 -0000
***************
*** 263,268 ****
--- 263,273 ----
    :type 'boolean
    :group 'spam)
  
+ (defcustom spam-use-crm114 nil
+   "Whether CRM114 Mailfilter should be used by `spam-split'."
+   :type 'boolean
+   :group 'spam)
+ 
  (defcustom spam-install-hooks (or
  			       spam-use-dig
  			       spam-use-gmane-xref
***************
*** 281,287 ****
  			       spam-use-BBDB-exclusive
  			       spam-use-ifile
  			       spam-use-stat
! 			       spam-use-spamoracle)
    "Whether the spam hooks should be installed.
  Default to t if one of the spam-use-* variables is set."
    :group 'spam
--- 286,293 ----
  			       spam-use-BBDB-exclusive
  			       spam-use-ifile
  			       spam-use-stat
! 			       spam-use-spamoracle
!                                spam-use-crm114)
    "Whether the spam hooks should be installed.
  Default to t if one of the spam-use-* variables is set."
    :group 'spam
***************
*** 515,520 ****
--- 521,573 ----
    :type 'string
    :group 'spam-spamassassin)
  
+ (defgroup spam-crm114 nil
+   "Spam CRM114 Mailfilter configuration."
+   :group 'spam)
+ 
+ (defcustom spam-crm114-path (executable-find "mailfilter.crm")
+   "File path of the CRM114 Mailfilter executable program."
+   :type '(choice (file :tag "Location of CRM114 Mailfilter")
+ 		 (const :tag "CRM114 Mailfilter is not installed"))
+   :group 'spam-crm114)
+ 
+ (defcustom spam-crm114-header "X-CRM114-Status"
+   "The header that CRM114 Mailfilter inserts in messages."
+   :type 'string
+   :group 'spam-crm114)
+ 
+ (defcustom spam-crm114-spam-switch "--learnspam"
+   "The switch that CRM114 Mailfilter uses to register spam messages."
+   :type 'string
+   :group 'spam-crm114)
+ 
+ (defcustom spam-crm114-ham-switch "--learnnonspam"
+   "The switch that CRM114 Mailfilter uses to register ham messages."
+   :type 'string
+   :group 'spam-crm114)
+ 
+ (defcustom spam-crm114-spam-strong-switch "--UNKNOWN"
+   "The switch that CRM114 Mailfilter uses to unregister ham messages."
+   :type 'string
+   :group 'spam-crm114)
+ 
+ (defcustom spam-crm114-ham-strong-switch "--UNKNOWN"
+   "The switch that CRM114 Mailfilter uses to unregister spam messages."
+   :type 'string
+   :group 'spam-crm114)
+ 
+ (defcustom spam-crm114-positive-spam-header "^SPAM"
+   "The regex on `spam-crm114-header' for positive spam identification."
+   :type 'regexp
+   :group 'spam-crm114)
+ 
+ (defcustom spam-crm114-database-directory nil
+   "Directory path of the CRM114 Mailfilter databases."
+   :type '(choice (directory
+ 		  :tag "Location of the CRM114 Mailfilter database directory")
+ 		 (const :tag "Use the default"))
+   :group 'spam-crm114)
+ 
  ;;; Key bindings for spam control.
  
  (gnus-define-keys gnus-summary-mode-map
***************
*** 605,610 ****
--- 658,664 ----
      (gnus-group-spam-exit-processor-stat         spam spam-use-stat)
      (gnus-group-spam-exit-processor-spamoracle   spam spam-use-spamoracle)
      (gnus-group-spam-exit-processor-spamassassin spam spam-use-spamassassin)
+     (gnus-group-spam-exit-processor-crm114       spam spam-use-crm114)
      (gnus-group-ham-exit-processor-ifile         ham spam-use-ifile)
      (gnus-group-ham-exit-processor-bogofilter    ham spam-use-bogofilter)
      (gnus-group-ham-exit-processor-stat          ham spam-use-stat)
***************
*** 612,618 ****
      (gnus-group-ham-exit-processor-BBDB          ham spam-use-BBDB)
      (gnus-group-ham-exit-processor-copy          ham spam-use-ham-copy)
      (gnus-group-ham-exit-processor-spamassassin  ham spam-use-spamassassin)
!     (gnus-group-ham-exit-processor-spamoracle    ham spam-use-spamoracle))
    "The `spam-list-of-processors' list.
  This list contains pairs associating a ham/spam exit processor
  variable with a classification and a spam-use-* variable.")
--- 666,673 ----
      (gnus-group-ham-exit-processor-BBDB          ham spam-use-BBDB)
      (gnus-group-ham-exit-processor-copy          ham spam-use-ham-copy)
      (gnus-group-ham-exit-processor-spamassassin  ham spam-use-spamassassin)
!     (gnus-group-ham-exit-processor-spamoracle    ham spam-use-spamoracle)
!     (gnus-group-ham-exit-processor-crm114        ham spam-use-crm114))
    "The `spam-list-of-processors' list.
  This list contains pairs associating a ham/spam exit processor
  variable with a classification and a spam-use-* variable.")
***************
*** 657,662 ****
--- 712,720 ----
  (defun spam-group-spam-processor-spamoracle-p (group)
    (spam-group-processor-p group 'gnus-group-spam-exit-processor-spamoracle))
  
+ (defun spam-group-spam-processor-crm114-p (group)
+   (spam-group-processor-p group 'gnus-group-spam-exit-processor-crm114))
+ 
  (defun spam-group-ham-processor-bogofilter-p (group)
    (spam-group-processor-p group 'gnus-group-ham-exit-processor-bogofilter))
  
***************
*** 678,683 ****
--- 736,744 ----
  (defun spam-group-ham-processor-spamoracle-p (group)
    (spam-group-processor-p group 'gnus-group-ham-exit-processor-spamoracle))
  
+ (defun spam-group-ham-processor-crm114-p (group)
+   (spam-group-processor-p group 'gnus-group-ham-exit-processor-crm114))
+ 
  (defun spam-report-articles-gmane (n)
    "Report the current message as spam.
  Respects the process/prefix convention."
***************
*** 686,691 ****
--- 747,753 ----
      (gnus-summary-remove-process-mark article)
      (spam-report-gmane article)))
  
+ ; asjo: Perhaps add spam-crm114-score here? How?
  (defun spam-generic-score ()
    (interactive)
    "Invoke whatever scoring method we can."
***************
*** 718,724 ****
  	     (new-articles (spam-list-articles
  			    gnus-newsgroup-articles
  			    classification))
! 	     (changed-articles (spam-set-difference new-articles old-articles)))
  	;; now that we have the changed articles, we go through the processors
  	(dolist (processor-param spam-list-of-processors)
  	  (let ((processor (nth 0 processor-param))
--- 780,786 ----
  	     (new-articles (spam-list-articles
  			    gnus-newsgroup-articles
  			    classification))
! 	     (changed-articles (gnus-set-difference new-articles old-articles)))
  	;; now that we have the changed articles, we go through the processors
  	(dolist (processor-param spam-list-of-processors)
  	  (let ((processor (nth 0 processor-param))
***************
*** 776,783 ****
        (spam-ham-copy-routine
         (gnus-parameter-ham-process-destination gnus-newsgroup-name)))
  
!     ;; now move all ham articles out of spam groups
!     (when (spam-group-spam-contents-p gnus-newsgroup-name)
        (gnus-message 5 "Moving ham messages from spam group")
        (spam-ham-move-routine
         (gnus-parameter-ham-process-destination gnus-newsgroup-name))))
--- 838,845 ----
        (spam-ham-copy-routine
         (gnus-parameter-ham-process-destination gnus-newsgroup-name)))
  
!     ;; now move all ham articles out of non-ham groups
!     (when (not (spam-group-ham-contents-p gnus-newsgroup-name))
        (gnus-message 5 "Moving ham messages from spam group")
        (spam-ham-move-routine
         (gnus-parameter-ham-process-destination gnus-newsgroup-name))))
***************
*** 785,804 ****
    (setq spam-old-ham-articles nil)
    (setq spam-old-spam-articles nil))
  
- (defun spam-set-difference (list1 list2)
-   "Return a set difference of LIST1 and LIST2.  
- When either list is nil, the other is returned."
-   (if (and list1 list2)
-       ;; we have two non-nil lists
-       (progn
- 	(dolist (item (append list1 list2))
- 	  (when (and (memq item list1) (memq item list2))
- 	    (setq list1 (delq item list1))
- 	    (setq list2 (delq item list2))))
- 	(append list1 list2))
-     ;; if either of the lists was nil, return the other one
-     (if list1 list1 list2)))
- 
  (defun spam-mark-junk-as-spam-routine ()
    ;; check the global list of group names spam-junk-mailgroups and the
    ;; group parameters
--- 847,852 ----
***************
*** 859,865 ****
  	 (gnus-check-backend-function
  	  'request-move-article gnus-newsgroup-name))
  	(respool-method (gnus-find-method-for-group gnus-newsgroup-name))
! 	article mark todo deletep respool)
  
      (when (member 'respool groups)
        (setq respool t)			; boolean for later
--- 907,913 ----
  	 (gnus-check-backend-function
  	  'request-move-article gnus-newsgroup-name))
  	(respool-method (gnus-find-method-for-group gnus-newsgroup-name))
! 	article mark deletep respool)
  
      (when (member 'respool groups)
        (setq respool t)			; boolean for later
***************
*** 1006,1012 ****
      (spam-use-spamassassin-headers 	. 	spam-check-spamassassin-headers)
      (spam-use-spamassassin 		. 	spam-check-spamassassin)
      (spam-use-bogofilter-headers 	. 	spam-check-bogofilter-headers)
!     (spam-use-bogofilter 	 	. 	spam-check-bogofilter))
    "The spam-list-of-checks list contains pairs associating a
  parameter variable with a spam checking function.  If the
  parameter variable is true, then the checking function is called,
--- 1054,1061 ----
      (spam-use-spamassassin-headers 	. 	spam-check-spamassassin-headers)
      (spam-use-spamassassin 		. 	spam-check-spamassassin)
      (spam-use-bogofilter-headers 	. 	spam-check-bogofilter-headers)
!     (spam-use-bogofilter 	 	. 	spam-check-bogofilter)
!     (spam-use-crm114 	 	        . 	spam-check-crm114))
    "The spam-list-of-checks list contains pairs associating a
  parameter variable with a spam checking function.  If the
  parameter variable is true, then the checking function is called,
***************
*** 1028,1034 ****
      spam-use-bogofilter
      spam-use-blackholes
      spam-use-spamassassin
!     spam-use-spamoracle)
    "The spam-list-of-statistical-checks list contains all the mail
  splitters that need to have the full message body available.
  Note that you should fetch extra headers if you don't like this,
--- 1077,1084 ----
      spam-use-bogofilter
      spam-use-blackholes
      spam-use-spamassassin
!     spam-use-spamoracle
!     spam-use-crm114)
    "The spam-list-of-statistical-checks list contains all the mail
  splitters that need to have the full message body available.
  Note that you should fetch extra headers if you don't like this,
***************
*** 1209,1215 ****
      (spam-use-bogofilter spam-bogofilter-register-ham-routine
  			 spam-bogofilter-register-spam-routine
  			 spam-bogofilter-unregister-ham-routine
! 			 spam-bogofilter-unregister-spam-routine))
    "The spam-registration-functions list contains pairs
  associating a parameter variable with the ham and spam
  registration functions, and the ham and spam unregistration
--- 1259,1270 ----
      (spam-use-bogofilter spam-bogofilter-register-ham-routine
  			 spam-bogofilter-register-spam-routine
  			 spam-bogofilter-unregister-ham-routine
! 			 spam-bogofilter-unregister-spam-routine)
!     (spam-use-crm114 spam-crm114-register-ham-routine
! 		     spam-crm114-register-spam-routine
!                      ;; does CRM114 Mailfilter support unregistration?
! 		     nil
! 		     nil))
    "The spam-registration-functions list contains pairs
  associating a parameter variable with the ham and spam
  registration functions, and the ham and spam unregistration
***************
*** 2162,2167 ****
--- 2217,2307 ----
  
  (defun spam-spamassassin-unregister-ham-routine (articles)
    (spam-spamassassin-register-with-sa-learn articles nil t))
+ \f
+ ;;;; CRM114 Mailfilter
+ (defun spam-check-crm114-headers (&optional score)
+   (let ((header (message-fetch-field spam-crm114-header))
+ 	(spam-split-group (if spam-split-symbolic-return
+ 			      'spam
+ 			    spam-split-group)))
+     (when header			; return nil when no header
+       (if score				; scoring mode
+ 	  (if (string-match "( pR: \\([0-9.-]+\\)" header)
+ 	      (match-string 1 header)
+ 	    "0")
+ 	;; spam detection mode
+ 	(when (string-match spam-crm114-positive-spam-header
+ 			    header)
+ 	  spam-split-group)))))
+ 
+ ;; return something sensible if the score can't be determined
+ (defun spam-crm114-score ()
+   "Get the CRM114 Mailfilter pR"
+   (interactive)
+   (save-window-excursion
+     (gnus-summary-show-article t)
+     (set-buffer gnus-article-buffer)
+     (let ((score (or (spam-check-crm114-headers t)
+ 		     (spam-check-crm114 t))))
+       (gnus-summary-show-article)
+       (message "pR: %s" score)
+       (or score "0"))))
+ 
+ (defun spam-check-crm114 (&optional score)
+   "Check the CRM114 Mailfilter backend for the classification of this message"
+   (let ((article-buffer-name (buffer-name))
+ 	(db spam-crm114-database-directory)
+ 	return)
+     (with-temp-buffer
+       (let ((temp-buffer-name (buffer-name)))
+ 	(save-excursion
+ 	  (set-buffer article-buffer-name)
+ 	  (apply 'call-process-region
+ 		 (point-min) (point-max)
+ 		 spam-crm114-path
+ 		 nil temp-buffer-name nil
+ 		 (if db `("", db) `("")))) ; asjo: I don't know how to do this right.
+                                            ;       I'd like to add --fileprefix=db if
+                                            ;       db is set.
+ 	(setq return (spam-check-crm114-headers score))))
+     return))
+ 
+ (defun spam-crm114-register-with-crm114 (articles
+ 						 spam
+ 						 &optional unregister)
+   "Register an article, given as a string, as spam or non-spam."
+   (dolist (article articles)
+     (let ((article-string (spam-get-article-as-string article))
+ 	  (db spam-crm114-database-directory)
+ 	  (switch (if unregister
+ 		      (if spam
+ 			  spam-crm114-spam-strong-switch
+ 			spam-crm114-ham-strong-switch)
+ 		    (if spam
+ 			spam-crm114-spam-switch
+ 		      spam-crm114-ham-switch))))
+       (when (stringp article-string)
+ 	(with-temp-buffer
+ 	  (insert article-string)
+ 
+ 	  (apply 'call-process-region
+ 		 (point-min) (point-max)
+ 		 spam-crm114-path
+ 		 nil nil nil switch
+ 		 (if db `("", db) `("")))))))) ; asjo: See above.
+ 
+ (defun spam-crm114-register-spam-routine (articles &optional unregister)
+   (spam-crm114-register-with-crm114 articles t unregister))
+ 
+ (defun spam-crm114-unregister-spam-routine (articles)
+   (spam-crm114-register-spam-routine articles t))
+ 
+ (defun spam-crm114-register-ham-routine (articles &optional unregister)
+   (spam-crm114-register-with-crm114 articles nil unregister))
+ 
+ (defun spam-crm114-unregister-ham-routine (articles)
+   (spam-crm114-register-ham-routine articles t))
+ 
  \f
  ;;;; Hooks