[PATCH 1/2] nnir.el (nnir-read-server-parm): Use default value.

[PATCH 1/2] nnir.el (nnir-read-server-parm): Use default value.

[Kan-Ru Chen]

Signed-off-by: Kan-Ru Chen <kanru@kanru.info>
---
 lisp/ChangeLog |    4 ++++
 lisp/nnir.el   |    1 +
 2 files changed, 5 insertions(+), 0 deletions(-)

diff --git a/lisp/ChangeLog b/lisp/ChangeLog
index 7fd853e..4a802c5 100644
--- a/lisp/ChangeLog
+++ b/lisp/ChangeLog
@@ -1,3 +1,7 @@
+2011-04-03  Kan-Ru Chen  <kanru@kanru.info>
+
+	* nnir.el (nnir-read-server-parm): Use default value.
+
 2011-04-02  Chong Yidong  <cyd@stupidchicken.com>
 
 	* nnimap.el (nnimap-open-connection-1): Pass explicit :end-of-command
diff --git a/lisp/nnir.el b/lisp/nnir.el
index eaaac3f..6f3cd9e 100644
--- a/lisp/nnir.el
+++ b/lisp/nnir.el
@@ -1518,6 +1518,7 @@ server is of form 'backend:name'."
   (let ((method (gnus-server-to-method server)))
     (cond ((and method (assq key (cddr method)))
     	   (nth 1 (assq key (cddr method))))
+          ((boundp key) (symbol-value key))
     	  (t nil))))
 
 (defun nnir-possibly-change-server (server)
-- 
1.7.4.1

[PATCH 2/2] nnir.el: New `notmuch' backend.

[Kan-Ru Chen]

Signed-off-by: Kan-Ru Chen <kanru@kanru.info>
---
 lisp/ChangeLog |    6 +++
 lisp/nnir.el   |  101 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 107 insertions(+), 0 deletions(-)

diff --git a/lisp/ChangeLog b/lisp/ChangeLog
index 4a802c5..30fc228 100644
--- a/lisp/ChangeLog
+++ b/lisp/ChangeLog
@@ -1,5 +1,11 @@
 2011-04-03  Kan-Ru Chen  <kanru@kanru.info>
 
+	* nnir.el (nnir-notmuch-program, nnir-notmuch-additional-switches)
+	(nnir-notmuch-remove-prefix, nnir-engines, nnir-run-notmuch): New nnir
+	`notmuch' backend.
+
+2011-04-03  Kan-Ru Chen  <kanru@kanru.info>
+
 	* nnir.el (nnir-read-server-parm): Use default value.
 
 2011-04-02  Chong Yidong  <cyd@stupidchicken.com>
diff --git a/lisp/nnir.el b/lisp/nnir.el
index 6f3cd9e..a2a56a8 100644
--- a/lisp/nnir.el
+++ b/lisp/nnir.el
@@ -499,6 +499,31 @@ arrive at the correct group name, \"mail.misc\"."
   :type '(directory)
   :group 'nnir)
 
+(defcustom nnir-notmuch-program "notmuch"
+  "*Name of notmuch search executable."
+  :type '(string)
+  :group 'nnir)
+
+(defcustom nnir-notmuch-additional-switches '()
+  "*A list of strings, to be given as additional arguments to notmuch.
+
+Note that this should be a list.  Ie, do NOT use the following:
+    (setq nnir-notmuch-additional-switches \"-i -w\") ; wrong
+Instead, use this:
+    (setq nnir-notmuch-additional-switches '(\"-i\" \"-w\"))"
+  :type '(repeat (string))
+  :group 'nnir)
+
+(defcustom nnir-notmuch-remove-prefix (concat (getenv "HOME") "/Mail/")
+  "*The prefix to remove from each file name returned by notmuch
+in order to get a group name (albeit with / instead of .).  This is a
+regular expression.
+
+This variable is very similar to `nnir-namazu-remove-prefix', except
+that it is for notmuch, not Namazu."
+  :type '(regexp)
+  :group 'nnir)
+
 ;;; Developer Extension Variable:
 
 (defvar nnir-engines
@@ -519,6 +544,8 @@ arrive at the correct group name, \"mail.misc\"."
              ((group . "Swish-e Group spec: ")))
     (namazu  nnir-run-namazu
              ())
+    (notmuch nnir-run-notmuch
+             ())
     (hyrex   nnir-run-hyrex
 	     ((group . "Hyrex Group spec: ")))
     (find-grep nnir-run-find-grep
@@ -1317,6 +1344,80 @@ Tested with Namazu 2.0.6 on a GNU/Linux system."
                                (> (nnir-artitem-rsv x)
                                   (nnir-artitem-rsv y)))))))))
 
+(defun nnir-run-notmuch (query server &optional group)
+  "Run QUERY against notmuch.
+Returns a vector of (group name, file name) pairs (also vectors,
+actually)."
+
+  ;; (when group
+  ;;   (error "The notmuch backend cannot search specific groups"))
+
+  (save-excursion
+    (let ( (qstring (cdr (assq 'query query)))
+	   (groupspec (cdr (assq 'group query)))
+	   (prefix (nnir-read-server-parm 'nnir-notmuch-remove-prefix server))
+           artlist
+           (article-pattern (if (string= (gnus-group-server server) "nnmaildir")
+			       ":[0-9]+"
+			     "^[0-9]+$"))
+           artno dirnam filenam)
+
+      (when (equal "" qstring)
+        (error "notmuch: You didn't enter anything"))
+
+      (set-buffer (get-buffer-create nnir-tmp-buffer))
+      (erase-buffer)
+
+      (if groupspec
+          (message "Doing notmuch query %s on %s..." qstring groupspec)
+        (message "Doing notmuch query %s..." qstring))
+
+      (let* ((cp-list `( ,nnir-notmuch-program
+                         nil            ; input from /dev/null
+                         t              ; output
+                         nil            ; don't redisplay
+                         "search"
+                         "--format=text"
+                         "--output=files"
+                         ,@(nnir-read-server-parm 'nnir-notmuch-additional-switches server)
+                         ,qstring       ; the query, in notmuch format
+                         ))
+             (exitstatus
+              (progn
+                (message "%s args: %s" nnir-notmuch-program
+                         (mapconcat 'identity (cddddr cp-list) " ")) ;; ???
+                (apply 'call-process cp-list))))
+        (unless (or (null exitstatus)
+                    (zerop exitstatus))
+          (nnheader-report 'nnir "Couldn't run notmuch: %s" exitstatus)
+          ;; notmuch failure reason is in this buffer, show it if
+          ;; the user wants it.
+          (when (> gnus-verbose 6)
+            (display-buffer nnir-tmp-buffer))))
+
+      ;; The results are output in the format of:
+      ;; absolute-path-name
+      (goto-char (point-min))
+      (while (not (eobp))
+        (setq filenam (buffer-substring-no-properties (line-beginning-position)
+                                                      (line-end-position))
+              artno (file-name-nondirectory filenam)
+              dirnam (file-name-directory filenam))
+        (forward-line 1)
+
+        ;; don't match directories
+        (when (string-match article-pattern artno)
+          (when (not (null dirnam))
+
+	    ;; maybe limit results to matching groups.
+	    (when (or (not groupspec)
+		      (string-match groupspec dirnam))
+	      (nnir-add-result dirnam artno "" prefix server artlist)))))
+
+      (message "Massaging notmuch output...done")
+
+      artlist)))
+
 (defun nnir-run-find-grep (query server &optional grouplist)
   "Run find and grep to obtain matching articles."
   (let* ((method (gnus-server-to-method server))
-- 
1.7.4.1

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Lars Magne Ingebrigtsen]

Kan-Ru Chen <kanru@kanru.info> writes:

> +	* nnir.el (nnir-notmuch-program, nnir-notmuch-additional-switches)
> +	(nnir-notmuch-remove-prefix, nnir-engines, nnir-run-notmuch): New nnir
> +	`notmuch' backend.

Looks good to me.  Do you have FSF copyright assignment papers on file?

-- 
(domestic pets only, the antidote for overdose, milk.)
  bloggy blog http://lars.ingebrigtsen.no/

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Kan-Ru Chen]

Lars Magne Ingebrigtsen <larsi@gnus.org> writes:

> Kan-Ru Chen <kanru@kanru.info> writes:
>
>> +	* nnir.el (nnir-notmuch-program, nnir-notmuch-additional-switches)
>> +	(nnir-notmuch-remove-prefix, nnir-engines, nnir-run-notmuch): New nnir
>> +	`notmuch' backend.
>
> Looks good to me.  Do you have FSF copyright assignment papers on file?

Not yet. I'm willing to sign one.

-- 
A badly written book is only a blunder. A bad translation of a good
book is a crime.
                -- Gilbert Highet

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Ted Zlatanov]

On Sun, 03 Apr 2011 19:43:28 +0200 Lars Magne Ingebrigtsen <larsi@gnus.org> wrote: 

LMI> Kan-Ru Chen <kanru@kanru.info> writes:
>> +	* nnir.el (nnir-notmuch-program, nnir-notmuch-additional-switches)
>> +	(nnir-notmuch-remove-prefix, nnir-engines, nnir-run-notmuch): New nnir
>> +	`notmuch' backend.

LMI> Looks good to me.  Do you have FSF copyright assignment papers on file?

Julien, I remember you didn't like notmuch at one point.  Are there any
issues with this nnir support for it?

Ted

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Kan-Ru Chen]

Ted Zlatanov <tzz@lifelogs.com> writes:

> On Sun, 03 Apr 2011 19:43:28 +0200 Lars Magne Ingebrigtsen <larsi@gnus.org> wrote: 
>
> LMI> Kan-Ru Chen <kanru@kanru.info> writes:
>>> +	* nnir.el (nnir-notmuch-program, nnir-notmuch-additional-switches)
>>> +	(nnir-notmuch-remove-prefix, nnir-engines, nnir-run-notmuch): New nnir
>>> +	`notmuch' backend.
>
> LMI> Looks good to me.  Do you have FSF copyright assignment papers on file?
>
> Julien, I remember you didn't like notmuch at one point.  Are there any
> issues with this nnir support for it?

Hey! I understand Julien's concern of notmuch but there is one thing
notmuch does besides tagging mail, that is index the mail using xapian
engine. Actually this is all what notmuch can do, plus some MUA friendly
interface so MUAs like mutt or Gnus can use it as a index engine.

I'm still using nnml backend, so nnir is really import to me. I have
experience with notmuch, it is easier to configure than namazu or
swish++ so I think it is a good idea to put notmuch support into Gnus.

Next step is to refactor the nnir interface (and write document!) so adding
a new backend can be very easy.

Kanru

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Ted Zlatanov]

On Tue, 05 Apr 2011 09:02:19 +0800 Kan-Ru Chen <kanru@kanru.info> wrote: 

KC> Hey! I understand Julien's concern of notmuch but there is one thing
KC> notmuch does besides tagging mail, that is index the mail using xapian
KC> engine. Actually this is all what notmuch can do, plus some MUA friendly
KC> interface so MUAs like mutt or Gnus can use it as a index engine.

KC> I'm still using nnml backend, so nnir is really import to me. I have
KC> experience with notmuch, it is easier to configure than namazu or
KC> swish++ so I think it is a good idea to put notmuch support into Gnus.

I have no problem with notmuch support in Gnus, just wondered if Julien
had comments since he has dealt with the software.

To me it seems useful and I can see how the Gnus registry marks could
synchronize with the notmuch tags.  spam.el could use notmuch tags as
well.

Ted

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Erik Colson]

Hi all,

After using gnus with offlineimap to have all mails also local and then
running notmuch with the data, I figured some oddities in
offlineimap. So I moved to nnimap from gnus and have no local mail
storage anymore. This works pretty nice. The only drawback is that some
mailboxes are stored with _huge_ mails and it is a pain in the a** to
wait while retrieving the mails from the IMAP server. I solved this by
putting those mail in the agent with '*'. Care has to be taken while
moving such mails to another mailbox as you need to remove such mails
from the agent before moving. But that's ok to me (although a var
setting to automate this would be welcome and probably not to hard to
hack).

One remaining issue is searching the mailboxes through the IMAP
server. It works, but it is slow due to the slowness of the IMAP
server. So my wish would be to have a _local_ index of the distant mail
on the IMAP server, without having a local cache of all the mails.

Probably it would be preferable to have a separate process running a
IMAP client which would read all new mails and index them locally. If
anyone knows of such a software, please tell me!

Best
-- 
erik

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Kan-Ru Chen]

Ted Zlatanov <tzz@lifelogs.com> writes:

> On Tue, 05 Apr 2011 09:02:19 +0800 Kan-Ru Chen <kanru@kanru.info> wrote: 
>
> KC> Hey! I understand Julien's concern of notmuch but there is one thing
> KC> notmuch does besides tagging mail, that is index the mail using xapian
> KC> engine. Actually this is all what notmuch can do, plus some MUA friendly
> KC> interface so MUAs like mutt or Gnus can use it as a index engine.
>
> KC> I'm still using nnml backend, so nnir is really import to me. I have
> KC> experience with notmuch, it is easier to configure than namazu or
> KC> swish++ so I think it is a good idea to put notmuch support into Gnus.
>
> I have no problem with notmuch support in Gnus, just wondered if Julien
> had comments since he has dealt with the software.
>
> To me it seems useful and I can see how the Gnus registry marks could
> synchronize with the notmuch tags.  spam.el could use notmuch tags as
> well.

Alright, I'm too excited and cannot wait to sign the paper and
contribute more code to Gnus ;-)

Kanru

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Julien Danjou]

[-- Attachment #1: Type: text/plain, Size: 322 bytes --]

On Mon, Apr 04 2011, Ted Zlatanov wrote:

> Julien, I remember you didn't like notmuch at one point.  Are there any
> issues with this nnir support for it?

Not at all. My point of view about notmuch is irrelevant in the case of
adding such a code into Gnus. :)

-- 
Julien Danjou
❱ http://julien.danjou.info

[-- Attachment #2: Type: application/pgp-signature, Size: 835 bytes --]

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Ted Zlatanov]

On Wed, 06 Apr 2011 08:08:24 +0200 Erik Colson <eco@ecocode.net> wrote: 

EC> One remaining issue is searching the mailboxes through the IMAP
EC> server. It works, but it is slow due to the slowness of the IMAP
EC> server. So my wish would be to have a _local_ index of the distant mail
EC> on the IMAP server, without having a local cache of all the mails.

EC> Probably it would be preferable to have a separate process running a
EC> IMAP client which would read all new mails and index them locally. If
EC> anyone knows of such a software, please tell me!

It's probably easier to use the Gnus registry for local searching after
the recent rewrite.  For instance there's a
`gnus-registry-wash-for-keywords' function which will store all the
keywords for an article in the registry, and IIRC nnir has some glue for
it as well.

Once you are tracking keywords (set up with
`gnus-registry-track-extra'), any keywords noticed in an article will
end up indexed.  So then you just say

(registry-lookup-secondary-value gnus-registry-db 'keyword "thekeyword")

and you'll get a list of message IDs (which you can then look up to get
the group).  The nice thing is that when you set this up, it works
anytime you visit a group so it doesn't require external processes or
indices.  And it's using hashtables so it's a very fast lookup.

It would not be hard to use the notmuch keywords instead of the keywords
from the article body, if they are in the headers or connected to the
message ID somehow.  But I don't know notmuch well enough to do it.

Ted

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Kan-Ru Chen]

Lars Magne Ingebrigtsen <larsi@gnus.org> writes:

> Kan-Ru Chen <kanru@kanru.info> writes:
>
>> +	* nnir.el (nnir-notmuch-program, nnir-notmuch-additional-switches)
>> +	(nnir-notmuch-remove-prefix, nnir-engines, nnir-run-notmuch): New nnir
>> +	`notmuch' backend.
>
> Looks good to me.  Do you have FSF copyright assignment papers on file?

Ping!

I have signed the paper :)

-- 
Kanru

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Lars Magne Ingebrigtsen]

Kan-Ru Chen <kanru@kanru.info> writes:

> I have signed the paper :)

Great!  I've now applied your patch.

-- 
(domestic pets only, the antidote for overdose, milk.)
  bloggy blog http://lars.ingebrigtsen.no/

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Gour]

On Tue, 05 Jul 2011 22:22:55 +0200
Lars Magne Ingebrigtsen <larsi@gnus.org> wrote:

> Great!  I've now applied your patch.

I'd like to use Gnus with orgmode and notmuch, but wonder where are
some up-to-date docs to be found considering the above patch since
http://www.emacswiki.org/emacs/NotMuch#toc1 page references 

http://roland.entierement.nu/blog/2010/09/08/gnus-dovecot-offlineimap-search-a-howto.html

& http://article.gmane.org/gmane.emacs.gnus.user/13308

which are older?


Sincerely,
Gour

-- 
As a strong wind sweeps away a boat on the water, 
even one of the roaming senses on which the mind 
focuses can carry away a man's intelligence.

http://www.atmarama.net | Hlapicina (Croatia) | GPG: 52B5C810

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Lars Magne Ingebrigtsen]

Gour <gour@atmarama.net> writes:

> I'd like to use Gnus with orgmode and notmuch, but wonder where are
> some up-to-date docs to be found considering the above patch since
> http://www.emacswiki.org/emacs/NotMuch#toc1 page references 
>
> http://roland.entierement.nu/blog/2010/09/08/gnus-dovecot-offlineimap-search-a-howto.html
>
> & http://article.gmane.org/gmane.emacs.gnus.user/13308
>
> which are older?

I've now made the Gnus manual mention notmuch, but I'm not quite sure
how to use it, either.

Perhaps Kan-Ru Chen can add a short usage description to the Gnus
manual?  >"?

-- 
(domestic pets only, the antidote for overdose, milk.)
  No Gnus T-Shirt for sale: http://ingebrigtsen.no/no.php

Re: [PATCH 2/2] nnir.el: New `notmuch' backend.

[Kan-Ru Chen (陳侃如)]

Lars Magne Ingebrigtsen <larsi@gnus.org> writes:

> Gour <gour@atmarama.net> writes:
>
>> I'd like to use Gnus with orgmode and notmuch, but wonder where are
>> some up-to-date docs to be found considering the above patch since
>> http://www.emacswiki.org/emacs/NotMuch#toc1 page references 
>>
>> http://roland.entierement.nu/blog/2010/09/08/gnus-dovecot-offlineimap-search-a-howto.html
>>
>> & http://article.gmane.org/gmane.emacs.gnus.user/13308
>>
>> which are older?
>
> I've now made the Gnus manual mention notmuch, but I'm not quite sure
> how to use it, either.
>
> Perhaps Kan-Ru Chen can add a short usage description to the Gnus
> manual?  >"?

Yeah, will do.

Kanru