Gnus development mailing list
 help / color / mirror / Atom feed
* Help testing gnus-search fixups
@ 2021-06-28 16:42 Eric Abrahamsen
  2021-06-28 18:00 ` Adam Sjøgren
  2021-06-29 11:27 ` Eric S Fraga
  0 siblings, 2 replies; 34+ messages in thread
From: Eric Abrahamsen @ 2021-06-28 16:42 UTC (permalink / raw)
  To: ding

[-- Attachment #1: Type: text/plain, Size: 320 bytes --]

Hi all,

I've reworked the way gnus-search parses the output from search
utilities like notmuch, which should eliminate some of the bugs and
funny behavior that's been reported here. I'd be much happier if I had
some more eyes on it -- if you're using notmuch or namazu, please help
me give this a whirl!

Thanks,
Eric


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Rework-gnus-search-indexed-parse-output.patch --]
[-- Type: text/x-patch, Size: 5012 bytes --]

From 7f9e962b890bfa0924ca17a7abd8cc83e1b8d2bf Mon Sep 17 00:00:00 2001
From: Eric Abrahamsen <eric@ericabrahamsen.net>
Date: Sat, 26 Jun 2021 10:16:19 -0700
Subject: [PATCH 1/1] Rework gnus-search-indexed-parse-output

* lisp/gnus/gnus-search.el (gnus-search-indexed-parse-output): Be more
careful about matching filesystem paths to Gnus group names; make
absolutely sure that we only return valid article numbers.
---
 lisp/gnus/gnus-search.el | 94 ++++++++++++++++++----------------------
 1 file changed, 42 insertions(+), 52 deletions(-)

diff --git a/lisp/gnus/gnus-search.el b/lisp/gnus/gnus-search.el
index 70bde264c1..add34e7e02 100644
--- a/lisp/gnus/gnus-search.el
+++ b/lisp/gnus/gnus-search.el
@@ -1351,68 +1351,58 @@ gnus-search-run-search
 
 (cl-defmethod gnus-search-indexed-parse-output ((engine gnus-search-indexed)
 						server query &optional groups)
-  (let ((prefix (slot-value engine 'remove-prefix))
-	(group-regexp (when groups
-			(mapconcat
-			 (lambda (group-name)
-			   (mapconcat #'regexp-quote
-				      (split-string
-				       (gnus-group-real-name group-name)
-				       "[.\\/]")
-				      "[.\\\\/]"))
-			 groups
-			 "\\|")))
-	artlist vectors article group)
+  (let ((prefix (or (slot-value engine 'remove-prefix)
+                    ""))
+	artlist article group)
     (goto-char (point-min))
+    ;; Prep prefix, we want to at least be removing the root
+    ;; filesystem separator.
+    (when (stringp prefix)
+      (setq prefix (file-name-as-directory
+                    (expand-file-name prefix "/"))))
     (while (not (or (eobp)
                     (looking-at-p
                      "\\(?:[[:space:]\n]+\\)?Process .+ finished")))
       (pcase-let ((`(,f-name ,score) (gnus-search-indexed-extract engine)))
 	(when (and f-name
                    (file-readable-p f-name)
-		   (null (file-directory-p f-name))
-		   (or (null groups)
-		       (and (gnus-search-single-p query)
-			    (alist-get 'thread query))
-		       (string-match-p group-regexp f-name)))
-	  (push (list f-name score) artlist))))
+		   (null (file-directory-p f-name)))
+          (setq group
+                (replace-regexp-in-string
+	         "[/\\]" "."
+	         (replace-regexp-in-string
+	          "/?\\(cur\\|new\\|tmp\\)?/\\'" ""
+	          (replace-regexp-in-string
+	           "\\`\\." ""
+	           (string-remove-prefix
+                    prefix (file-name-directory f-name))
+                   nil t)
+	          nil t)
+	         nil t))
+          (setq article (file-name-nondirectory f-name)
+                article
+                ;; TODO: Provide a cleaner way of producing final
+                ;; article numbers for the various backends.
+                (if (string-match-p "\\`[[:digit:]]+\\'" article)
+		    (string-to-number article)
+		  (nnmaildir-base-name-to-article-number
+		   (substring article 0 (string-match ":" article))
+		   group (string-remove-prefix "nnmaildir:" server))))
+          (when (and (numberp article)
+                     (or (null groups)
+                         (member group groups)))
+	    (push (list f-name article group score)
+                  artlist)))))
     ;; Are we running an additional grep query?
     (when-let ((grep-reg (alist-get 'grep query)))
       (setq artlist (gnus-search-grep-search engine artlist grep-reg)))
-    ;; Prep prefix.
-    (when (and prefix (null (string-empty-p prefix)))
-      (setq prefix (file-name-as-directory (expand-file-name prefix))))
-    ;; Turn (file-name score) into [group article score].
-    (pcase-dolist (`(,f-name ,score) artlist)
-      (setq article (file-name-nondirectory f-name)
-	    group (file-name-directory f-name))
-      ;; Remove prefix.
-      (when prefix
-	(setq group (string-remove-prefix prefix group)))
-      ;; Break the directory name down until it's something that
-      ;; (probably) can be used as a group name.
-      (setq group
-	    (replace-regexp-in-string
-	     "[/\\]" "."
-	     (replace-regexp-in-string
-	      "/?\\(cur\\|new\\|tmp\\)?/\\'" ""
-	      (replace-regexp-in-string
-	       "^[./\\]" ""
-	       group nil t)
-	      nil t)
-	     nil t))
-
-      (push (vector (gnus-group-full-name group server)
-		    (if (string-match-p "\\`[[:digit:]]+\\'" article)
-			(string-to-number article)
-		      (nnmaildir-base-name-to-article-number
-		       (substring article 0 (string-match ":" article))
-		       group (string-remove-prefix "nnmaildir:" server)))
-		    (if (numberp score)
-			score
-		      (string-to-number score)))
-	    vectors))
-    vectors))
+    ;; Munge into the list of vectors expected by nnselect.
+    (mapcar (pcase-lambda (`(,_ ,article ,group ,score))
+              (vector group article
+                      (if (numberp score)
+			  score
+		        (string-to-number score))))
+            artlist)))
 
 (cl-defmethod gnus-search-indexed-extract ((_engine gnus-search-indexed))
   "Base implementation treats the whole line as a filename, and
-- 
2.32.0


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-06-28 16:42 Help testing gnus-search fixups Eric Abrahamsen
@ 2021-06-28 18:00 ` Adam Sjøgren
  2021-06-28 23:15   ` Eric Abrahamsen
  2021-06-29 11:27 ` Eric S Fraga
  1 sibling, 1 reply; 34+ messages in thread
From: Adam Sjøgren @ 2021-06-28 18:00 UTC (permalink / raw)
  To: ding

Eric writes:

> if you're using notmuch or namazu, please help me give this a whirl!

Which version of Emacs is your patch against? I just tried applying it
to master@2a33fc8d19 and got:

  checking file lisp/gnus/gnus-search.el
  Hunk #1 FAILED at 1351.
  1 out of 1 hunk FAILED


  Best regards,

    Adam

-- 
 "Synge? Hvorfor skal vi altid synge,                       Adam Sjøgren
  bare fordi det er en tegnefilm?"                     asjo@koldfront.dk



^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-06-28 18:00 ` Adam Sjøgren
@ 2021-06-28 23:15   ` Eric Abrahamsen
  2021-06-29  6:04     ` Thomas Alexander Gerds
  2021-06-29  9:30     ` Adam Sjøgren
  0 siblings, 2 replies; 34+ messages in thread
From: Eric Abrahamsen @ 2021-06-28 23:15 UTC (permalink / raw)
  To: Adam Sjøgren; +Cc: ding

Adam Sjøgren <asjo@koldfront.dk> writes:

> Eric writes:
>
>> if you're using notmuch or namazu, please help me give this a whirl!
>
> Which version of Emacs is your patch against? I just tried applying it
> to master@2a33fc8d19 and got:

It applies to up-to-date master -- there's one other change that went
into this function since your current revision. The changes are confined
to the one function, so you could also just eval this definition:

(cl-defmethod gnus-search-indexed-parse-output ((engine gnus-search-indexed)
						server query &optional groups)
  (let ((prefix (or (slot-value engine 'remove-prefix)
                    ""))
	artlist article group)
    (goto-char (point-min))
    ;; Prep prefix, we want to at least be removing the root
    ;; filesystem separator.
    (when (stringp prefix)
      (setq prefix (file-name-as-directory
                    (expand-file-name prefix "/"))))
    (while (not (or (eobp)
                    (looking-at-p
                     "\\(?:[[:space:]\n]+\\)?Process .+ finished")))
      (pcase-let ((`(,f-name ,score) (gnus-search-indexed-extract engine)))
	(when (and f-name
                   (file-readable-p f-name)
		   (null (file-directory-p f-name)))
          (setq group
                (replace-regexp-in-string
	         "[/\\]" "."
	         (replace-regexp-in-string
	          "/?\\(cur\\|new\\|tmp\\)?/\\'" ""
	          (replace-regexp-in-string
	           "\\`\\." ""
	           (string-remove-prefix
                    prefix (file-name-directory f-name))
                   nil t)
	          nil t)
	         nil t))
          (setq article (file-name-nondirectory f-name)
                article
                ;; TODO: Provide a cleaner way of producing final
                ;; article numbers for the various backends.
                (if (string-match-p "\\`[[:digit:]]+\\'" article)
		    (string-to-number article)
		  (nnmaildir-base-name-to-article-number
		   (substring article 0 (string-match ":" article))
		   group (string-remove-prefix "nnmaildir:" server))))
          (when (and (numberp article)
                     (or (null groups)
                         (member group groups)))
	    (push (list f-name article group score)
                  artlist)))))
    ;; Are we running an additional grep query?
    (when-let ((grep-reg (alist-get 'grep query)))
      (setq artlist (gnus-search-grep-search engine artlist grep-reg)))
    ;; Munge into the list of vectors expected by nnselect.
    (mapcar (pcase-lambda (`(,_ ,article ,group ,score))
              (vector group article
                      (if (numberp score)
			  score
		        (string-to-number score))))
            artlist)))


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-06-28 23:15   ` Eric Abrahamsen
@ 2021-06-29  6:04     ` Thomas Alexander Gerds
  2021-06-29  9:30     ` Adam Sjøgren
  1 sibling, 0 replies; 34+ messages in thread
From: Thomas Alexander Gerds @ 2021-06-29  6:04 UTC (permalink / raw)
  To: Eric Abrahamsen; +Cc: Adam Sjøgren, ding


I see 2 issues:

1. you may need 

(setq group (gnus-group-full-name group server))

before

(setq article (file-name-nondirectory f-name) ...

2. with the patch 

 (member group groups)

is nil when I use search-group-spec to '("nnml:" . ("nnml:")) to match
all nnml groups.

cheers

Eric Abrahamsen <eric@ericabrahamsen.net> writes:

> Adam Sjøgren <asjo@koldfront.dk> writes:
>
>> Eric writes:
>>
>>> if you're using notmuch or namazu, please help me give this a whirl!
>>
>> Which version of Emacs is your patch against? I just tried applying it
>> to master@2a33fc8d19 and got:
>
> It applies to up-to-date master -- there's one other change that went
> into this function since your current revision. The changes are confined
> to the one function, so you could also just eval this definition:
>
> (cl-defmethod gnus-search-indexed-parse-output ((engine gnus-search-indexed)
> 						server query &optional groups)
>   (let ((prefix (or (slot-value engine 'remove-prefix)
>                     ""))
> 	artlist article group)
>     (goto-char (point-min))
>     ;; Prep prefix, we want to at least be removing the root
>     ;; filesystem separator.
>     (when (stringp prefix)
>       (setq prefix (file-name-as-directory
>                     (expand-file-name prefix "/"))))
>     (while (not (or (eobp)
>                     (looking-at-p
>                      "\\(?:[[:space:]\n]+\\)?Process .+ finished")))
>       (pcase-let ((`(,f-name ,score) (gnus-search-indexed-extract engine)))
> 	(when (and f-name
>                    (file-readable-p f-name)
> 		   (null (file-directory-p f-name)))
>           (setq group
>                 (replace-regexp-in-string
> 	         "[/\\]" "."
> 	         (replace-regexp-in-string
> 	          "/?\\(cur\\|new\\|tmp\\)?/\\'" ""
> 	          (replace-regexp-in-string
> 	           "\\`\\." ""
> 	           (string-remove-prefix
>                     prefix (file-name-directory f-name))
>                    nil t)
> 	          nil t)
> 	         nil t))
>           (setq article (file-name-nondirectory f-name)
>                 article
>                 ;; TODO: Provide a cleaner way of producing final
>                 ;; article numbers for the various backends.
>                 (if (string-match-p "\\`[[:digit:]]+\\'" article)
> 		    (string-to-number article)
> 		  (nnmaildir-base-name-to-article-number
> 		   (substring article 0 (string-match ":" article))
> 		   group (string-remove-prefix "nnmaildir:" server))))
>           (when (and (numberp article)
>                      (or (null groups)
>                          (member group groups)))
> 	    (push (list f-name article group score)
>                   artlist)))))
>     ;; Are we running an additional grep query?
>     (when-let ((grep-reg (alist-get 'grep query)))
>       (setq artlist (gnus-search-grep-search engine artlist grep-reg)))
>     ;; Munge into the list of vectors expected by nnselect.
>     (mapcar (pcase-lambda (`(,_ ,article ,group ,score))
>               (vector group article
>                       (if (numberp score)
> 			  score
> 		        (string-to-number score))))
>             artlist)))
>
>
>

-- 
7LL-1 Make peace with your past so it doesn’t spoil your present.


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-06-28 23:15   ` Eric Abrahamsen
  2021-06-29  6:04     ` Thomas Alexander Gerds
@ 2021-06-29  9:30     ` Adam Sjøgren
  2021-06-29  9:55       ` Adam Sjøgren
  1 sibling, 1 reply; 34+ messages in thread
From: Adam Sjøgren @ 2021-06-29  9:30 UTC (permalink / raw)
  To: ding

Eric writes:

> Adam Sjøgren <asjo@koldfront.dk> writes:
>
>> Eric writes:
>>
>>> if you're using notmuch or namazu, please help me give this a whirl!
>>
>> Which version of Emacs is your patch against? I just tried applying it
>> to master@2a33fc8d19 and got:
>
> It applies to up-to-date master

Ok, I did a git pull just before trying - well, no ~25 minutes before
trying the patch, as I built in between. D'oh!

I will give it another go, thanks!


  Best regards,

    Adam

-- 
 "Here comes my darling, saying hello you                   Adam Sjøgren
  Why you look so worried, whats a-wrong with you?"    asjo@koldfront.dk



^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-06-29  9:30     ` Adam Sjøgren
@ 2021-06-29  9:55       ` Adam Sjøgren
  2021-06-29 11:03         ` Thomas Alexander Gerds
  0 siblings, 1 reply; 34+ messages in thread
From: Adam Sjøgren @ 2021-06-29  9:55 UTC (permalink / raw)
  To: ding

Adam writes:

> I will give it another go, thanks!

It applied cleanly to master@6b4043833c. I am using notmuch with nnml,
by having (gnus-search-engine gnus-search-notmuch) on my nnml-method.

If I put point at nnml:normal in the *Group* buffer and press GG and
type "Honda" as the search term, with the patch, I get:

  Doing notmuch query on (nnml:normal)...
  Group nnselect:nnselect-87lf6tq9s1.fsf contains no messages

Without the patch, I get a group with search results:

   O. [ 859: Honda Danmark          ] [...]
   O. [1028: Honda Danmark          ]
   O  [ 819: Honda Danmark          ]
   O  [1048: Honda Danmark          ]
   O  [1002: Honda Danmark          ]
   O  [1001: Honda Danmark          ]
   O  [1006: Honda Danmark          ]
   O  [ 855: Honda Danmark          ]
   O  [1113: Honda Danmark          ]

Doing the search with notmuch on the commandline:

  $ notmuch search Honda | cut -d' ' -f1-5
  thread:000000000000f081   2020-06-09 [1/1]
  thread:000000000000ed7b   2020-06-03 [1/1]
  thread:000000000000ee56   2020-05-26 [1/1]
  thread:000000000000efe8   2020-05-18 [1/1]
  thread:000000000000ed4e   2020-03-12 [1/1]
  thread:000000000000ed4d   2020-03-12 [1/1]
  thread:000000000000ef24   2020-01-14 [1/1]
  thread:00000000000123a0   2019-09-05 [1/1]
  thread:000000000001239f   2019-09-04 [1/1]
  thread:000000000000c7cf   2018-10-22 [1/1]
  thread:0000000000014de8   2018-03-04 [1/1]
  thread:0000000000004e0d   2014-05-16 [1/1]
  thread:0000000000009b6a   2000-04-12 [1/1(2)]

(I cut the output to remove information I don't want to share).


  Best regards,

    Adam

-- 
 "the office is quiet now, i am at the desk,                Adam Sjøgren
  preparing to landscape for another day."             asjo@koldfront.dk



^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-06-29  9:55       ` Adam Sjøgren
@ 2021-06-29 11:03         ` Thomas Alexander Gerds
  2021-06-29 11:13           ` Adam Sjøgren
  0 siblings, 1 reply; 34+ messages in thread
From: Thomas Alexander Gerds @ 2021-06-29 11:03 UTC (permalink / raw)
  To: Adam Sjøgren; +Cc: ding

[-- Attachment #1: Type: text/plain, Size: 1924 bytes --]


the attached patch applied to the patched file fixes this for me. 


Adam Sjøgren <asjo@koldfront.dk> writes:

> Adam writes:
>
>> I will give it another go, thanks!
>
> It applied cleanly to master@6b4043833c. I am using notmuch with nnml,
> by having (gnus-search-engine gnus-search-notmuch) on my nnml-method.
>
> If I put point at nnml:normal in the *Group* buffer and press GG and
> type "Honda" as the search term, with the patch, I get:
>
>   Doing notmuch query on (nnml:normal)...
>   Group nnselect:nnselect-87lf6tq9s1.fsf contains no messages
>
> Without the patch, I get a group with search results:
>
>    O. [ 859: Honda Danmark          ] [...]
>    O. [1028: Honda Danmark          ]
>    O  [ 819: Honda Danmark          ]
>    O  [1048: Honda Danmark          ]
>    O  [1002: Honda Danmark          ]
>    O  [1001: Honda Danmark          ]
>    O  [1006: Honda Danmark          ]
>    O  [ 855: Honda Danmark          ]
>    O  [1113: Honda Danmark          ]
>
> Doing the search with notmuch on the commandline:
>
>   $ notmuch search Honda | cut -d' ' -f1-5
>   thread:000000000000f081   2020-06-09 [1/1]
>   thread:000000000000ed7b   2020-06-03 [1/1]
>   thread:000000000000ee56   2020-05-26 [1/1]
>   thread:000000000000efe8   2020-05-18 [1/1]
>   thread:000000000000ed4e   2020-03-12 [1/1]
>   thread:000000000000ed4d   2020-03-12 [1/1]
>   thread:000000000000ef24   2020-01-14 [1/1]
>   thread:00000000000123a0   2019-09-05 [1/1]
>   thread:000000000001239f   2019-09-04 [1/1]
>   thread:000000000000c7cf   2018-10-22 [1/1]
>   thread:0000000000014de8   2018-03-04 [1/1]
>   thread:0000000000004e0d   2014-05-16 [1/1]
>   thread:0000000000009b6a   2000-04-12 [1/1(2)]
>
> (I cut the output to remove information I don't want to share).
>
>
>   Best regards,
>
>     Adam

-- 
7LL-7 Smile, you don’t own all the problems in the world.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0002-Rework-gnus-search-indexed-parse-output.patch --]
[-- Type: text/x-diff; name="0002-Rework-gnus-search-indexed-parse-output.patch", Size: 403 bytes --]

--- gnus-search-old.el	2021-06-29 13:00:20.478849987 +0200
+++ gnus-search.el	2021-06-29 13:00:28.226895659 +0200
@@ -1373,6 +1373,7 @@
                    nil t)
 	          nil t)
 	         nil t))
+	  (setq group (gnus-group-full-name group server))
           (setq article (file-name-nondirectory f-name)
                 article
                 ;; TODO: Provide a cleaner way of producing final

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-06-29 11:03         ` Thomas Alexander Gerds
@ 2021-06-29 11:13           ` Adam Sjøgren
  2021-06-29 11:25             ` Adam Sjøgren
  0 siblings, 1 reply; 34+ messages in thread
From: Adam Sjøgren @ 2021-06-29 11:13 UTC (permalink / raw)
  To: Thomas Alexander Gerds; +Cc: ding

Thomas writes:

> the attached patch applied to the patched file fixes this for me. 

That makes search work for me as well.


  👍,

   Adam

-- 
 "This is not going exactly as planned."                    Adam Sjøgren
                                                       asjo@koldfront.dk


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-06-29 11:13           ` Adam Sjøgren
@ 2021-06-29 11:25             ` Adam Sjøgren
  2021-06-30 16:58               ` Eric Abrahamsen
  0 siblings, 1 reply; 34+ messages in thread
From: Adam Sjøgren @ 2021-06-29 11:25 UTC (permalink / raw)
  To: ding

Adam writes:

> Thomas writes:
>
>> the attached patch applied to the patched file fixes this for me. 
>
> That makes search work for me as well.

In case Thomas' email doesn't reach the mailing list, here is the
combined diff I ended up with:

diff --git a/lisp/gnus/gnus-search.el b/lisp/gnus/gnus-search.el
index 70bde264c1..1b7b38a1cc 100644
--- a/lisp/gnus/gnus-search.el
+++ b/lisp/gnus/gnus-search.el
@@ -1351,68 +1351,59 @@ gnus-search-run-search
 
 (cl-defmethod gnus-search-indexed-parse-output ((engine gnus-search-indexed)
 						server query &optional groups)
-  (let ((prefix (slot-value engine 'remove-prefix))
-	(group-regexp (when groups
-			(mapconcat
-			 (lambda (group-name)
-			   (mapconcat #'regexp-quote
-				      (split-string
-				       (gnus-group-real-name group-name)
-				       "[.\\/]")
-				      "[.\\\\/]"))
-			 groups
-			 "\\|")))
-	artlist vectors article group)
+  (let ((prefix (or (slot-value engine 'remove-prefix)
+                    ""))
+	artlist article group)
     (goto-char (point-min))
+    ;; Prep prefix, we want to at least be removing the root
+    ;; filesystem separator.
+    (when (stringp prefix)
+      (setq prefix (file-name-as-directory
+                    (expand-file-name prefix "/"))))
     (while (not (or (eobp)
                     (looking-at-p
                      "\\(?:[[:space:]\n]+\\)?Process .+ finished")))
       (pcase-let ((`(,f-name ,score) (gnus-search-indexed-extract engine)))
 	(when (and f-name
                    (file-readable-p f-name)
-		   (null (file-directory-p f-name))
-		   (or (null groups)
-		       (and (gnus-search-single-p query)
-			    (alist-get 'thread query))
-		       (string-match-p group-regexp f-name)))
-	  (push (list f-name score) artlist))))
+		   (null (file-directory-p f-name)))
+          (setq group
+                (replace-regexp-in-string
+	         "[/\\]" "."
+	         (replace-regexp-in-string
+	          "/?\\(cur\\|new\\|tmp\\)?/\\'" ""
+	          (replace-regexp-in-string
+	           "\\`\\." ""
+	           (string-remove-prefix
+                    prefix (file-name-directory f-name))
+                   nil t)
+	          nil t)
+	         nil t))
+	  (setq group (gnus-group-full-name group server))
+          (setq article (file-name-nondirectory f-name)
+                article
+                ;; TODO: Provide a cleaner way of producing final
+                ;; article numbers for the various backends.
+                (if (string-match-p "\\`[[:digit:]]+\\'" article)
+		    (string-to-number article)
+		  (nnmaildir-base-name-to-article-number
+		   (substring article 0 (string-match ":" article))
+		   group (string-remove-prefix "nnmaildir:" server))))
+          (when (and (numberp article)
+                     (or (null groups)
+                         (member group groups)))
+	    (push (list f-name article group score)
+                  artlist)))))
     ;; Are we running an additional grep query?
     (when-let ((grep-reg (alist-get 'grep query)))
       (setq artlist (gnus-search-grep-search engine artlist grep-reg)))
-    ;; Prep prefix.
-    (when (and prefix (null (string-empty-p prefix)))
-      (setq prefix (file-name-as-directory (expand-file-name prefix))))
-    ;; Turn (file-name score) into [group article score].
-    (pcase-dolist (`(,f-name ,score) artlist)
-      (setq article (file-name-nondirectory f-name)
-	    group (file-name-directory f-name))
-      ;; Remove prefix.
-      (when prefix
-	(setq group (string-remove-prefix prefix group)))
-      ;; Break the directory name down until it's something that
-      ;; (probably) can be used as a group name.
-      (setq group
-	    (replace-regexp-in-string
-	     "[/\\]" "."
-	     (replace-regexp-in-string
-	      "/?\\(cur\\|new\\|tmp\\)?/\\'" ""
-	      (replace-regexp-in-string
-	       "^[./\\]" ""
-	       group nil t)
-	      nil t)
-	     nil t))
-
-      (push (vector (gnus-group-full-name group server)
-		    (if (string-match-p "\\`[[:digit:]]+\\'" article)
-			(string-to-number article)
-		      (nnmaildir-base-name-to-article-number
-		       (substring article 0 (string-match ":" article))
-		       group (string-remove-prefix "nnmaildir:" server)))
-		    (if (numberp score)
-			score
-		      (string-to-number score)))
-	    vectors))
-    vectors))
+    ;; Munge into the list of vectors expected by nnselect.
+    (mapcar (pcase-lambda (`(,_ ,article ,group ,score))
+              (vector group article
+                      (if (numberp score)
+			  score
+		        (string-to-number score))))
+            artlist)))
 
 (cl-defmethod gnus-search-indexed-extract ((_engine gnus-search-indexed))
   "Base implementation treats the whole line as a filename, and

-- 
 "Wiggle, wiggle, wiggle like a bowl of soup,               Adam Sjøgren
  Wiggle, wiggle, wiggle like a rolling hoop,          asjo@koldfront.dk



^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-06-28 16:42 Help testing gnus-search fixups Eric Abrahamsen
  2021-06-28 18:00 ` Adam Sjøgren
@ 2021-06-29 11:27 ` Eric S Fraga
  1 sibling, 0 replies; 34+ messages in thread
From: Eric S Fraga @ 2021-06-29 11:27 UTC (permalink / raw)
  To: ding

Hi Eric,
I'm very busy today and tomorrow but I'll try to give this a whirl soon.
-- 
Eric S Fraga via Emacs 28.0.50 & org 9.4.6 on Debian bullseye/sid



^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-06-29 11:25             ` Adam Sjøgren
@ 2021-06-30 16:58               ` Eric Abrahamsen
       [not found]                 ` <87bl6tuqcb.fsf@posteo.org>
  0 siblings, 1 reply; 34+ messages in thread
From: Eric Abrahamsen @ 2021-06-30 16:58 UTC (permalink / raw)
  To: Adam Sjøgren; +Cc: ding

[-- Attachment #1: Type: text/plain, Size: 589 bytes --]

Adam Sjøgren <asjo@koldfront.dk> writes:

> Adam writes:
>
>> Thomas writes:
>>
>>> the attached patch applied to the patched file fixes this for me. 
>>
>> That makes search work for me as well.
>
> In case Thomas' email doesn't reach the mailing list, here is the
> combined diff I ended up with:

Thanks to you both! That happened to slip by the test cases I was using,
and that's why we get extra testers! I've attached a new version of the
patch with your change applied; I'll give it a few days in case anyone
else wants to test it, then push.

Thanks,
Eric



[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-Rework-gnus-search-indexed-parse-output.patch --]
[-- Type: text/x-patch, Size: 5068 bytes --]

From b15fd217335f6711560ef88205ccfc262b4d2f25 Mon Sep 17 00:00:00 2001
From: Eric Abrahamsen <eric@ericabrahamsen.net>
Date: Sat, 26 Jun 2021 10:16:19 -0700
Subject: [PATCH] Rework gnus-search-indexed-parse-output

* lisp/gnus/gnus-search.el (gnus-search-indexed-parse-output): Be more
careful about matching filesystem paths to Gnus group names; make
absolutely sure that we only return valid article numbers.
---
 lisp/gnus/gnus-search.el | 95 ++++++++++++++++++----------------------
 1 file changed, 43 insertions(+), 52 deletions(-)

diff --git a/lisp/gnus/gnus-search.el b/lisp/gnus/gnus-search.el
index 70bde264c1..898b57bcef 100644
--- a/lisp/gnus/gnus-search.el
+++ b/lisp/gnus/gnus-search.el
@@ -1351,68 +1351,59 @@ gnus-search-run-search
 
 (cl-defmethod gnus-search-indexed-parse-output ((engine gnus-search-indexed)
 						server query &optional groups)
-  (let ((prefix (slot-value engine 'remove-prefix))
-	(group-regexp (when groups
-			(mapconcat
-			 (lambda (group-name)
-			   (mapconcat #'regexp-quote
-				      (split-string
-				       (gnus-group-real-name group-name)
-				       "[.\\/]")
-				      "[.\\\\/]"))
-			 groups
-			 "\\|")))
-	artlist vectors article group)
+  (let ((prefix (or (slot-value engine 'remove-prefix)
+                    ""))
+	artlist article group)
     (goto-char (point-min))
+    ;; Prep prefix, we want to at least be removing the root
+    ;; filesystem separator.
+    (when (stringp prefix)
+      (setq prefix (file-name-as-directory
+                    (expand-file-name prefix "/"))))
     (while (not (or (eobp)
                     (looking-at-p
                      "\\(?:[[:space:]\n]+\\)?Process .+ finished")))
       (pcase-let ((`(,f-name ,score) (gnus-search-indexed-extract engine)))
 	(when (and f-name
                    (file-readable-p f-name)
-		   (null (file-directory-p f-name))
-		   (or (null groups)
-		       (and (gnus-search-single-p query)
-			    (alist-get 'thread query))
-		       (string-match-p group-regexp f-name)))
-	  (push (list f-name score) artlist))))
+		   (null (file-directory-p f-name)))
+          (setq group
+                (replace-regexp-in-string
+	         "[/\\]" "."
+	         (replace-regexp-in-string
+	          "/?\\(cur\\|new\\|tmp\\)?/\\'" ""
+	          (replace-regexp-in-string
+	           "\\`\\." ""
+	           (string-remove-prefix
+                    prefix (file-name-directory f-name))
+                   nil t)
+	          nil t)
+	         nil t))
+          (setq group (gnus-group-full-name group server))
+          (setq article (file-name-nondirectory f-name)
+                article
+                ;; TODO: Provide a cleaner way of producing final
+                ;; article numbers for the various backends.
+                (if (string-match-p "\\`[[:digit:]]+\\'" article)
+		    (string-to-number article)
+		  (nnmaildir-base-name-to-article-number
+		   (substring article 0 (string-match ":" article))
+		   group (string-remove-prefix "nnmaildir:" server))))
+          (when (and (numberp article)
+                     (or (null groups)
+                         (member group groups)))
+	    (push (list f-name article group score)
+                  artlist)))))
     ;; Are we running an additional grep query?
     (when-let ((grep-reg (alist-get 'grep query)))
       (setq artlist (gnus-search-grep-search engine artlist grep-reg)))
-    ;; Prep prefix.
-    (when (and prefix (null (string-empty-p prefix)))
-      (setq prefix (file-name-as-directory (expand-file-name prefix))))
-    ;; Turn (file-name score) into [group article score].
-    (pcase-dolist (`(,f-name ,score) artlist)
-      (setq article (file-name-nondirectory f-name)
-	    group (file-name-directory f-name))
-      ;; Remove prefix.
-      (when prefix
-	(setq group (string-remove-prefix prefix group)))
-      ;; Break the directory name down until it's something that
-      ;; (probably) can be used as a group name.
-      (setq group
-	    (replace-regexp-in-string
-	     "[/\\]" "."
-	     (replace-regexp-in-string
-	      "/?\\(cur\\|new\\|tmp\\)?/\\'" ""
-	      (replace-regexp-in-string
-	       "^[./\\]" ""
-	       group nil t)
-	      nil t)
-	     nil t))
-
-      (push (vector (gnus-group-full-name group server)
-		    (if (string-match-p "\\`[[:digit:]]+\\'" article)
-			(string-to-number article)
-		      (nnmaildir-base-name-to-article-number
-		       (substring article 0 (string-match ":" article))
-		       group (string-remove-prefix "nnmaildir:" server)))
-		    (if (numberp score)
-			score
-		      (string-to-number score)))
-	    vectors))
-    vectors))
+    ;; Munge into the list of vectors expected by nnselect.
+    (mapcar (pcase-lambda (`(,_ ,article ,group ,score))
+              (vector group article
+                      (if (numberp score)
+			  score
+		        (string-to-number score))))
+            artlist)))
 
 (cl-defmethod gnus-search-indexed-extract ((_engine gnus-search-indexed))
   "Base implementation treats the whole line as a filename, and
-- 
2.32.0


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
       [not found]                 ` <87bl6tuqcb.fsf@posteo.org>
@ 2021-07-23 15:18                   ` Eric Abrahamsen
       [not found]                     ` <87bl6sual8.fsf@posteo.org>
  0 siblings, 1 reply; 34+ messages in thread
From: Eric Abrahamsen @ 2021-07-23 15:18 UTC (permalink / raw)
  To: Alexandr Vityazev; +Cc: ding


On 07/23/21 13:12 PM, Alexandr Vityazev wrote:
> Hi!
>
> GNU Emacs 28.0.50 (build 1, x86_64-pc-linux-gnu, GTK+ Version 3.24.24,
> cairo version 1.16.0). Gnus v5.13.
>
> I tested the new version of gnus-search-indexed-parse-output yesterday
> and the search with notmuch completely fails. gnus-verbose is set to
> 10. press GG and type any search query, with the patch, I get:
>
>   Doing /home/akagi/.guix-profile/bin/notmuch query on
>   (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>   nnmaildir+posteo:Trash nnmaildir+posteo:Notes)...
>   Group nnselect:nnselect-87lf5xus2s.fsf contains no messages

Hi, thanks for the report. Would you mind running your query on the
command line (using the same syntax as gnus-search does, mostly
output=files), and sending me the resulting list of file names
(off-list, if you prefer). I'll try to find out why the parsing process
is failing.


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
       [not found]                     ` <87bl6sual8.fsf@posteo.org>
@ 2021-07-24 22:16                       ` Eric Abrahamsen
  2021-07-25 11:17                         ` Alexandr Vityazev
  0 siblings, 1 reply; 34+ messages in thread
From: Eric Abrahamsen @ 2021-07-24 22:16 UTC (permalink / raw)
  To: Alexandr Vityazev; +Cc: ding


On 07/23/21 18:52 PM, Alexandr Vityazev wrote:
> On 2021-07-23, 08:18 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>
>> On 07/23/21 13:12 PM, Alexandr Vityazev wrote:
>>> Hi!
>>>
>>> GNU Emacs 28.0.50 (build 1, x86_64-pc-linux-gnu, GTK+ Version 3.24.24,
>>> cairo version 1.16.0). Gnus v5.13.
>>>
>>> I tested the new version of gnus-search-indexed-parse-output yesterday
>>> and the search with notmuch completely fails. gnus-verbose is set to
>>> 10. press GG and type any search query, with the patch, I get:
>>>
>>>   Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>   (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>   nnmaildir+posteo:Trash nnmaildir+posteo:Notes)...
>>>   Group nnselect:nnselect-87lf5xus2s.fsf contains no messages
>>
>> Hi, thanks for the report. Would you mind running your query on the
>> command line (using the same syntax as gnus-search does, mostly
>> output=files), and sending me the resulting list of file names
>> (off-list, if you prefer). I'll try to find out why the parsing process
>> is failing.
>>
>>
> notmuch search --output=files test
>
>
>
> in *Group* GG with "test" query, I get:
>
>    Doing /home/akagi/.guix-profile/bin/notmuch query on
>    (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>    nnmaildir+posteo:Trash nnmaildir+posteo:Notes nnmaildir+posteo:Junk)...
>    Group nnselect:nnselect-87czr8uasp.fsf contains no messages

Hmm, I'm not able to reproduce this: it's a little difficult since I
don't actually have the files on my machine, but as far as I can tell
the search results are being parsed correctly. Are you familiar with
edebug at all? Is there any chance I could get you to step through the
function and see where things are going off?

Thanks,
Eric


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-07-24 22:16                       ` Eric Abrahamsen
@ 2021-07-25 11:17                         ` Alexandr Vityazev
  2021-07-25 22:11                           ` Eric Abrahamsen
  2021-07-30 18:17                           ` Joseph Mingrone
  0 siblings, 2 replies; 34+ messages in thread
From: Alexandr Vityazev @ 2021-07-25 11:17 UTC (permalink / raw)
  To: Eric Abrahamsen; +Cc: ding

[-- Attachment #1: Type: text/plain, Size: 2059 bytes --]

On 2021-07-24, 15:16 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:

> On 07/23/21 18:52 PM, Alexandr Vityazev wrote:
>> On 2021-07-23, 08:18 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>>
>>> On 07/23/21 13:12 PM, Alexandr Vityazev wrote:
>>>> Hi!
>>>>
>>>> GNU Emacs 28.0.50 (build 1, x86_64-pc-linux-gnu, GTK+ Version 3.24.24,
>>>> cairo version 1.16.0). Gnus v5.13.
>>>>
>>>> I tested the new version of gnus-search-indexed-parse-output yesterday
>>>> and the search with notmuch completely fails. gnus-verbose is set to
>>>> 10. press GG and type any search query, with the patch, I get:
>>>>
>>>>   Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>>   (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>>   nnmaildir+posteo:Trash nnmaildir+posteo:Notes)...
>>>>   Group nnselect:nnselect-87lf5xus2s.fsf contains no messages
>>>
>>> Hi, thanks for the report. Would you mind running your query on the
>>> command line (using the same syntax as gnus-search does, mostly
>>> output=files), and sending me the resulting list of file names
>>> (off-list, if you prefer). I'll try to find out why the parsing process
>>> is failing.
>>>
>>>
>> notmuch search --output=files test
>>
>>
>>
>> in *Group* GG with "test" query, I get:
>>
>>    Doing /home/akagi/.guix-profile/bin/notmuch query on
>>    (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>    nnmaildir+posteo:Trash nnmaildir+posteo:Notes nnmaildir+posteo:Junk)...
>>    Group nnselect:nnselect-87czr8uasp.fsf contains no messages
>
> Hmm, I'm not able to reproduce this: it's a little difficult since I
> don't actually have the files on my machine, but as far as I can tell
> the search results are being parsed correctly. Are you familiar with
> edebug at all? Is there any chance I could get you to step through the
> function and see where things are going off?
>
> Thanks,
> Eric
>
>
I tested gnus-search-indexed-parse-output with edebug and found where
something is going on. The attached patch fixes the issue for me.


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: gnus-search-indexed-parse-output --]
[-- Type: text/x-patch, Size: 1383 bytes --]

From bfba4f38289223c3e78b19a73ad9133d015f8420 Mon Sep 17 00:00:00 2001
From: Alexandr Vityazev <avityazev@posteo.org>
Date: Sun, 25 Jul 2021 14:09:39 +0300
Subject: [PATCH] gnus-search: gnus-search-indexed-parse-output: Fix search
 issue.

* lisp/gnus/gnus-search.el (gnus-search-indexed-parse-output):
Setting the group variable after the article variable.
---
 lisp/gnus/gnus-search.el | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lisp/gnus/gnus-search.el b/lisp/gnus/gnus-search.el
index 39bde837b3..80c3ce62f1 100644
--- a/lisp/gnus/gnus-search.el
+++ b/lisp/gnus/gnus-search.el
@@ -1384,7 +1384,6 @@ Returns a list of [group article score] vectors."
                    nil t)
 	          nil t)
 	         nil t))
-          (setq group (gnus-group-full-name group server))
           (setq article (file-name-nondirectory f-name)
                 article
                 ;; TODO: Provide a cleaner way of producing final
@@ -1394,6 +1393,7 @@ Returns a list of [group article score] vectors."
 		  (nnmaildir-base-name-to-article-number
 		   (substring article 0 (string-match ":" article))
 		   group (string-remove-prefix "nnmaildir:" server))))
+          (setq group (gnus-group-full-name group server))
           (when (and (numberp article)
                      (or (null groups)
                          (member group groups)))
-- 
2.32.0


[-- Attachment #3: Type: text/plain, Size: 37 bytes --]


-- 
Best regards,
Alexandr Vityazev

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-07-25 11:17                         ` Alexandr Vityazev
@ 2021-07-25 22:11                           ` Eric Abrahamsen
  2021-07-30 18:17                           ` Joseph Mingrone
  1 sibling, 0 replies; 34+ messages in thread
From: Eric Abrahamsen @ 2021-07-25 22:11 UTC (permalink / raw)
  To: Alexandr Vityazev; +Cc: ding


On 07/25/21 11:17 AM, Alexandr Vityazev wrote:
> On 2021-07-24, 15:16 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>
>> On 07/23/21 18:52 PM, Alexandr Vityazev wrote:
>>> On 2021-07-23, 08:18 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>>>
>>>> On 07/23/21 13:12 PM, Alexandr Vityazev wrote:
>>>>> Hi!
>>>>>
>>>>> GNU Emacs 28.0.50 (build 1, x86_64-pc-linux-gnu, GTK+ Version 3.24.24,
>>>>> cairo version 1.16.0). Gnus v5.13.
>>>>>
>>>>> I tested the new version of gnus-search-indexed-parse-output yesterday
>>>>> and the search with notmuch completely fails. gnus-verbose is set to
>>>>> 10. press GG and type any search query, with the patch, I get:
>>>>>
>>>>>   Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>>>   (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>>>   nnmaildir+posteo:Trash nnmaildir+posteo:Notes)...
>>>>>   Group nnselect:nnselect-87lf5xus2s.fsf contains no messages
>>>>
>>>> Hi, thanks for the report. Would you mind running your query on the
>>>> command line (using the same syntax as gnus-search does, mostly
>>>> output=files), and sending me the resulting list of file names
>>>> (off-list, if you prefer). I'll try to find out why the parsing process
>>>> is failing.
>>>>
>>>>
>>> notmuch search --output=files test
>>>
>>>
>>>
>>> in *Group* GG with "test" query, I get:
>>>
>>>    Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>    (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>    nnmaildir+posteo:Trash nnmaildir+posteo:Notes nnmaildir+posteo:Junk)...
>>>    Group nnselect:nnselect-87czr8uasp.fsf contains no messages
>>
>> Hmm, I'm not able to reproduce this: it's a little difficult since I
>> don't actually have the files on my machine, but as far as I can tell
>> the search results are being parsed correctly. Are you familiar with
>> edebug at all? Is there any chance I could get you to step through the
>> function and see where things are going off?
>>
>> Thanks,
>> Eric
>>
>>
> I tested gnus-search-indexed-parse-output with edebug and found where
> something is going on. The attached patch fixes the issue for me.

Wonderful, thank you very much for doing that. That code was added in to
fix a different bug, and I think we're just playing the usual "is the
group prefixed or not" game. I need to more thoroughly test the
primary/secondary select method cases, but at least we know what the
issue is -- thanks again for your debugging!

Eric


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-07-25 11:17                         ` Alexandr Vityazev
  2021-07-25 22:11                           ` Eric Abrahamsen
@ 2021-07-30 18:17                           ` Joseph Mingrone
  2021-07-30 18:22                             ` Joseph Mingrone
  2021-07-30 19:34                             ` Alexandr Vityazev
  1 sibling, 2 replies; 34+ messages in thread
From: Joseph Mingrone @ 2021-07-30 18:17 UTC (permalink / raw)
  To: ding; +Cc: Eric Abrahamsen, Alexandr Vityazev

On Sun, 2021-07-25 at 11:17, Alexandr Vityazev <avityazev@posteo.org> wrote:

> On 2021-07-24, 15:16 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:

>> On 07/23/21 18:52 PM, Alexandr Vityazev wrote:
>>> On 2021-07-23, 08:18 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:

>>>> On 07/23/21 13:12 PM, Alexandr Vityazev wrote:
>>>>> Hi!

>>>>> GNU Emacs 28.0.50 (build 1, x86_64-pc-linux-gnu, GTK+ Version 3.24.24,
>>>>> cairo version 1.16.0). Gnus v5.13.

>>>>> I tested the new version of gnus-search-indexed-parse-output yesterday
>>>>> and the search with notmuch completely fails. gnus-verbose is set to
>>>>> 10. press GG and type any search query, with the patch, I get:

>>>>>   Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>>>   (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>>>   nnmaildir+posteo:Trash nnmaildir+posteo:Notes)...
>>>>>   Group nnselect:nnselect-87lf5xus2s.fsf contains no messages

>>>> Hi, thanks for the report. Would you mind running your query on the
>>>> command line (using the same syntax as gnus-search does, mostly
>>>> output=files), and sending me the resulting list of file names
>>>> (off-list, if you prefer). I'll try to find out why the parsing process
>>>> is failing.


>>> notmuch search --output=files test



>>> in *Group* GG with "test" query, I get:

>>>    Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>    (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>    nnmaildir+posteo:Trash nnmaildir+posteo:Notes nnmaildir+posteo:Junk)...
>>>    Group nnselect:nnselect-87czr8uasp.fsf contains no messages

>> Hmm, I'm not able to reproduce this: it's a little difficult since I
>> don't actually have the files on my machine, but as far as I can tell
>> the search results are being parsed correctly. Are you familiar with
>> edebug at all? Is there any chance I could get you to step through the
>> function and see where things are going off?

>> Thanks,
>> Eric


> I tested gnus-search-indexed-parse-output with edebug and found where
> something is going on. The attached patch fixes the issue for me.

Hello all,

Alexandr, I applied this patch against the latest commit to master at
the time of writing (55a9c17cef) and I still get no search results when
there should be.  With `gnus-verbose' and `gnus-verbose-backends', I
just see

Retrieving newsgroup: nnselect:nnselect-86im0r65gy.fsf...
Doing notmuch query on (test)...
Group nnselect:nnselect-86im0r65gy.fsf contains no messages

in *Messages*.

My search setup is simple.  I just have

'((nnimap . imap) (nntp . gmane) (nnml . notmuch)) as the value of
`gnus-search-default-engines'.

The only way I can get searching working again is to revert
gnus-search.el to 0897ade8f9.

Regards,

Joe


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-07-30 18:17                           ` Joseph Mingrone
@ 2021-07-30 18:22                             ` Joseph Mingrone
  2021-07-30 18:47                               ` Eric Abrahamsen
  2021-07-30 19:34                             ` Alexandr Vityazev
  1 sibling, 1 reply; 34+ messages in thread
From: Joseph Mingrone @ 2021-07-30 18:22 UTC (permalink / raw)
  To: ding; +Cc: Eric Abrahamsen, Alexandr Vityazev

On Fri, 2021-07-30 at 15:17, Joseph Mingrone <jrm@ftfl.ca> wrote:

> On Sun, 2021-07-25 at 11:17, Alexandr Vityazev <avityazev@posteo.org> wrote:

>> On 2021-07-24, 15:16 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:

>>> On 07/23/21 18:52 PM, Alexandr Vityazev wrote:
>>>> On 2021-07-23, 08:18 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:

>>>>> On 07/23/21 13:12 PM, Alexandr Vityazev wrote:
>>>>>> Hi!

>>>>>> GNU Emacs 28.0.50 (build 1, x86_64-pc-linux-gnu, GTK+ Version 3.24.24,
>>>>>> cairo version 1.16.0). Gnus v5.13.

>>>>>> I tested the new version of gnus-search-indexed-parse-output yesterday
>>>>>> and the search with notmuch completely fails. gnus-verbose is set to
>>>>>> 10. press GG and type any search query, with the patch, I get:

>>>>>>   Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>>>>   (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>>>>   nnmaildir+posteo:Trash nnmaildir+posteo:Notes)...
>>>>>>   Group nnselect:nnselect-87lf5xus2s.fsf contains no messages

>>>>> Hi, thanks for the report. Would you mind running your query on the
>>>>> command line (using the same syntax as gnus-search does, mostly
>>>>> output=files), and sending me the resulting list of file names
>>>>> (off-list, if you prefer). I'll try to find out why the parsing process
>>>>> is failing.


>>>> notmuch search --output=files test



>>>> in *Group* GG with "test" query, I get:

>>>>    Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>>    (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>>    nnmaildir+posteo:Trash nnmaildir+posteo:Notes nnmaildir+posteo:Junk)...
>>>>    Group nnselect:nnselect-87czr8uasp.fsf contains no messages

>>> Hmm, I'm not able to reproduce this: it's a little difficult since I
>>> don't actually have the files on my machine, but as far as I can tell
>>> the search results are being parsed correctly. Are you familiar with
>>> edebug at all? Is there any chance I could get you to step through the
>>> function and see where things are going off?

>>> Thanks,
>>> Eric


>> I tested gnus-search-indexed-parse-output with edebug and found where
>> something is going on. The attached patch fixes the issue for me.

> Hello all,

> Alexandr, I applied this patch against the latest commit to master at
> the time of writing (55a9c17cef) and I still get no search results when
> there should be.  With `gnus-verbose' and `gnus-verbose-backends', I
> just see

Sorry.  That should have read with `gnus-verbose' and
`gnus-verbose-backends' both set to 10.

> Retrieving newsgroup: nnselect:nnselect-86im0r65gy.fsf...
> Doing notmuch query on (test)...
> Group nnselect:nnselect-86im0r65gy.fsf contains no messages

> in *Messages*.

> My search setup is simple.  I just have

> '((nnimap . imap) (nntp . gmane) (nnml . notmuch)) as the value of
> `gnus-search-default-engines'.

> The only way I can get searching working again is to revert
> gnus-search.el to 0897ade8f9.

> Regards,

> Joe


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-07-30 18:22                             ` Joseph Mingrone
@ 2021-07-30 18:47                               ` Eric Abrahamsen
  2021-07-30 19:29                                 ` Joseph Mingrone
  0 siblings, 1 reply; 34+ messages in thread
From: Eric Abrahamsen @ 2021-07-30 18:47 UTC (permalink / raw)
  To: Joseph Mingrone; +Cc: ding, Alexandr Vityazev

Joseph Mingrone <jrm@ftfl.ca> writes:

> On Fri, 2021-07-30 at 15:17, Joseph Mingrone <jrm@ftfl.ca> wrote:
>
>> On Sun, 2021-07-25 at 11:17, Alexandr Vityazev <avityazev@posteo.org> wrote:
>
>>> On 2021-07-24, 15:16 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>
>>>> On 07/23/21 18:52 PM, Alexandr Vityazev wrote:
>>>>> On 2021-07-23, 08:18 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>
>>>>>> On 07/23/21 13:12 PM, Alexandr Vityazev wrote:
>>>>>>> Hi!
>
>>>>>>> GNU Emacs 28.0.50 (build 1, x86_64-pc-linux-gnu, GTK+ Version 3.24.24,
>>>>>>> cairo version 1.16.0). Gnus v5.13.
>
>>>>>>> I tested the new version of gnus-search-indexed-parse-output yesterday
>>>>>>> and the search with notmuch completely fails. gnus-verbose is set to
>>>>>>> 10. press GG and type any search query, with the patch, I get:
>
>>>>>>>   Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>>>>>   (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>>>>>   nnmaildir+posteo:Trash nnmaildir+posteo:Notes)...
>>>>>>>   Group nnselect:nnselect-87lf5xus2s.fsf contains no messages
>
>>>>>> Hi, thanks for the report. Would you mind running your query on the
>>>>>> command line (using the same syntax as gnus-search does, mostly
>>>>>> output=files), and sending me the resulting list of file names
>>>>>> (off-list, if you prefer). I'll try to find out why the parsing process
>>>>>> is failing.
>
>
>>>>> notmuch search --output=files test
>
>
>
>>>>> in *Group* GG with "test" query, I get:
>
>>>>>    Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>>>    (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>>>    nnmaildir+posteo:Trash nnmaildir+posteo:Notes nnmaildir+posteo:Junk)...
>>>>>    Group nnselect:nnselect-87czr8uasp.fsf contains no messages
>
>>>> Hmm, I'm not able to reproduce this: it's a little difficult since I
>>>> don't actually have the files on my machine, but as far as I can tell
>>>> the search results are being parsed correctly. Are you familiar with
>>>> edebug at all? Is there any chance I could get you to step through the
>>>> function and see where things are going off?
>
>>>> Thanks,
>>>> Eric
>
>
>>> I tested gnus-search-indexed-parse-output with edebug and found where
>>> something is going on. The attached patch fixes the issue for me.
>
>> Hello all,
>
>> Alexandr, I applied this patch against the latest commit to master at
>> the time of writing (55a9c17cef) and I still get no search results when
>> there should be.  With `gnus-verbose' and `gnus-verbose-backends', I
>> just see
>
> Sorry.  That should have read with `gnus-verbose' and
> `gnus-verbose-backends' both set to 10.

Remind me which of your servers is primary and which secondary? I'm
still pretty sure that the issue is prefixed vs unprefixed group names.


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-07-30 18:47                               ` Eric Abrahamsen
@ 2021-07-30 19:29                                 ` Joseph Mingrone
  2021-07-30 20:23                                   ` Eric Abrahamsen
  0 siblings, 1 reply; 34+ messages in thread
From: Joseph Mingrone @ 2021-07-30 19:29 UTC (permalink / raw)
  To: Eric Abrahamsen; +Cc: ding, Alexandr Vityazev

On Fri, 2021-07-30 at 11:47, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
> Remind me which of your servers is primary and which secondary? I'm
> still pretty sure that the issue is prefixed vs unprefixed group names.

`gnus-message-archive-method' is '(nnml ""), both `gnus-select-method'
and `gnus-message-archive-method' are set to (nnml ""), and
`gnus-search-default-engines' is '((nnimap . imap) (nntp . gmane) (nnml
. notmuch))).  Gnus stores mail in the standard location, ~/Mail.

Is that what you mean?  I can step through the code if you can give me a
hint of what I should be looking for.


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-07-30 18:17                           ` Joseph Mingrone
  2021-07-30 18:22                             ` Joseph Mingrone
@ 2021-07-30 19:34                             ` Alexandr Vityazev
  2021-07-30 20:25                               ` Eric Abrahamsen
  1 sibling, 1 reply; 34+ messages in thread
From: Alexandr Vityazev @ 2021-07-30 19:34 UTC (permalink / raw)
  To: Joseph Mingrone; +Cc: ding, Eric Abrahamsen

On 2021-07-30, 15:17 -0300, Joseph Mingrone <jrm@ftfl.ca> wrote:

> On Sun, 2021-07-25 at 11:17, Alexandr Vityazev <avityazev@posteo.org> wrote:
>
>> On 2021-07-24, 15:16 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>
>>> On 07/23/21 18:52 PM, Alexandr Vityazev wrote:
>>>> On 2021-07-23, 08:18 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>
>>>>> On 07/23/21 13:12 PM, Alexandr Vityazev wrote:
>>>>>> Hi!
>
>>>>>> GNU Emacs 28.0.50 (build 1, x86_64-pc-linux-gnu, GTK+ Version 3.24.24,
>>>>>> cairo version 1.16.0). Gnus v5.13.
>
>>>>>> I tested the new version of gnus-search-indexed-parse-output yesterday
>>>>>> and the search with notmuch completely fails. gnus-verbose is set to
>>>>>> 10. press GG and type any search query, with the patch, I get:
>
>>>>>>   Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>>>>   (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>>>>   nnmaildir+posteo:Trash nnmaildir+posteo:Notes)...
>>>>>>   Group nnselect:nnselect-87lf5xus2s.fsf contains no messages
>
>>>>> Hi, thanks for the report. Would you mind running your query on the
>>>>> command line (using the same syntax as gnus-search does, mostly
>>>>> output=files), and sending me the resulting list of file names
>>>>> (off-list, if you prefer). I'll try to find out why the parsing process
>>>>> is failing.
>
>
>>>> notmuch search --output=files test
>
>
>
>>>> in *Group* GG with "test" query, I get:
>
>>>>    Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>>    (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>>    nnmaildir+posteo:Trash nnmaildir+posteo:Notes nnmaildir+posteo:Junk)...
>>>>    Group nnselect:nnselect-87czr8uasp.fsf contains no messages
>
>>> Hmm, I'm not able to reproduce this: it's a little difficult since I
>>> don't actually have the files on my machine, but as far as I can tell
>>> the search results are being parsed correctly. Are you familiar with
>>> edebug at all? Is there any chance I could get you to step through the
>>> function and see where things are going off?
>
>>> Thanks,
>>> Eric
>
>
>> I tested gnus-search-indexed-parse-output with edebug and found where
>> something is going on. The attached patch fixes the issue for me.
>
> Hello all,
>
> Alexandr, I applied this patch against the latest commit to master at
> the time of writing (55a9c17cef) and I still get no search results when
> there should be.  With `gnus-verbose' and `gnus-verbose-backends', I
> just see
>
> Retrieving newsgroup: nnselect:nnselect-86im0r65gy.fsf...
> Doing notmuch query on (test)...
> Group nnselect:nnselect-86im0r65gy.fsf contains no messages
>
> in *Messages*.
>
> My search setup is simple.  I just have
>
> '((nnimap . imap) (nntp . gmane) (nnml . notmuch)) as the value of
> `gnus-search-default-engines'.
>
> The only way I can get searching working again is to revert
> gnus-search.el to 0897ade8f9.
>
> Regards,
>
> Joe
>
>

Hi, Joseph!

All my methods are set in gnus-secondary-select-methods. Most likely the
patch solves the problem only with these settings. I did not research
whether it will work with other settings. To solve the problem with the
search it is necessary to conduct a more complete analysis, I still
cannot give an answer.

-- 
Best regards,
Alexandr Vityazev



^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-07-30 19:29                                 ` Joseph Mingrone
@ 2021-07-30 20:23                                   ` Eric Abrahamsen
  0 siblings, 0 replies; 34+ messages in thread
From: Eric Abrahamsen @ 2021-07-30 20:23 UTC (permalink / raw)
  To: Joseph Mingrone; +Cc: ding, Alexandr Vityazev

Joseph Mingrone <jrm@ftfl.ca> writes:

> On Fri, 2021-07-30 at 11:47, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>> Remind me which of your servers is primary and which secondary? I'm
>> still pretty sure that the issue is prefixed vs unprefixed group names.
>
> `gnus-message-archive-method' is '(nnml ""), both `gnus-select-method'
> and `gnus-message-archive-method' are set to (nnml ""), and
> `gnus-search-default-engines' is '((nnimap . imap) (nntp . gmane) (nnml
> . notmuch))).  Gnus stores mail in the standard location, ~/Mail.
>
> Is that what you mean?  I can step through the code if you can give me a
> hint of what I should be looking for.

That's fine, it means your `gnus-select-method' is 'nnml, and the other
servers are all `gnus-secondary-select-methods'. That's all I was asking.


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-07-30 19:34                             ` Alexandr Vityazev
@ 2021-07-30 20:25                               ` Eric Abrahamsen
  2021-08-05 17:15                                 ` Eric Abrahamsen
  0 siblings, 1 reply; 34+ messages in thread
From: Eric Abrahamsen @ 2021-07-30 20:25 UTC (permalink / raw)
  To: Alexandr Vityazev; +Cc: Joseph Mingrone, ding

Alexandr Vityazev <avityazev@posteo.org> writes:

> On 2021-07-30, 15:17 -0300, Joseph Mingrone <jrm@ftfl.ca> wrote:
>
>> On Sun, 2021-07-25 at 11:17, Alexandr Vityazev <avityazev@posteo.org> wrote:
>>
>>> On 2021-07-24, 15:16 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>>
>>>> On 07/23/21 18:52 PM, Alexandr Vityazev wrote:
>>>>> On 2021-07-23, 08:18 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>>
>>>>>> On 07/23/21 13:12 PM, Alexandr Vityazev wrote:
>>>>>>> Hi!
>>
>>>>>>> GNU Emacs 28.0.50 (build 1, x86_64-pc-linux-gnu, GTK+ Version 3.24.24,
>>>>>>> cairo version 1.16.0). Gnus v5.13.
>>
>>>>>>> I tested the new version of gnus-search-indexed-parse-output yesterday
>>>>>>> and the search with notmuch completely fails. gnus-verbose is set to
>>>>>>> 10. press GG and type any search query, with the patch, I get:
>>
>>>>>>>   Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>>>>>   (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>>>>>   nnmaildir+posteo:Trash nnmaildir+posteo:Notes)...
>>>>>>>   Group nnselect:nnselect-87lf5xus2s.fsf contains no messages
>>
>>>>>> Hi, thanks for the report. Would you mind running your query on the
>>>>>> command line (using the same syntax as gnus-search does, mostly
>>>>>> output=files), and sending me the resulting list of file names
>>>>>> (off-list, if you prefer). I'll try to find out why the parsing process
>>>>>> is failing.
>>
>>
>>>>> notmuch search --output=files test
>>
>>
>>
>>>>> in *Group* GG with "test" query, I get:
>>
>>>>>    Doing /home/akagi/.guix-profile/bin/notmuch query on
>>>>>    (nnmaildir+posteo:Drafts nnmaildir+posteo:Inbox nnmaildir+posteo:Sent
>>>>>    nnmaildir+posteo:Trash nnmaildir+posteo:Notes nnmaildir+posteo:Junk)...
>>>>>    Group nnselect:nnselect-87czr8uasp.fsf contains no messages
>>
>>>> Hmm, I'm not able to reproduce this: it's a little difficult since I
>>>> don't actually have the files on my machine, but as far as I can tell
>>>> the search results are being parsed correctly. Are you familiar with
>>>> edebug at all? Is there any chance I could get you to step through the
>>>> function and see where things are going off?
>>
>>>> Thanks,
>>>> Eric
>>
>>
>>> I tested gnus-search-indexed-parse-output with edebug and found where
>>> something is going on. The attached patch fixes the issue for me.
>>
>> Hello all,
>>
>> Alexandr, I applied this patch against the latest commit to master at
>> the time of writing (55a9c17cef) and I still get no search results when
>> there should be.  With `gnus-verbose' and `gnus-verbose-backends', I
>> just see
>>
>> Retrieving newsgroup: nnselect:nnselect-86im0r65gy.fsf...
>> Doing notmuch query on (test)...
>> Group nnselect:nnselect-86im0r65gy.fsf contains no messages
>>
>> in *Messages*.
>>
>> My search setup is simple.  I just have
>>
>> '((nnimap . imap) (nntp . gmane) (nnml . notmuch)) as the value of
>> `gnus-search-default-engines'.
>>
>> The only way I can get searching working again is to revert
>> gnus-search.el to 0897ade8f9.
>>
>> Regards,
>>
>> Joe
>>
>>
>
> Hi, Joseph!
>
> All my methods are set in gnus-secondary-select-methods. Most likely the
> patch solves the problem only with these settings. I did not research
> whether it will work with other settings. To solve the problem with the
> search it is necessary to conduct a more complete analysis, I still
> cannot give an answer.

I'm getting to it! I just need to find a moment to sit down and set up
some dummy data and test the various possible configurations. It
shouldn't be to hard to unilaterally force prefixed group names where
that's needed, and force unprefixed names elsewhere, I just need to
find the time. I'd like to get this fixed once and for all.


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-07-30 20:25                               ` Eric Abrahamsen
@ 2021-08-05 17:15                                 ` Eric Abrahamsen
  2021-08-05 19:54                                   ` Alexandr Vityazev
  0 siblings, 1 reply; 34+ messages in thread
From: Eric Abrahamsen @ 2021-08-05 17:15 UTC (permalink / raw)
  To: Alexandr Vityazev; +Cc: Joseph Mingrone, ding

[-- Attachment #1: Type: text/plain, Size: 788 bytes --]

Eric Abrahamsen <eric@ericabrahamsen.net> writes:


[...]

> I'm getting to it! I just need to find a moment to sit down and set up
> some dummy data and test the various possible configurations. It
> shouldn't be to hard to unilaterally force prefixed group names where
> that's needed, and force unprefixed names elsewhere, I just need to
> find the time. I'd like to get this fixed once and for all.

Okay, I think I've finally got it. The incoming group names are indeed
prefixed or not depending on whether the server is primary or not
(arguably this should be fixed to always unprefixed in nnselect, but
I'll worry about that later), so this patch enforces unprefixed group
names within the function body, and prefixed group names in the return
value. Please give it a shot!

Eric


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: gnus-search-groups-fix.diff --]
[-- Type: text/x-patch, Size: 1172 bytes --]

diff --git a/lisp/gnus/gnus-search.el b/lisp/gnus/gnus-search.el
index 53af2f6fe6..d92122f2bd 100644
--- a/lisp/gnus/gnus-search.el
+++ b/lisp/gnus/gnus-search.el
@@ -1358,6 +1358,7 @@ gnus-search-indexed-parse-output
 						server query &optional groups)
   (let ((prefix (or (slot-value engine 'remove-prefix)
                     ""))
+        (groups (mapcar #'gnus-group-short-name groups))
 	artlist article group)
     (goto-char (point-min))
     ;; Prep prefix, we want to at least be removing the root
@@ -1404,10 +1405,12 @@ gnus-search-indexed-parse-output
       (setq artlist (gnus-search-grep-search engine artlist grep-reg)))
     ;; Munge into the list of vectors expected by nnselect.
     (mapcar (pcase-lambda (`(,_ ,article ,group ,score))
-              (vector group article
-                      (if (numberp score)
-			  score
-		        (string-to-number score))))
+              (vector
+               (gnus-group-full-name group server)
+               article
+               (if (numberp score)
+		   score
+		 (string-to-number score))))
             artlist)))
 
 (cl-defmethod gnus-search-indexed-extract ((_engine gnus-search-indexed))

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-08-05 17:15                                 ` Eric Abrahamsen
@ 2021-08-05 19:54                                   ` Alexandr Vityazev
  2021-08-05 20:19                                     ` Joseph Mingrone
  0 siblings, 1 reply; 34+ messages in thread
From: Alexandr Vityazev @ 2021-08-05 19:54 UTC (permalink / raw)
  To: Eric Abrahamsen; +Cc: Joseph Mingrone, ding

On 2021-08-05, 10:15 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:

> Eric Abrahamsen <eric@ericabrahamsen.net> writes:
>
>
> [...]
>
>> I'm getting to it! I just need to find a moment to sit down and set up
>> some dummy data and test the various possible configurations. It
>> shouldn't be to hard to unilaterally force prefixed group names where
>> that's needed, and force unprefixed names elsewhere, I just need to
>> find the time. I'd like to get this fixed once and for all.
>
> Okay, I think I've finally got it. The incoming group names are indeed
> prefixed or not depending on whether the server is primary or not
> (arguably this should be fixed to always unprefixed in nnselect, but
> I'll worry about that later), so this patch enforces unprefixed group
> names within the function body, and prefixed group names in the return
> value. Please give it a shot!
>
> Eric
>
>

The patch was applied to the master branch and it doesn't work for me.
In my case because of line 1388 with the code:

#+begin_src elisp 
(setq group (gnus-group-full-name group server))
#+end_src

code on 1395 line:

#+begin_src elisp
(nnmaildir-base-name-to-article-number
		   (substring article 0 (string-match ":" article))
		   group (string-remove-prefix "nnmaildir:" server))
#+end_src

always return nil, because the group on line 1397 always gets something
like "nnmaildir+gmail:Inbox" and not "Inbox".

And in the code on 1400 line:

#+begin_src elisp
(member group groups)
#+end_src

also requires the group similar to "Inbox", so it can only be solved by
deleting line 1388:

#+begin_src elisp
(setq group (gnus-group-full-name group server))
#+end_src

-- 
Best regards,
Alexandr Vityazev


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-08-05 19:54                                   ` Alexandr Vityazev
@ 2021-08-05 20:19                                     ` Joseph Mingrone
  2021-08-05 21:24                                       ` Eric Abrahamsen
  0 siblings, 1 reply; 34+ messages in thread
From: Joseph Mingrone @ 2021-08-05 20:19 UTC (permalink / raw)
  To: Alexandr Vityazev; +Cc: Eric Abrahamsen, ding

On Thu, 2021-08-05 at 19:54, Alexandr Vityazev <avityazev@posteo.org> wrote:

> On 2021-08-05, 10:15 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:

>> Eric Abrahamsen <eric@ericabrahamsen.net> writes:


>> [...]

>>> I'm getting to it! I just need to find a moment to sit down and set up
>>> some dummy data and test the various possible configurations. It
>>> shouldn't be to hard to unilaterally force prefixed group names where
>>> that's needed, and force unprefixed names elsewhere, I just need to
>>> find the time. I'd like to get this fixed once and for all.

>> Okay, I think I've finally got it. The incoming group names are indeed
>> prefixed or not depending on whether the server is primary or not
>> (arguably this should be fixed to always unprefixed in nnselect, but
>> I'll worry about that later), so this patch enforces unprefixed group
>> names within the function body, and prefixed group names in the return
>> value. Please give it a shot!

>> Eric



> The patch was applied to the master branch and it doesn't work for me.
> In my case because of line 1388 with the code:

> #+begin_src elisp 
> (setq group (gnus-group-full-name group server))
> #+end_src


> code on 1395 line:

> #+begin_src elisp
> (nnmaildir-base-name-to-article-number
> 		   (substring article 0 (string-match ":" article))
> 		   group (string-remove-prefix "nnmaildir:" server))
> #+end_src


> always return nil, because the group on line 1397 always gets something
> like "nnmaildir+gmail:Inbox" and not "Inbox".

> And in the code on 1400 line:

> #+begin_src elisp
> (member group groups)
> #+end_src


> also requires the group similar to "Inbox", so it can only be solved by
> deleting line 1388:

> #+begin_src elisp
> (setq group (gnus-group-full-name group server))
> #+end_src

I see the same as you Alexandr, i.e., with the patch applied to a recent
commit from the master branch, still no search results, but after
removing line 1388, results are returned.

Thanks both,

Joe


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-08-05 20:19                                     ` Joseph Mingrone
@ 2021-08-05 21:24                                       ` Eric Abrahamsen
  2021-08-05 21:38                                         ` Joseph Mingrone
  0 siblings, 1 reply; 34+ messages in thread
From: Eric Abrahamsen @ 2021-08-05 21:24 UTC (permalink / raw)
  To: Joseph Mingrone; +Cc: Alexandr Vityazev, ding

[-- Attachment #1: Type: text/plain, Size: 2255 bytes --]

Joseph Mingrone <jrm@ftfl.ca> writes:

> On Thu, 2021-08-05 at 19:54, Alexandr Vityazev <avityazev@posteo.org> wrote:
>
>> On 2021-08-05, 10:15 -0700, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>
>>> Eric Abrahamsen <eric@ericabrahamsen.net> writes:
>
>
>>> [...]
>
>>>> I'm getting to it! I just need to find a moment to sit down and set up
>>>> some dummy data and test the various possible configurations. It
>>>> shouldn't be to hard to unilaterally force prefixed group names where
>>>> that's needed, and force unprefixed names elsewhere, I just need to
>>>> find the time. I'd like to get this fixed once and for all.
>
>>> Okay, I think I've finally got it. The incoming group names are indeed
>>> prefixed or not depending on whether the server is primary or not
>>> (arguably this should be fixed to always unprefixed in nnselect, but
>>> I'll worry about that later), so this patch enforces unprefixed group
>>> names within the function body, and prefixed group names in the return
>>> value. Please give it a shot!
>
>>> Eric
>
>
>
>> The patch was applied to the master branch and it doesn't work for me.
>> In my case because of line 1388 with the code:
>
>> #+begin_src elisp 
>> (setq group (gnus-group-full-name group server))
>> #+end_src
>
>
>> code on 1395 line:
>
>> #+begin_src elisp
>> (nnmaildir-base-name-to-article-number
>> 		   (substring article 0 (string-match ":" article))
>> 		   group (string-remove-prefix "nnmaildir:" server))
>> #+end_src
>
>
>> always return nil, because the group on line 1397 always gets something
>> like "nnmaildir+gmail:Inbox" and not "Inbox".
>
>> And in the code on 1400 line:
>
>> #+begin_src elisp
>> (member group groups)
>> #+end_src
>
>
>> also requires the group similar to "Inbox", so it can only be solved by
>> deleting line 1388:
>
>> #+begin_src elisp
>> (setq group (gnus-group-full-name group server))
>> #+end_src
>
> I see the same as you Alexandr, i.e., with the patch applied to a recent
> commit from the master branch, still no search results, but after
> removing line 1388, results are returned.

Grr, sorry, I sent the wrong diff -- the original call to
`gnus-group-full-name' was still in there. I promise I'm sending the
correct, freshly-tested diff this time!


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: gnus-search-groups.diff --]
[-- Type: text/x-patch, Size: 1501 bytes --]

diff --git a/lisp/gnus/gnus-search.el b/lisp/gnus/gnus-search.el
index 53af2f6fe6..8182630dfe 100644
--- a/lisp/gnus/gnus-search.el
+++ b/lisp/gnus/gnus-search.el
@@ -1358,6 +1358,7 @@ gnus-search-indexed-parse-output
 						server query &optional groups)
   (let ((prefix (or (slot-value engine 'remove-prefix)
                     ""))
+        (groups (mapcar #'gnus-group-short-name groups))
 	artlist article group)
     (goto-char (point-min))
     ;; Prep prefix, we want to at least be removing the root
@@ -1384,7 +1385,6 @@ gnus-search-indexed-parse-output
                    nil t)
 	          nil t)
 	         nil t))
-          (setq group (gnus-group-full-name group server))
           (setq article (file-name-nondirectory f-name)
                 article
                 ;; TODO: Provide a cleaner way of producing final
@@ -1404,10 +1404,12 @@ gnus-search-indexed-parse-output
       (setq artlist (gnus-search-grep-search engine artlist grep-reg)))
     ;; Munge into the list of vectors expected by nnselect.
     (mapcar (pcase-lambda (`(,_ ,article ,group ,score))
-              (vector group article
-                      (if (numberp score)
-			  score
-		        (string-to-number score))))
+              (vector
+               (gnus-group-full-name group server)
+               article
+               (if (numberp score)
+		   score
+		 (string-to-number score))))
             artlist)))
 
 (cl-defmethod gnus-search-indexed-extract ((_engine gnus-search-indexed))

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-08-05 21:24                                       ` Eric Abrahamsen
@ 2021-08-05 21:38                                         ` Joseph Mingrone
  2021-08-07 16:07                                           ` Eric Abrahamsen
  0 siblings, 1 reply; 34+ messages in thread
From: Joseph Mingrone @ 2021-08-05 21:38 UTC (permalink / raw)
  To: Eric Abrahamsen; +Cc: Alexandr Vityazev, ding

On Thu, 2021-08-05 at 14:24, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
> Grr, sorry, I sent the wrong diff -- the original call to
> `gnus-group-full-name' was still in there. I promise I'm sending the
> correct, freshly-tested diff this time!

Looks good.

Joe


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-08-05 21:38                                         ` Joseph Mingrone
@ 2021-08-07 16:07                                           ` Eric Abrahamsen
  2021-08-10  7:32                                             ` Daniel Jensen
  0 siblings, 1 reply; 34+ messages in thread
From: Eric Abrahamsen @ 2021-08-07 16:07 UTC (permalink / raw)
  To: Joseph Mingrone; +Cc: Alexandr Vityazev, ding

Joseph Mingrone <jrm@ftfl.ca> writes:

> On Thu, 2021-08-05 at 14:24, Eric Abrahamsen <eric@ericabrahamsen.net> wrote:
>> Grr, sorry, I sent the wrong diff -- the original call to
>> `gnus-group-full-name' was still in there. I promise I'm sending the
>> correct, freshly-tested diff this time!
>
> Looks good.

In it goes, thanks for testing!


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-08-07 16:07                                           ` Eric Abrahamsen
@ 2021-08-10  7:32                                             ` Daniel Jensen
  2021-08-13 16:50                                               ` Eric Abrahamsen
  0 siblings, 1 reply; 34+ messages in thread
From: Daniel Jensen @ 2021-08-10  7:32 UTC (permalink / raw)
  To: Eric Abrahamsen, ding

Eric,

I tried your latest gnus-search code, but it didn't work for me. I
noticed that in gnus-search-indexed-parse-output the `group' variable
was wrong, resulting in nnmaildir-base-name-to-article-number returning
nil. This is my setup, having multiple secondary nnmaildir servers:

(setq gnus-secondary-select-methods
      '((nnmaildir "One" (directory "~/Mail/one/"))
        (nnmaildir "Two" (directory "~/Mail/two/"))
        ...)
      gnus-search-default-engines
      '((nnmaildir . notmuch)))

I found that I could make it work by giving `prefix' its value from the
servers at the start of the function. Like so:

(let ((prefix (or (nnmaildir--srv-dir
                    (alist-get (string-remove-prefix "nnmaildir:" server)
                                nnmaildir--servers nil nil #'equal))
                  ""))
       ...))

This uses the correct prefix "~/Mail/one/" and so on, not just "~/Mail/"
from the engine. I have no idea what a general solution would look like.
It could be my setup that is wrong, I don't know.


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-08-10  7:32                                             ` Daniel Jensen
@ 2021-08-13 16:50                                               ` Eric Abrahamsen
  2021-08-13 19:30                                                 ` Daniel Jensen
  0 siblings, 1 reply; 34+ messages in thread
From: Eric Abrahamsen @ 2021-08-13 16:50 UTC (permalink / raw)
  To: Daniel Jensen; +Cc: ding

Daniel Jensen <daniel@bigwalter.net> writes:

> Eric,
>
> I tried your latest gnus-search code, but it didn't work for me. I
> noticed that in gnus-search-indexed-parse-output the `group' variable
> was wrong, resulting in nnmaildir-base-name-to-article-number returning
> nil. This is my setup, having multiple secondary nnmaildir servers:
>
> (setq gnus-secondary-select-methods
>       '((nnmaildir "One" (directory "~/Mail/one/"))
>         (nnmaildir "Two" (directory "~/Mail/two/"))
>         ...)
>       gnus-search-default-engines
>       '((nnmaildir . notmuch)))
>
> I found that I could make it work by giving `prefix' its value from the
> servers at the start of the function. Like so:
>
> (let ((prefix (or (nnmaildir--srv-dir
>                     (alist-get (string-remove-prefix "nnmaildir:" server)
>                                 nnmaildir--servers nil nil #'equal))
>                   ""))
>        ...))
>
> This uses the correct prefix "~/Mail/one/" and so on, not just "~/Mail/"
> from the engine. I have no idea what a general solution would look like.
> It could be my setup that is wrong, I don't know.

Sorry for the slow response, I was camping...

If you're only using one notmuch index, it's enough to set the global
options like `gnus-search-notmuch-remove-prefix'. If you've got two
nnmaildir servers each with its own notmuch index, you'll have to
specific the different prefixes within the engine configuration:

((nnmaildir "One"
   (directory "~/Mail/one/"
     (gnus-search-engine gnus-search-notmuch
       (remove-prefix "~/Mail/one/"))))
 (nnmaildir "Two"
   (directory "~/Mail/two/"
     (gnus-search-engine gnus-search-notmuch
       (remove-prefix "~/Mail/two/")))))

Note how the sexps are nested, it's important that the engine-related
variables go within the engine configuration sexp.

I realize that 'remove-prefix is redundant with the nnmaildir 'directory
setting here, and hope to eventually be able to get rid of that.

Hope that helps,
Eric


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-08-13 16:50                                               ` Eric Abrahamsen
@ 2021-08-13 19:30                                                 ` Daniel Jensen
  2021-08-13 19:37                                                   ` Eric Abrahamsen
  0 siblings, 1 reply; 34+ messages in thread
From: Daniel Jensen @ 2021-08-13 19:30 UTC (permalink / raw)
  To: Eric Abrahamsen; +Cc: ding

On 2021-08-13 at 09:50, Eric Abrahamsen wrote:

> Sorry for the slow response, I was camping...

No worries!

> If you're only using one notmuch index, it's enough to set the global
> options like `gnus-search-notmuch-remove-prefix'. If you've got two
> nnmaildir servers each with its own notmuch index, you'll have to
> specific the different prefixes within the engine configuration:
>
> ((nnmaildir "One"
>    (directory "~/Mail/one/"
>      (gnus-search-engine gnus-search-notmuch
>        (remove-prefix "~/Mail/one/"))))
>  (nnmaildir "Two"
>    (directory "~/Mail/two/"
>      (gnus-search-engine gnus-search-notmuch
>        (remove-prefix "~/Mail/two/")))))
>
> Note how the sexps are nested, it's important that the engine-related
> variables go within the engine configuration sexp.

I had tried something like this before, but could never get it right.
Actually your example above needed a small change:

((nnmaildir "One"
   (directory "~/Mail/one/")
   (gnus-search-engine gnus-search-notmuch
     (remove-prefix "~/Mail/one/")))
 (nnmaildir "Two"
   (directory "~/Mail/two/")
   (gnus-search-engine gnus-search-notmuch
     (remove-prefix "~/Mail/two/"))))

Notice the gnus-search-engine sexps had to move up for it to work. But
now all is well. Thank you very much!


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-08-13 19:30                                                 ` Daniel Jensen
@ 2021-08-13 19:37                                                   ` Eric Abrahamsen
  0 siblings, 0 replies; 34+ messages in thread
From: Eric Abrahamsen @ 2021-08-13 19:37 UTC (permalink / raw)
  To: Daniel Jensen; +Cc: ding

Daniel Jensen <daniel@bigwalter.net> writes:

> On 2021-08-13 at 09:50, Eric Abrahamsen wrote:
>
>> Sorry for the slow response, I was camping...
>
> No worries!
>
>> If you're only using one notmuch index, it's enough to set the global
>> options like `gnus-search-notmuch-remove-prefix'. If you've got two
>> nnmaildir servers each with its own notmuch index, you'll have to
>> specific the different prefixes within the engine configuration:
>>
>> ((nnmaildir "One"
>>    (directory "~/Mail/one/"
>>      (gnus-search-engine gnus-search-notmuch
>>        (remove-prefix "~/Mail/one/"))))
>>  (nnmaildir "Two"
>>    (directory "~/Mail/two/"
>>      (gnus-search-engine gnus-search-notmuch
>>        (remove-prefix "~/Mail/two/")))))
>>
>> Note how the sexps are nested, it's important that the engine-related
>> variables go within the engine configuration sexp.
>
> I had tried something like this before, but could never get it right.
> Actually your example above needed a small change:
>
> ((nnmaildir "One"
>    (directory "~/Mail/one/")
>    (gnus-search-engine gnus-search-notmuch
>      (remove-prefix "~/Mail/one/")))
>  (nnmaildir "Two"
>    (directory "~/Mail/two/")
>    (gnus-search-engine gnus-search-notmuch
>      (remove-prefix "~/Mail/two/"))))
>
> Notice the gnus-search-engine sexps had to move up for it to work. But
> now all is well. Thank you very much!

"Now watch carefully as I do this incorrectly..."

It really is easy to screw up! Glad everything's working now.


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: Help testing gnus-search fixups
  2021-09-11 13:29 Sergey Makarov
@ 2021-09-11 15:04 ` Eric Abrahamsen
  0 siblings, 0 replies; 34+ messages in thread
From: Eric Abrahamsen @ 2021-09-11 15:04 UTC (permalink / raw)
  To: Sergey Makarov; +Cc: ding

Sergey Makarov <setser200018@gmail.com> writes:

> Hello!
>
> I have a problem with gnus with notmuch search. When I open Gnus, I can
> see my mail, but when I try to press "G G" and search for example mail
> from Bandcamp with query "Bandcamp" I get a message "Group
> nnselect:nnselect-<some number>.fsf" contains no messages. I tried to
> repeat notmuch query, which Gnus sends, it renders some results, but
> they seem not to be parsed correctly. Maybe the reason is that search
> results contain encoded unicode characters so they cannot be parsed?

Hi there,

Can you run the equivalent notmuch search in a terminal, and paste the
output here? I'll see if the parsing thing is choking on something in
particular.

Thanks,
Eric


^ permalink raw reply	[flat|nested] 34+ messages in thread

* Help testing gnus-search fixups
@ 2021-09-11 13:29 Sergey Makarov
  2021-09-11 15:04 ` Eric Abrahamsen
  0 siblings, 1 reply; 34+ messages in thread
From: Sergey Makarov @ 2021-09-11 13:29 UTC (permalink / raw)
  To: ding

[-- Attachment #1: Type: text/plain, Size: 531 bytes --]


Hello!

I have a problem with gnus with notmuch search. When I open Gnus, I can
see my mail, but when I try to press "G G" and search for example mail
from Bandcamp with query "Bandcamp" I get a message "Group
nnselect:nnselect-<some number>.fsf" contains no messages. I tried to
repeat notmuch query, which Gnus sends, it renders some results, but
they seem not to be parsed correctly. Maybe the reason is that search
results contain encoded unicode characters so they cannot be parsed?

Best wishes,
Makarov Sergey

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 865 bytes --]

^ permalink raw reply	[flat|nested] 34+ messages in thread

end of thread, other threads:[~2021-09-11 15:04 UTC | newest]

Thread overview: 34+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-06-28 16:42 Help testing gnus-search fixups Eric Abrahamsen
2021-06-28 18:00 ` Adam Sjøgren
2021-06-28 23:15   ` Eric Abrahamsen
2021-06-29  6:04     ` Thomas Alexander Gerds
2021-06-29  9:30     ` Adam Sjøgren
2021-06-29  9:55       ` Adam Sjøgren
2021-06-29 11:03         ` Thomas Alexander Gerds
2021-06-29 11:13           ` Adam Sjøgren
2021-06-29 11:25             ` Adam Sjøgren
2021-06-30 16:58               ` Eric Abrahamsen
     [not found]                 ` <87bl6tuqcb.fsf@posteo.org>
2021-07-23 15:18                   ` Eric Abrahamsen
     [not found]                     ` <87bl6sual8.fsf@posteo.org>
2021-07-24 22:16                       ` Eric Abrahamsen
2021-07-25 11:17                         ` Alexandr Vityazev
2021-07-25 22:11                           ` Eric Abrahamsen
2021-07-30 18:17                           ` Joseph Mingrone
2021-07-30 18:22                             ` Joseph Mingrone
2021-07-30 18:47                               ` Eric Abrahamsen
2021-07-30 19:29                                 ` Joseph Mingrone
2021-07-30 20:23                                   ` Eric Abrahamsen
2021-07-30 19:34                             ` Alexandr Vityazev
2021-07-30 20:25                               ` Eric Abrahamsen
2021-08-05 17:15                                 ` Eric Abrahamsen
2021-08-05 19:54                                   ` Alexandr Vityazev
2021-08-05 20:19                                     ` Joseph Mingrone
2021-08-05 21:24                                       ` Eric Abrahamsen
2021-08-05 21:38                                         ` Joseph Mingrone
2021-08-07 16:07                                           ` Eric Abrahamsen
2021-08-10  7:32                                             ` Daniel Jensen
2021-08-13 16:50                                               ` Eric Abrahamsen
2021-08-13 19:30                                                 ` Daniel Jensen
2021-08-13 19:37                                                   ` Eric Abrahamsen
2021-06-29 11:27 ` Eric S Fraga
2021-09-11 13:29 Sergey Makarov
2021-09-11 15:04 ` Eric Abrahamsen

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).