Emacs unicode merge changes to Gnus

Emacs unicode merge changes to Gnus

[Miles Bader]

I am merging the Emacs trunk changes to Gnus, and as the Emacs unicode
branch was merged to the trunk, some rather old changes from that branch
have shown up.

I've appended a patch file below; let me know if you see anything
suspicious.

[I eliminated some changes to contrib/ (unicode branch changes to
sendmail.el, deletion of ucs-tables.el), because as I understand it
those files are intended for people with old versions of emacs, so
changes specific to new versions of Emacs should not be applied to
them.]

Thanks,

-Miles


2008-02-01  Zhang Wei  <id.brep@gmail.com>

	* rfc2047.el (rfc2047-charset-encoding-alist): Add gbk and GB18030.

	* mm-util.el (mm-mime-mule-charset-alist): Add gbk and GB18030.

2008-02-01  Miles Bader  <miles@gnu.org>

	* mml.el (mml-parse-1): Remove apparently vestigial use of
	`mm-hack-charsets'.
	* mm-bodies.el (mm-encode-body): Likewise.

2008-02-01  Kenichi Handa  <handa@m17n.org>

	* rfc2104.el (rfc2104-hexstring-to-byte-list): Renamed from
	rfc2104-hexstring-to-bitstring and changed to return a byte list.
	(rfc2104-hash): Convert the result of concat to unibyte string.

2008-02-01  Dave Love  <fx@gnu.org>

	* gnus-start.el (gnus-read-newsrc-el-file): Don't bind
	coding-system-for-read.
	(gnus-gnus-to-quick-newsrc-format): Insert coding cookie.

	* mm-util.el (mm-hack-charsets, mm-iso-8859-15-compatible)
	(mm-iso-8859-x-to-15-table, mm-iso-8859-x-to-15-region): Deleted.
	(mm-find-mime-charset-region): Remove hack-charsets stuff.


M  {arch}/=tagline-rules
M  lisp/ChangeLog
M  lisp/mm-bodies.el
M  lisp/mml.el
M  lisp/mm-util.el
M  lisp/rfc2047.el
M  lisp/rfc2104.el
M  lisp/gnus-start.el

--- orig/lisp/ChangeLog
+++ mod/lisp/ChangeLog
@@ -1,3 +1,31 @@
+2008-02-01  Zhang Wei  <id.brep@gmail.com>
+
+	* rfc2047.el (rfc2047-charset-encoding-alist): Add gbk and GB18030.
+
+	* mm-util.el (mm-mime-mule-charset-alist): Add gbk and GB18030.
+
+2008-02-01  Miles Bader  <miles@gnu.org>
+
+	* mml.el (mml-parse-1): Remove apparently vestigial use of
+	`mm-hack-charsets'.
+	* mm-bodies.el (mm-encode-body): Likewise.
+
+2008-02-01  Kenichi Handa  <handa@m17n.org>
+
+	* rfc2104.el (rfc2104-hexstring-to-byte-list): Renamed from
+	rfc2104-hexstring-to-bitstring and changed to return a byte list.
+	(rfc2104-hash): Convert the result of concat to unibyte string.
+
+2008-02-01  Dave Love  <fx@gnu.org>
+
+	* gnus-start.el (gnus-read-newsrc-el-file): Don't bind
+	coding-system-for-read.
+	(gnus-gnus-to-quick-newsrc-format): Insert coding cookie.
+
+	* mm-util.el (mm-hack-charsets, mm-iso-8859-15-compatible)
+	(mm-iso-8859-x-to-15-table, mm-iso-8859-x-to-15-region): Deleted.
+	(mm-find-mime-charset-region): Remove hack-charsets stuff.
+
 2008-02-03  Reiner Steib  <Reiner.Steib@gmx.de>
 
 	* gnus.el (gnus-group-startup-message): Add `find-image' call before


--- orig/lisp/gnus-start.el
+++ mod/lisp/gnus-start.el
@@ -463,6 +463,8 @@
 
 ;;; Internal variables
 
+;; Fixme: deal with old emacs-mule when mm-universal-coding-system is
+;; utf-8-emacs.
 (defvar gnus-ding-file-coding-system mm-universal-coding-system
   "Coding system for ding file.")
 
@@ -2404,8 +2406,7 @@
       ;; We always, always read the .eld file.
       (gnus-message 5 "Reading %s..." ding-file)
       (let (gnus-newsrc-assoc)
-	(let ((coding-system-for-read gnus-ding-file-coding-system))
-	  (gnus-load ding-file))
+	(gnus-load ding-file)
 	;; Older versions of `gnus-format-specs' are no longer valid
 	;; in Oort Gnus 0.01.
 	(let ((version
@@ -2837,7 +2838,8 @@
 
 (defun gnus-gnus-to-quick-newsrc-format (&optional minimal name &rest specific-variables)
   "Print Gnus variables such as `gnus-newsrc-alist' in Lisp format."
-    (princ ";; -*- emacs-lisp -*-\n")
+    (princ (format ";; -*- mode:emacs-lisp; coding: %s; -*-\n"
+		   gnus-ding-file-coding-system))
     (if name
 	(princ (format ";; %s\n" name))
       (princ ";; Gnus startup file.\n"))


--- orig/lisp/mm-bodies.el
+++ mod/lisp/mm-bodies.el
@@ -104,8 +104,7 @@
 				     (mm-charset-to-coding-system charset))
 	    charset)
 	(goto-char (point-min))
-	(let ((charsets (mm-find-mime-charset-region (point-min) (point-max)
-						     mm-hack-charsets)))
+	(let ((charsets (mm-find-mime-charset-region (point-min) (point-max))))
 	  (cond
 	   ;; No encoding.
 	   ((null charsets)


--- orig/lisp/mm-util.el
+++ mod/lisp/mm-util.el
@@ -492,6 +492,10 @@
     (iso-2022-jp latin-jisx0201 japanese-jisx0208 japanese-jisx0208-1978)
     (euc-kr korean-ksc5601)
     (gb2312 chinese-gb2312)
+    (gbk chinese-gbk)
+    (gb18030 gb18030-2-byte
+	     gb18030-4-byte-bmp gb18030-4-byte-smp
+	     gb18030-4-byte-ext-1 gb18030-4-byte-ext-2)
     (big5 chinese-big5-1 chinese-big5-2)
     (tibetan tibetan)
     (thai-tis620 thai-tis620)
@@ -560,7 +564,7 @@
 	  cs mime mule alist)
       (while css
 	(setq cs (pop css)
-	      mime (or (coding-system-get cs :mime-charset) ; Emacs 23 (unicode)
+	      mime (or (coding-system-get cs :mime-charset); Emacs 23 (unicode)
 		       (coding-system-get cs 'mime-charset)))
 	(when (and mime
 		   (not (eq t (setq mule
@@ -569,36 +573,6 @@
 	  (push (cons mime (delq 'ascii mule)) alist)))
       (setq mm-mime-mule-charset-alist (nreverse alist)))))
 
-(defvar mm-hack-charsets '(iso-8859-15 iso-2022-jp-2)
-  "A list of special charsets.
-Valid elements include:
-`iso-8859-15'    convert ISO-8859-1, -9 to ISO-8859-15 if ISO-8859-15 exists.
-`iso-2022-jp-2'  convert ISO-2022-jp to ISO-2022-jp-2 if ISO-2022-jp-2 exists."
-)
-
-(defvar mm-iso-8859-15-compatible
-  '((iso-8859-1 "\xA4\xA6\xA8\xB4\xB8\xBC\xBD\xBE")
-    (iso-8859-9 "\xA4\xA6\xA8\xB4\xB8\xBC\xBD\xBE\xD0\xDD\xDE\xF0\xFD\xFE"))
-  "ISO-8859-15 exchangeable coding systems and inconvertible characters.")
-
-(defvar mm-iso-8859-x-to-15-table
-  (and (fboundp 'coding-system-p)
-       (mm-coding-system-p 'iso-8859-15)
-       (mapcar
-	(lambda (cs)
-	  (if (mm-coding-system-p (car cs))
-	      (let ((c (string-to-char
-			(decode-coding-string "\341" (car cs)))))
-		(cons (char-charset c)
-		      (cons
-		       (- (string-to-char
-			   (decode-coding-string "\341" 'iso-8859-15)) c)
-		       (string-to-list (decode-coding-string (car (cdr cs))
-							     (car cs))))))
-	    '(gnus-charset 0)))
-	mm-iso-8859-15-compatible))
-  "A table of the difference character between ISO-8859-X and ISO-8859-15.")
-
 (defcustom mm-coding-system-priorities
   (if (boundp 'current-language-environment)
       (let ((lang (symbol-value 'current-language-environment)))
@@ -852,27 +826,6 @@
 	  default-enable-multibyte-characters
 	t)))
 
-(defun mm-iso-8859-x-to-15-region (&optional b e)
-  (if (fboundp 'char-charset)
-      (let (charset item c inconvertible)
-	(save-restriction
-	  (if e (narrow-to-region b e))
-	  (goto-char (point-min))
-	  (skip-chars-forward "\0-\177")
-	  (while (not (eobp))
-	    (cond
-	     ((not (setq item (assq (char-charset (setq c (char-after)))
-				    mm-iso-8859-x-to-15-table)))
-	      (forward-char))
-	     ((memq c (cdr (cdr item)))
-	      (setq inconvertible t)
-	      (forward-char))
-	     (t
-	      (insert-before-markers (prog1 (+ c (car (cdr item)))
-				       (delete-char 1)))))
-	    (skip-chars-forward "\0-\177")))
-	(not inconvertible))))
-
 (defun mm-sort-coding-systems-predicate (a b)
   (let ((priorities
 	 (mapcar (lambda (cs)
@@ -1012,32 +965,14 @@
 	;; Otherwise, we'll get nil, and the next setq will get invoked.
 	(setq charsets (mm-xemacs-find-mime-charset b e))
 
+	;; Fixme: won't work for unibyte Emacs 23:
+
 	;; We're not multibyte, or a single coding system won't cover it.
 	(setq charsets
 	      (mm-delete-duplicates
 	       (mapcar 'mm-mime-charset
 		       (delq 'ascii
 			     (mm-find-charset-region b e))))))
-    (if (and (> (length charsets) 1)
-	     (memq 'iso-8859-15 charsets)
-	     (memq 'iso-8859-15 hack-charsets)
-	     (save-excursion (mm-iso-8859-x-to-15-region b e)))
-	(dolist (x mm-iso-8859-15-compatible)
-	  (setq charsets (delq (car x) charsets))))
-    (if (and (memq 'iso-2022-jp-2 charsets)
-	     (memq 'iso-2022-jp-2 hack-charsets))
-	(setq charsets (delq 'iso-2022-jp charsets)))
-    ;; Attempt to reduce the number of charsets if utf-8 is available.
-    (if (and (featurep 'xemacs)
-	     (> (length charsets) 1)
-	     (mm-coding-system-p 'utf-8))
-	(let ((mm-coding-system-priorities
-	       (cons 'utf-8 mm-coding-system-priorities)))
-	  (setq charsets
-		(mm-delete-duplicates
-		 (mapcar 'mm-mime-charset
-			 (delq 'ascii
-			       (mm-find-charset-region b e)))))))
     charsets))
 
 (defmacro mm-with-unibyte-buffer (&rest forms)


--- orig/lisp/mml.el
+++ mod/lisp/mml.el
@@ -284,8 +284,7 @@
 			 (list
 			  (intern (downcase (cdr (assq 'charset tag))))))
 			(t
-			 (mm-find-mime-charset-region point (point)
-						      mm-hack-charsets))))
+			 (mm-find-mime-charset-region point (point)))))
 	(when (and (not raw) (memq nil charsets))
 	  (if (or (memq 'unknown-encoding mml-confirmation-set)
 		  (message-options-get 'unknown-encoding)


--- orig/lisp/rfc2047.el
+++ mod/lisp/rfc2047.el
@@ -78,6 +78,8 @@
     (iso-2022-jp . B)
     (iso-2022-kr . B)
     (gb2312 . B)
+    (gbk . B)
+    (gb18030 . B)
     (big5 . B)
     (cn-big5 . B)
     (cn-gb . B)
@@ -344,7 +346,7 @@
   ;; (make-char-table 'syntax-table '(2)) only works in Emacs.
   (let ((table (make-syntax-table)))
     ;; The following is done to work for setting all elements of the table
-    ;; in Emacs 21 and 22 and XEmacs; it appears to be the cleanest way.
+    ;; in Emacs 21-23 and XEmacs; it appears to be the cleanest way.
     ;; Play safe and don't assume the form of the word syntax entry --
     ;; copy it from ?a.
     (if (fboundp 'set-char-table-range)	; Emacs


--- orig/lisp/rfc2104.el
+++ mod/lisp/rfc2104.el
@@ -53,6 +53,7 @@
 ;;; 1999-10-23  included in pgnus
 ;;; 2000-08-15  `rfc2104-hexstring-to-bitstring'
 ;;; 2000-05-12  added sha-1 example, added test case reference
+;;; 2003-11-13  change rfc2104-hexstring-to-bitstring to ...-byte-list
 
 ;;; Code:
 
@@ -88,12 +89,12 @@
 	(rfc2104-hex-to-int (reverse (append str nil))))
     0))
 
-(defun rfc2104-hexstring-to-bitstring (str)
+(defun rfc2104-hexstring-to-byte-list (str)
   (let (out)
     (while (< 0 (length str))
       (push (rfc2104-hex-to-int (substring str -2)) out)
       (setq str (substring str 0 -2)))
-    (concat out)))
+    out))
 
 (defun rfc2104-hash (hash block-length hash-length key text)
   (let* (;; if key is longer than B, reset it to HASH(key)
@@ -110,9 +111,12 @@
     (setq k_ipad (mapcar (lambda (c) (logxor c rfc2104-ipad)) k_ipad))
     (setq k_opad (mapcar (lambda (c) (logxor c rfc2104-opad)) k_opad))
     ;; perform outer hash
-    (funcall hash (concat k_opad (rfc2104-hexstring-to-bitstring
-				  ;; perform inner hash
-				  (funcall hash (concat k_ipad text)))))))
+    (funcall hash
+	     (encode-coding-string
+	      (concat k_opad (rfc2104-hexstring-to-byte-list
+			      ;; perform inner hash
+			      (funcall hash (concat k_ipad text))))
+	      'iso-latin-1))))
 
 (provide 'rfc2104)
 

Re: Emacs unicode merge changes to Gnus

[Reiner Steib]

On Wed, Feb 06 2008, Miles Bader wrote:

> I've appended a patch file below; let me know if you see anything
> suspicious.

See below.

> [I eliminated some changes to contrib/ (unicode branch changes to
> sendmail.el, deletion of ucs-tables.el), because as I understand it
> those files are intended for people with old versions of emacs, so
> changes specific to new versions of Emacs should not be applied to
> them.]

Correct.  These are not for Emacs 22 and up.

> 2008-02-01  Zhang Wei  <id.brep@gmail.com>
>
> 	* rfc2047.el (rfc2047-charset-encoding-alist): Add gbk and GB18030.

This doesn't do any harm, AFAICS.

,----[ <f1> v rfc2047-charset-encoding-alist RET ]
| Documentation:
| Alist of MIME charsets to RFC2047 encodings.
| Valid encodings are nil, `Q' and `B'.  These indicate binary (no) encoding,
| quoted-printable and base64 respectively.
`----

> 	* mm-util.el (mm-mime-mule-charset-alist): Add gbk and GB18030.

Similar here.

> 2008-02-01  Miles Bader  <miles@gnu.org>
>
> 	* mml.el (mml-parse-1): Remove apparently vestigial use of
> 	`mm-hack-charsets'.
> 	* mm-bodies.el (mm-encode-body): Likewise.
[...]
> 2008-02-01  Dave Love  <fx@gnu.org>
[...]
> 	* mm-util.el (mm-hack-charsets, mm-iso-8859-15-compatible)
> 	(mm-iso-8859-x-to-15-table, mm-iso-8859-x-to-15-region): Deleted.
> 	(mm-find-mime-charset-region): Remove hack-charsets stuff.

I think this changes should be reverted.  AFAIK, `mm-hack-charsets' is
used for Emacs 21.[12] (w/o ucs-tables.el) and maybe XEmacs.  Does
anyone see any problem in keeping `mm-hack-charsets' in Emacs 23?

> 2008-02-01  Kenichi Handa  <handa@m17n.org>
>
> 	* rfc2104.el (rfc2104-hexstring-to-byte-list): Renamed from
> 	rfc2104-hexstring-to-bitstring and changed to return a byte list.
> 	(rfc2104-hash): Convert the result of concat to unibyte string.

`rfc2104.el' is only used in mail/smtpmail.el and net/imap.el, AFAICS.
Maybe Simon can comment on this.

> 2008-02-01  Dave Love  <fx@gnu.org>
>
> 	* gnus-start.el (gnus-read-newsrc-el-file): Don't bind
> 	coding-system-for-read.
> 	(gnus-gnus-to-quick-newsrc-format): Insert coding cookie.

--8<---------------cut here---------------start------------->8---
--- gnus-start.el	8 Jan 2008 20:45:12 -0000	1.59
+++ gnus-start.el	1 Feb 2008 16:01:20 -0000	1.60
@@ -463,6 +463,8 @@
 
 ;;; Internal variables
 
+;; Fixme: deal with old emacs-mule when mm-universal-coding-system is
+;; utf-8-emacs.
 (defvar gnus-ding-file-coding-system mm-universal-coding-system
   "Coding system for ding file.")
 
@@ -2404,8 +2406,7 @@
       ;; We always, always read the .eld file.
       (gnus-message 5 "Reading %s..." ding-file)
       (let (gnus-newsrc-assoc)
-	(let ((coding-system-for-read gnus-ding-file-coding-system))
-	  (gnus-load ding-file))
+	(gnus-load ding-file)
 	;; Older versions of `gnus-format-specs' are no longer valid
 	;; in Oort Gnus 0.01.
 	(let ((version
@@ -2837,7 +2838,8 @@
 
 (defun gnus-gnus-to-quick-newsrc-format (&optional minimal name &rest specific-variables)
   "Print Gnus variables such as `gnus-newsrc-alist' in Lisp format."
-    (princ ";; -*- emacs-lisp -*-\n")
+    (princ (format ";; -*- mode:emacs-lisp; coding: %s; -*-\n"
+		   gnus-ding-file-coding-system))
--8<---------------cut here---------------end--------------->8---

This could cause problems for people switching (from Emacs 23) to
Emacs <= 22 (or XEmacs).  AFAICS, Emacs 21 and 22 simply ignore the
cookie if the coding is unknown.  I'd expect problems for people using
e.g. non-ASCII group names.  Any ideas how to fix this problem?

Bye, Reiner.
-- 
       ,,,
      (o o)
---ooO-(_)-Ooo---  |  PGP key available  |  http://rsteib.home.pages.de/

Re: Emacs unicode merge changes to Gnus

[Miles Bader]

> -       (let ((coding-system-for-read gnus-ding-file-coding-system))
> -         (gnus-load ding-file))
> +       (gnus-load ding-file)
...
> -    (princ ";; -*- emacs-lisp -*-\n")
> +    (princ (format ";; -*- mode:emacs-lisp; coding: %s; -*-\n"
> +                  gnus-ding-file-coding-system))
...
> This could cause problems for people switching (from Emacs 23) to
> Emacs <= 22 (or XEmacs).  AFAICS, Emacs 21 and 22 simply ignore the
> cookie if the coding is unknown.  I'd expect problems for people using
> e.g. non-ASCII group names.  Any ideas how to fix this problem?

It seems like this change wouldn't affect that case either way:  if
they switch emacs to an old version, and gnus-ding-file-coding-system
is something unknown, and they actually have some non-ascii chars in
there, they're screwed, whether with the old code or the new code.

There's a case where the old code could work better, if the file is
written with utf-8-emacs (the default now, I guess), and the
switched-to-system happens to use a coding system of utf-8 which is
largely compatible; for Emacs 21, this won't happen (the switcher is
just plain screwed in this case, no matter what, I think), as Emacs 21
doesn't support utf-8.

Now, the new code could also work (more properly, instead of by
accident) in the switching-to-emacs-22 case, if Gnus used a coding:
tag of "utf-8" instead of "utf-8-emacs" when possible.

In any case, I think it definitely should always write a coding: tag,
and clearly using it is the way for the future.  To handle the
switch-to-emacs-22 case, it could try to use "coding: utf-8" except
when that can't encode something.  The only case I know of when
utf-8-emacs works  but utf-8 doesn't, is when you have random binary
bytes embedded in the file, in which case I think the proper thing to
do is say "don't an idiot."

It seems to me that the switch-to-emacs-21 case is going to break no
matter what (unless you only use ascii), so there shouldn't be any
effort expended to fix it.

-Miles

-- 
Do not taunt Happy Fun Ball.

Re: Emacs unicode merge changes to Gnus

[Reiner Steib]

On Wed, Feb 06 2008, Miles Bader wrote:

>> This could cause problems for people switching (from Emacs 23) to
>> Emacs <= 22 (or XEmacs).  AFAICS, Emacs 21 and 22 simply ignore the
>> cookie if the coding is unknown.  I'd expect problems for people using
>> e.g. non-ASCII group names.  

WARNING: I just notice that the drafts articles saved with Emacs 23
are messed up when read with Emacs 22.  So I'd recommend not to switch
between Emacs 22 and 23 for now.

>> Any ideas how to fix this problem?
>
> It seems like this change wouldn't affect that case either way:  if
> they switch emacs to an old version, and gnus-ding-file-coding-system
> is something unknown, and they actually have some non-ascii chars in
> there, they're screwed, whether with the old code or the new code.

Thanks for your comments.  I will come back to it when I have more
time (Friday, I hope) to think about your mail.  For now, just a few
remarks...

> There's a case where the old code could work better, if the file is
> written with utf-8-emacs (the default now, I guess), and the
> switched-to-system happens to use a coding system of utf-8 which is
> largely compatible; for Emacs 21, this won't happen (the switcher is
> just plain screwed in this case, no matter what, I think), as Emacs 21
> doesn't support utf-8.

Emacs 21 has support for utf-8 to some limited extend (for European,
latin languages, its quite sufficient).

> Now, the new code could also work (more properly, instead of by
> accident) in the switching-to-emacs-22 case, if Gnus used a coding:
> tag of "utf-8" instead of "utf-8-emacs" when possible.

This will probably not work well for people switching between Emacs
and XEmacs.

Bye, Reiner.
-- 
       ,,,
      (o o)
---ooO-(_)-Ooo---  |  PGP key available  |  http://rsteib.home.pages.de/

Re: Emacs unicode merge changes to Gnus

[Miles Bader]

Reiner Steib <reinersteib+gmane@imap.cc> writes:
>> Now, the new code could also work (more properly, instead of by
>> accident) in the switching-to-emacs-22 case, if Gnus used a coding:
>> tag of "utf-8" instead of "utf-8-emacs" when possible.
>
> This will probably not work well for people switching between Emacs
> and XEmacs.

Why not?  If XEmacs doesn't support utf-8 at all, then the problem lies
not with the method of applying the decoding (the change being
discussed), but with the choice of coding system used by default.
If XEmacs _does_ support uft-8, then why can't it handle a "utf-8"
coding:  tag (or doesn't XEmacs support coding: tags...)?

-Miles

-- 
Erudition, n. Dust shaken out of a book into an empty skull.

Re: Emacs unicode merge changes to Gnus

[Katsumi Yamaoka]

>>>>> Reiner Steib wrote:

> +;; Fixme: deal with old emacs-mule when mm-universal-coding-system is
> +;; utf-8-emacs.
>  (defvar gnus-ding-file-coding-system mm-universal-coding-system
>    "Coding system for ding file.")

[...]

> -	(let ((coding-system-for-read gnus-ding-file-coding-system))
> -	  (gnus-load ding-file))
> +	(gnus-load ding-file)

[...]

>  (defun gnus-gnus-to-quick-newsrc-format (&optional minimal name &rest specific-variables)
>    "Print Gnus variables such as `gnus-newsrc-alist' in Lisp format."
> -    (princ ";; -*- emacs-lisp -*-\n")
> +    (princ (format ";; -*- mode:emacs-lisp; coding: %s; -*-\n"
> +		   gnus-ding-file-coding-system))

> This could cause problems for people switching (from Emacs 23) to
> Emacs <= 22 (or XEmacs).  AFAICS, Emacs 21 and 22 simply ignore the
> cookie if the coding is unknown.  I'd expect problems for people using
> e.g. non-ASCII group names.  Any ideas how to fix this problem?

I think this would not be a problem at least for non-ASCII group
names.  Because a non-ASCII group name will be expressed with
ASCII characters in the .newsrc.eld file in this way.

(prin1 (encode-coding-string "group.name" 'CODING))

What we should take notice of will be byte-compiled format specs.
Performing `M-x gnus-compile' causes them.

Re: Emacs unicode merge changes to Gnus

[Reiner Steib]

On Thu, Feb 07 2008, Katsumi Yamaoka wrote:

> I think this would not be a problem at least for non-ASCII group
> names.  Because a non-ASCII group name will be expressed with
> ASCII characters in the .newsrc.eld file in this way.

Are Topic names stored in ASCII as well?

Bye, Reiner.
-- 
       ,,,
      (o o)
---ooO-(_)-Ooo---  |  PGP key available  |  http://rsteib.home.pages.de/

Re: Emacs unicode merge changes to Gnus

[Katsumi Yamaoka]

>>>>> Reiner Steib wrote:
> On Thu, Feb 07 2008, Katsumi Yamaoka wrote:

>> I think this would not be a problem at least for non-ASCII group
>> names.  Because a non-ASCII group name will be expressed with
>> ASCII characters in the .newsrc.eld file in this way.

> Are Topic names stored in ASCII as well?

Oops.  I confirmed they aren't cooked like group names.  In
addition, I realized what you pointed to.  That is, even if
`utf-8-emacs', that covers even byte-code, is the best choice
for Emacs 23, it cannot be used as a coding cookie.  Sorry for
the noise.

Maybe, the best for people who use various versions of Emacsen
(like me) will be neither to use non-ASCII topic names nor to
run `gnus-compile'.

Re: Emacs unicode merge changes to Gnus

[Miles Bader]

Katsumi Yamaoka <yamaoka@jpl.org> writes:
> Oops.  I confirmed they aren't cooked like group names.  In
> addition, I realized what you pointed to.  That is, even if
> `utf-8-emacs', that covers even byte-code, is the best choice
> for Emacs 23, it cannot be used as a coding cookie.  Sorry for
> the noise.

What about (1) using the coding: cookie, (2) change the coding system
used to be utf-8 if possible (no binary), and utf-8-emacs otherwise.

That should more or less work correctly except when people have binary
junk, and that should be very rare (right...?).

I guess it won't work for some other cases, like someone with CJK text
who switches from Emacs 23 to Emacs 21, but ...

-Miles

-- 
`To alcohol!  The cause of, and solution to,
 all of life's problems' --Homer J. Simpson

Re: Emacs unicode merge changes to Gnus

[Miles Bader]

Miles Bader <miles.bader@necel.com> writes:
> What about (1) using the coding: cookie, (2) change the coding system
> used to be utf-8 if possible (no binary), and utf-8-emacs otherwise.
>
> That should more or less work correctly except when people have binary
> junk, and that should be very rare (right...?).
>
> I guess it won't work for some other cases, like someone with CJK text
> who switches from Emacs 23 to Emacs 21, but ...

So, I still don't know what to do about this.

[By default, I'll just commit those changes.]

-Miles

-- 
Ich bin ein Virus. Mach' mit und kopiere mich in Deine .signature.

Re: Emacs unicode merge changes to Gnus

[Reiner Steib]

On Fri, Feb 15 2008, Miles Bader wrote:

> So, I still don't know what to do about this.

Sorry, I didn't have time to look into the coding system issue yet.

> [By default, I'll just commit those changes.]

I think all changes beside `gnus-ding-file-coding-system'
(`mm-universal-coding-system') are clear.  I.e. proceed as I suggested
in my first reply <news:v9prv9rb28.fsf@marauder.physik.uni-ulm.de>:

- Merge "Add gbk and GB18030" changes

- Revert `mm-hack-charsets' removal in Emacs CVS (and don't merge it
  to Gnus, of course).

- `rfc2104-hexstring-to-byte-list':  I'm not sure.  Unless someone
  sees a problem now, merge it to Gnus.

> Miles Bader <miles.bader@necel.com> writes:
>> What about (1) using the coding: cookie, (2) change the coding system
>> used to be utf-8 if possible (no binary), and utf-8-emacs otherwise.
>>
>> That should more or less work correctly except when people have binary
>> junk, and that should be very rare (right...?).
>>
>> I guess it won't work for some other cases, like someone with CJK text
>> who switches from Emacs 23 to Emacs 21, but ...

Emacs 23 still has `emacs-mule' as coding system.  Does it allow to
encode everything?  If so, we might use `emacs-mule' instead of
`utf-8-emacs'.  If not, we could recommend setting
`mm-auto-save-coding-system' to `emacs-mule' for those who want to use
Gnus in Emacs 23 and Emacs 21/22.

Bye, Reiner.
-- 
       ,,,
      (o o)
---ooO-(_)-Ooo---  |  PGP key available  |  http://rsteib.home.pages.de/

Re: Emacs unicode merge changes to Gnus

[Miles Bader]

Reiner Steib <reinersteib+gmane@imap.cc> writes:
>>> What about (1) using the coding: cookie, (2) change the coding system
>>> used to be utf-8 if possible (no binary), and utf-8-emacs otherwise.
>>>
>>> That should more or less work correctly except when people have binary
>>> junk, and that should be very rare (right...?).
>>>
>>> I guess it won't work for some other cases, like someone with CJK text
>>> who switches from Emacs 23 to Emacs 21, but ...
>
> Emacs 23 still has `emacs-mule' as coding system.  Does it allow to
> encode everything?  If so, we might use `emacs-mule' instead of
> `utf-8-emacs'.  If not, we could recommend setting
> `mm-auto-save-coding-system' to `emacs-mule' for those who want to use
> Gnus in Emacs 23 and Emacs 21/22.

Emacs 23 still supports emacs-mule (and AFAIK, it encodes "everything"),
though in general I think it's better to move away from it when possible
(utf-8 and utf-8-emacs are readable by other programs in non-binary
cases, and by older versions in "many" cases).

Regardless of what happens, it seems to me that Gnus should be writing
the cookie, which makes future upgrades easier, and makes the file more
self-describing.


So there are really three independent questions (patch hunks :-):

  1) Should gnus write the cookie?   (obviously yes, I think)

  2) Should Gnus stop forcing the coding system when it reads the file?
     (i.e., should Gnus let coding: do its job)

  3) Should Gnus switch to utf-8-emacs as the default coding system for
     writing files?


It seems like the safest thing, if you're worried about "back switchers"
(as well as users of older Emacs versions using CVS Gnus) is to apply
hunks (1) and (2) but not (3); that should at least set things up for an
easier switch to utf-8 in the future when it's deemed more acceptable.

-Miles

-- 
Sabbath, n. A weekly festival having its origin in the fact that God made the
world in six days and was arrested on the seventh.

Re: Emacs unicode merge changes to Gnus

[Reiner Steib]

On Fri, Feb 15 2008, Miles Bader wrote:

> Reiner Steib <reinersteib+gmane@imap.cc> writes:
>> Emacs 23 still has `emacs-mule' as coding system.  Does it allow to
>> encode everything?  If so, we might use `emacs-mule' instead of
>> `utf-8-emacs'.  If not, we could recommend setting
>> `mm-auto-save-coding-system' to `emacs-mule' for those who want to use
>> Gnus in Emacs 23 and Emacs 21/22.
>
> Emacs 23 still supports emacs-mule (and AFAIK, it encodes "everything"),
> though in general I think it's better to move away from it when possible
> (utf-8 and utf-8-emacs are readable by other programs in non-binary
> cases, and by older versions in "many" cases).

For drafts, I can imagine to read the files with other programs, but
for ~/.newsrc.eld?

> So there are really three independent questions (patch hunks :-):
>
>   1) Should gnus write the cookie?   (obviously yes, I think)
>
>   2) Should Gnus stop forcing the coding system when it reads the file?
>      (i.e., should Gnus let coding: do its job)
>
>   3) Should Gnus switch to utf-8-emacs as the default coding system for
>      writing files?
>
> It seems like the safest thing, if you're worried about "back switchers"
> (as well as users of older Emacs versions using CVS Gnus) is to apply
> hunks (1) and (2) but not (3); that should at least set things up for an
> easier switch to utf-8 in the future when it's deemed more acceptable.

Sounds reasonable.  Please apply (1) and (2).

Bye, Reiner.
-- 
       ,,,
      (o o)
---ooO-(_)-Ooo---  |  PGP key available  |  http://rsteib.home.pages.de/

Re: Emacs unicode merge changes to Gnus

[Miles Bader]

On Feb 16, 2008 9:39 AM, Reiner Steib <reinersteib+gmane@imap.cc> wrote:
> >   1) Should gnus write the cookie?   (obviously yes, I think)
> >
> >   2) Should Gnus stop forcing the coding system when it reads the file?
> >      (i.e., should Gnus let coding: do its job)
> >
> >   3) Should Gnus switch to utf-8-emacs as the default coding system for
> >      writing files?
>
> Sounds reasonable.  Please apply (1) and (2).

... However I now notice that Gnus has used "utf-8-emacs" by default
for a _long_ time, if it's supported!!  Here's the ChangeLog entry:

2003-04-24  Dave Love  <fx@gnu.org>

        * mm-util.el (mm-make-temp-file, mm-insert-byte): New.
        (mm-auto-save-coding-system): Consider utf-8-emacs.

That means anybody who's been running the unicode branch (a lot of
people), or the trunk (since the branch was merged) has had their Gnus
files encoding using utf-8-emacs for quite a while...

So I'm not sure what to do, really...  It seems like a step backward
(and perhaps dangerous) to explicitly change that to emacs-mule.

Thanks,

-Miles

-- 
Do not taunt Happy Fun Ball.

Re: Emacs unicode merge changes to Gnus

[Miles Bader]

Also, what about the removal of mm-hack-charsets?  I didn't notice any
replies to your query to the list about that...

Thanks,

-Miles
-- 
Do not taunt Happy Fun Ball.

Re: Emacs unicode merge changes to Gnus

[Reiner Steib]

On Wed, Feb 20 2008, Miles Bader wrote:

> Also, what about the removal of mm-hack-charsets?  I didn't notice any
> replies to your query to the list about that...

I replied via Gmane (and didn't Cc you, sorry), but the message didn't
appear on the list.  Here it is...

--8<---------------cut here---------------start------------->8---
From: Reiner Steib <reinersteib+gmane@imap.cc>
Subject: Re: Emacs unicode merge changes to Gnus
Newsgroups: gmane.emacs.gnus.general
Date: Fri, 15 Feb 2008 18:30:04 +0100
Message-ID: <v98x1m3z6r.fsf@marauder.physik.uni-ulm.de>

On Fri, Feb 15 2008, Miles Bader wrote:

> So, I still don't know what to do about this.

Sorry, I didn't have time to look into the coding system issue yet.

> [By default, I'll just commit those changes.]

I think all changes beside `gnus-ding-file-coding-system'
(`mm-universal-coding-system') are clear.  I.e. proceed as I suggested
in my first reply <news:v9prv9rb28.fsf@marauder.physik.uni-ulm.de>:

- Merge "Add gbk and GB18030" changes

- Revert `mm-hack-charsets' removal in Emacs CVS (and don't merge it
  to Gnus, of course).

- `rfc2104-hexstring-to-byte-list':  I'm not sure.  Unless someone
  sees a problem now, merge it to Gnus.

> Miles Bader <miles.bader@necel.com> writes:
>> What about (1) using the coding: cookie, (2) change the coding system
>> used to be utf-8 if possible (no binary), and utf-8-emacs otherwise.
>>
>> That should more or less work correctly except when people have binary
>> junk, and that should be very rare (right...?).
>>
>> I guess it won't work for some other cases, like someone with CJK text
>> who switches from Emacs 23 to Emacs 21, but ...

Emacs 23 still has `emacs-mule' as coding system.  Does it allow to
encode everything?  If so, we might use `emacs-mule' instead of
`utf-8-emacs'.  If not, we could recommend setting
`mm-auto-save-coding-system' to `emacs-mule' for those who want to use
Gnus in Emacs 23 and Emacs 21/22.
--8<---------------cut here---------------end--------------->8---

Bye, Reiner.
-- 
       ,,,
      (o o)
---ooO-(_)-Ooo---  |  PGP key available  |  http://rsteib.home.pages.de/

Re: Emacs unicode merge changes to Gnus

[Reiner Steib]

On Wed, Feb 20 2008, Miles Bader wrote:

> On Feb 16, 2008 9:39 AM, Reiner Steib <reinersteib+gmane@imap.cc> wrote:
>> >   1) Should gnus write the cookie?   (obviously yes, I think)
>> >
>> >   2) Should Gnus stop forcing the coding system when it reads the file?
>> >      (i.e., should Gnus let coding: do its job)
>> >
>> >   3) Should Gnus switch to utf-8-emacs as the default coding system for
>> >      writing files?
>>
>> Sounds reasonable.  Please apply (1) and (2).
>
> ... However I now notice that Gnus has used "utf-8-emacs" by default
> for a _long_ time, if it's supported!!  Here's the ChangeLog entry:
>
> 2003-04-24  Dave Love  <fx@gnu.org>
>
>         * mm-util.el (mm-make-temp-file, mm-insert-byte): New.
>         (mm-auto-save-coding-system): Consider utf-8-emacs.
>
> That means anybody who's been running the unicode branch (a lot of
> people), or the trunk (since the branch was merged) has had their Gnus
> files encoding using utf-8-emacs for quite a while...
>
> So I'm not sure what to do, really...  It seems like a step backward
> (and perhaps dangerous) to explicitly change that to emacs-mule.

Okay, let's use utf-8-emacs.

We should warn about the incompatibility in GNUS-NEWS (info "(gnus)No
Gnus") and emacs/etc/NEWS.

Bye, Reiner.
-- 
       ,,,
      (o o)
---ooO-(_)-Ooo---  |  PGP key available  |  http://rsteib.home.pages.de/

Re: Emacs unicode merge changes to Gnus

[Reiner Steib]

On Wed, Feb 20 2008, Reiner Steib wrote:

> On Wed, Feb 20 2008, Miles Bader wrote:
>> Also, what about the removal of mm-hack-charsets?  I didn't notice any
>> replies to your query to the list about that...
> [...]
> - Merge "Add gbk and GB18030" changes
> - Revert `mm-hack-charsets' removal in Emacs CVS (and don't merge it
>   to Gnus, of course).
> - `rfc2104-hexstring-to-byte-list':  I'm not sure.  Unless someone
>   sees a problem now, merge it to Gnus.

[...]
>> So I'm not sure what to do, really...  It seems like a step backward
>> (and perhaps dangerous) to explicitly change that to emacs-mule.
>
> Okay, let's use utf-8-emacs.

I.e. let us install this in Gnus' trunk:

	* gnus-start.el (gnus-read-newsrc-el-file): Don't bind
	coding-system-for-read.
	(gnus-gnus-to-quick-newsrc-format): Insert coding cookie.

Miles, do you need more information or do you just need some time?

> We should warn about the incompatibility in GNUS-NEWS (info "(gnus)No
> Gnus") and emacs/etc/NEWS.

[I can do this.]

Bye, Reiner.
-- 
       ,,,
      (o o)
---ooO-(_)-Ooo---  |  PGP key available  |  http://rsteib.home.pages.de/

Re: Emacs unicode merge changes to Gnus

[Miles Bader]

Reiner Steib <reinersteib+gmane@imap.cc> writes:
> Miles, do you need more information or do you just need some time?

I'll do it today.

-Miles

-- 
Genealogy, n. An account of one's descent from an ancestor who did not
particularly care to trace his own.

Re: Emacs unicode merge changes to Gnus

[Reiner Steib]

On Wed, Feb 20 2008, Reiner Steib wrote:

> We should warn about the incompatibility in GNUS-NEWS (info "(gnus)No
> Gnus") and emacs/etc/NEWS.

--8<---------------cut here---------------start------------->8---
@item Incompatibity when switching from Emacs 23 to Emacs 22 
In Emacs 23, Gnus uses Emacs' new internal coding system @code{utf-8-emacs}
for saving articles drafts and @file{~/.newsrc.eld}.  These file may not
be read correctly in Emacs 22 and below.  If you want to Gnus across
different Emacs versions, you may set @code{mm-auto-save-coding-system}
to @code{emacs-mule}.
@c FIXME: Untested.  (Or did anyone test it?)
@c Cf. http://thread.gmane.org/gmane.emacs.gnus.general/66251/focus=66344
--8<---------------cut here---------------end--------------->8---

Bye, Reiner.
-- 
       ,,,
      (o o)
---ooO-(_)-Ooo---  |  PGP key available  |  http://rsteib.home.pages.de/