Re: Re: a bug in bindtextdomain() and strip '.UTF-8'

[He X <xw897002528@gmail.com>]

[-- Attachment #1.1: Type: text/plain, Size: 4037 bytes --]

> As discussed on irc, .charset suffixes should be dropped before the
loop even begins (never used in pathnames), and they occur before the
@mod, not after it, so the logic for dropping them is different.

1. drop .charset: Sorry for proposing it again, i forget this case after
around three weeks, as i said before, vim will generate three different .mo
files with different charset -> zh_CN.UTF-8.po, zh_CN.cp936.po, zh_CN.po.
In that case, dropping is to generate a lots of junk.

I now found it's not a bug of msgfmt. That is charset is converted by:
iconv -f UTF-8 -t cp936 zh_CN.UTF-8.po | sed -e
's/charset=utf-8/charset=gbk/ > ... So that means, charset and pathname is
decided by softwares, msgfmt does not do charset converting at all, just a
format-translator. (btw, iconv.c is from alpine)

How do we deal with the case? Patch the special software(since these
special cases are rare, maybe other editors like emacs have similar
actions, general softwares only prepare UTF-8 files)?
Or put the drop back in the loop? I prefer the latter, it seems more
elegant and correct than ugly patches, my idea is: ```
tt_TT.XXX@mod
tt_TT.XXX
tt.XXX@mod
tt.XXX
tt_TT@mod
tt_TT
tt@mod
tt
```
I have made it in a similar way as @mod in the attached. Here's the output
from strace:
```
15:14:32.235463 open("./es_MX.utf8@euro/LC_MESSAGES/hellogt.mo",
O_RDONLY|O_NONBLOCK|O_CLOEXEC) = -1 ENOENT (No such file or directory)
15:14:32.235495 open("./es_MX.utf8/LC_MESSAGES/hellogt.mo",
O_RDONLY|O_NONBLOCK|O_CLOEXEC) = -1 ENOENT (No such file or directory)
15:14:32.235560 open("./es.utf8@euro/LC_MESSAGES/hellogt.mo",
O_RDONLY|O_NONBLOCK|O_CLOEXEC) = -1 ENOENT (No such file or directory)
15:14:32.235585 open("./es.utf8/LC_MESSAGES/hellogt.mo",
O_RDONLY|O_NONBLOCK|O_CLOEXEC) = -1 ENOENT (No such file or directory)
15:14:32.235621 open("./es_MX@euro/LC_MESSAGES/hellogt.mo",
O_RDONLY|O_NONBLOCK|O_CLOEXEC) = -1 ENOENT (No such file or directory)
15:14:32.235656 open("./es_MX/LC_MESSAGES/hellogt.mo",
O_RDONLY|O_NONBLOCK|O_CLOEXEC) = -1 ENOENT (No such file or directory)
15:14:32.235683 open("./es@euro/LC_MESSAGES/hellogt.mo",
O_RDONLY|O_NONBLOCK|O_CLOEXEC) = -1 ENOENT (No such file or directory)
15:14:32.235706 open("./es/LC_MESSAGES/hellogt.mo",
O_RDONLY|O_NONBLOCK|O_CLOEXEC) = 3
```
Other possible LANG is tested, too.

> One bug in my version: loclen+1 should be loclen+modlen+1. Without
changing that, the pathname gets truncated if @mod is used. Otherwise
it seems to work.
2. Yes, it is, made it in the attached.

> then we should probably increase this limit a bit, perhaps to 24 or
so.
3. the longest line is `tt_TT.iso885915@euro`, 21. 24 seems good for the
current standard. Made it in the attached.

2017-03-17 19:37 GMT+00:00 Rich Felker <dalias@libc.org>:

> On Fri, Mar 17, 2017 at 03:27:49PM -0400, Rich Felker wrote:
> > +             /* Logically split @mod suffix from locale name. */
> > +             modname = memchr(locname, '@', loclen);
> > +             if (!modname) modname = locname + loclen;
> > +             alt_modlen = modlen = loclen - (modname-locname);
> > +             loclen = modname-locname;
> > +
> > +             /* Drop .charset identifier; it is not used. */
> > +             const char *csp = memchr(locname, '.', loclen);
> > +             if (csp) loclen = csp-locname;
> > +
> > +             char name[dirlen+1 + loclen+1 + catlen+1 + domlen+3 + 1];
> > +             const void *map;
>
> One bug in my version: loclen+1 should be loclen+modlen+1. Without
> changing that, the pathname gets truncated if @mod is used. Otherwise
> it seems to work.
>
> One separate but related problem I noticed while testing is that
> musl's locale name length limit of 16 is insufficient for something
> like "en_GB.UTF-8@euro". If we want to allow users to have an explicit
> ".UTF-8" in the locale name (which, as we discussed a while back,
> users may want to do if they plan to ssh into glibc-based systems)
> then we should probably increase this limit a bit, perhaps to 24 or
> so.
>
> Rich
>

[-- Attachment #1.2: Type: text/html, Size: 6082 bytes --]

[-- Attachment #2: locale.diff --]
[-- Type: text/plain, Size: 4100 bytes --]

diff --git a/src/locale/dcngettext.c b/src/locale/dcngettext.c
index b68e24b..abaa414 100644
--- a/src/locale/dcngettext.c
+++ b/src/locale/dcngettext.c
@@ -100,7 +100,9 @@ struct msgcat {
 	size_t map_size;
 	void *volatile plural_rule;
 	volatile int nplurals;
-	char name[];
+	struct binding *binding;
+	const struct __locale_map *lm;
+	int cat;
 };

 static char *dummy_gettextdomain()
@@ -120,8 +122,8 @@ char *dcngettext(const char *domainname, const char *msgid1, const char *msgid2,
 	struct msgcat *p;
 	struct __locale_struct *loc = CURRENT_LOCALE;
 	const struct __locale_map *lm;
-	const char *dirname, *locname, *catname;
-	size_t dirlen, loclen, catlen, domlen;
+	size_t domlen;
+	struct binding *q;

 	if ((unsigned)category >= LC_ALL) goto notrans;

@@ -130,55 +132,83 @@ char *dcngettext(const char *domainname, const char *msgid1, const char *msgid2,
 	domlen = strnlen(domainname, NAME_MAX+1);
 	if (domlen > NAME_MAX) goto notrans;

-	dirname = gettextdir(domainname, &dirlen);
-	if (!dirname) goto notrans;
+	for (q=bindings; q; q=q->next)
+		if (!strcmp(q->domainname, domainname) && q->active)
+			break;
+	if (!q) goto notrans;

 	lm = loc->cat[category];
 	if (!lm) {
 notrans:
 		return (char *) ((n == 1) ? msgid1 : msgid2);
 	}
-	locname = lm->name;
-
-	catname = catnames[category];
-	catlen = catlens[category];
-	loclen = strlen(locname);
-
-	size_t namelen = dirlen+1 + loclen+1 + catlen+1 + domlen+3;
-	char name[namelen+1], *s = name;
-
-	memcpy(s, dirname, dirlen);
-	s[dirlen] = '/';
-	s += dirlen + 1;
-	memcpy(s, locname, loclen);
-	s[loclen] = '/';
-	s += loclen + 1;
-	memcpy(s, catname, catlen);
-	s[catlen] = '/';
-	s += catlen + 1;
-	memcpy(s, domainname, domlen);
-	s[domlen] = '.';
-	s[domlen+1] = 'm';
-	s[domlen+2] = 'o';
-	s[domlen+3] = 0;

 	for (p=cats; p; p=p->next)
-		if (!strcmp(p->name, name))
+		if (p->binding == q && p->lm == lm && p->cat == category)
 			break;

 	if (!p) {
+		const char *dirname, *locname, *catname, *modname, *csname, *locp;
+		size_t dirlen, loclen, catlen, modlen, alt_modlen, cslen, alt_charset;
 		void *old_cats;
 		size_t map_size;
-		const void *map = __map_file(name, &map_size);
+
+		dirname = q->dirname;
+		locname = lm->name;
+		catname = catnames[category];
+
+		dirlen = q->dirlen;
+		loclen = strlen(locname);
+		catlen = catlens[category];
+
+		/* Logically split @mod suffix from locale name. */
+		modname = memchr(locname, '@', loclen);
+		if (!modname) modname = locname + loclen;
+		alt_modlen = modlen = loclen - (modname-locname);
+		loclen -= modlen;
+
+		/* Split .charset suffix as @mod. */
+		csname = memchr(locname, '.', loclen);
+		if (!csname) csname = locname + loclen;
+		alt_charset = cslen = loclen - (csname-locname);
+		loclen -= cslen;
+
+		char name[dirlen+1 + loclen+modlen+cslen+1 + catlen+1 + domlen+3 + 1];
+		const void *map;
+
+		for (;;) {
+			snprintf(name, sizeof name, "%s/%.*s%.*s%.*s/%s/%s.mo\0",
+				dirname, (int)loclen, locname,
+                               (int)alt_charset, csname,
+				(int)alt_modlen, modname, catname, domainname);
+			if (map = __map_file(name, &map_size)) break;
+
+			/* Try dropping @mod, _YY, then both. */
+			if (alt_modlen) {
+				alt_modlen = 0;
+			} else if ((locp = memchr(locname, '_', loclen))) {
+				loclen = locp-locname;
+				alt_modlen = modlen;
+			} else if (alt_charset) {
+				alt_charset = 0;
+				alt_modlen = modlen;
+				loclen = strlen(locname) - modlen - cslen;
+			} else {
+				break;
+			}
+		}
 		if (!map) goto notrans;
-		p = calloc(sizeof *p + namelen + 1, 1);
+
+		p = calloc(sizeof *p, 1);
 		if (!p) {
 			__munmap((void *)map, map_size);
 			goto notrans;
 		}
+		p->cat = category;
+		p->binding = q;
+		p->lm = lm;
 		p->map = map;
 		p->map_size = map_size;
-		memcpy(p->name, name, namelen+1);
 		do {
 			old_cats = cats;
 			p->next = old_cats;
--- musl-1.1.16/src/internal/locale_impl.h
+++ musl-1.1.16/src/internal/locale_impl.h
@@ -6,7 +6,7 @@
 #include "libc.h"
 #include "pthread_impl.h"
 
-#define LOCALE_NAME_MAX 15
+#define LOCALE_NAME_MAX 23
 
 struct __locale_map {
 	const void *map;