Src/Zle/zle_word.c iword debacle

Src/Zle/zle_word.c iword debacle

[Clint Adams]

Should WORDCHARS be a wide-char string?

Re: Src/Zle/zle_word.c iword debacle

[Bart Schaefer]

On Mar 20,  8:39pm, Clint Adams wrote:
}
} Should WORDCHARS be a wide-char string?

Some discussion of this is in zsh-workers/20869.

PATCH: zle TODOs etc.

[Peter Stephenson]

Some more minor stuff.  It doesn't fix, er, todos los TODOs.  Some of
them I'd already taken account of, so I've just deleted the comment.

I looked at vichgbuf and it seemed to me that it would work fine left as
it was.  This is because it's used at the level of key binding lookups,
which are still raw bytes.

I think the next things are:

Prompts: probably easiest to convert to wchar_t on entry to zleread and
free at the end.

Word stuff: this will need a whole new way of doing iword().  Probably
that's going to have to be a function call.  The obvious thing to do is
to have it on each call check that either the character iswalnum() or
the multibyte representation of the character is in the string set by
$WORDCHARS (which we could cache as wchar_t in zle).  That's slower, but
probably not prohibitively so.  (The functions supplied with the shell
can in principle do much more, but the underlying pattern matching tests
there don't no about multibyte characters either.)

The suffixlen stuff in zle_misc.c might want looking at, although it
should be OK for Unicode itself.  I will therefore leave it at least for
now.

There may be some other tests like idigit, iident that I missed.  There
are quite a lot in the completion code but I'm leaving that for now
(which is horrible and is going to be buggy for a long time as soon as
we start tweaking it).

Index: Src/Zle/zle.h
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle.h,v
retrieving revision 1.13
diff -u -r1.13 zle.h
--- Src/Zle/zle.h	25 Feb 2005 10:21:02 -0000	1.13
+++ Src/Zle/zle.h	25 Feb 2005 14:33:51 -0000
@@ -59,8 +59,8 @@
 #define ZS_strncpy wcsncpy
 #define ZS_strncmp wcsncmp
 
-#define ZC_icntrl iswcntrl
 #define ZC_iblank iswspace
+#define ZC_icntrl iswcntrl
 /*
  * TODO: doesn't work on arguments with side effects.
  * Also YUK.  Not even sure this is guaranteed to work.
@@ -68,6 +68,7 @@
 #define ZC_iident(x)	(x < 256 && iident((int)x))
 
 #define ZC_tolower towlower
+#define ZC_toupper towupper
 
 #define LASTFULLCHAR	lastchar_wide
 
@@ -93,11 +94,12 @@
 #define ZS_strncpy strncpy
 #define ZS_strncmp strncmp
 
-#define ZC_icntrl icntrl
 #define ZC_iblank iblank
+#define ZC_icntrl icntrl
 #define ZC_iident iident
 
 #define ZC_tolower tulower
+#define ZC_toupper tuupper
 
 #define LASTFULLCHAR	lastchar
 
Index: Src/Zle/zle_hist.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_hist.c,v
retrieving revision 1.21
diff -u -r1.21 zle_hist.c
--- Src/Zle/zle_hist.c	25 Feb 2005 10:21:02 -0000	1.21
+++ Src/Zle/zle_hist.c	25 Feb 2005 14:33:51 -0000
@@ -865,10 +865,6 @@
 #define NORM_PROMPT_POS		8
 #define FIRST_SEARCH_CHAR	(NORM_PROMPT_POS + 14)
 
-/*
- * TODO: use of isearch buffer and strings need fixing for Unicode.
- */
-
 /**/
 static void
 doisearch(char **args, int dir)
Index: Src/Zle/zle_main.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_main.c,v
retrieving revision 1.63
diff -u -r1.63 zle_main.c
--- Src/Zle/zle_main.c	24 Feb 2005 15:32:57 -0000	1.63
+++ Src/Zle/zle_main.c	25 Feb 2005 14:33:52 -0000
@@ -702,8 +702,8 @@
 	ret = STOUC(cc);
     }
     /*
-     * TODO: if vichgbuf is to be characters instead of a multibyte
-     * string the following needs moving to getfullchar().
+     * vichgbuf is raw bytes, not wide characters, so is dealt
+     * with here.
      */
     if (vichgflag) {
 	if (vichgbufptr == vichgbufsz)
Index: Src/Zle/zle_refresh.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_refresh.c,v
retrieving revision 1.20
diff -u -r1.20 zle_refresh.c
--- Src/Zle/zle_refresh.c	24 Feb 2005 15:32:57 -0000	1.20
+++ Src/Zle/zle_refresh.c	25 Feb 2005 14:33:52 -0000
@@ -105,7 +105,8 @@
 # define zwrite(a, b)		zwcwrite(a, b)
 #endif
 
-static int
+/**/
+int
 zwcputc(ZLE_CHAR_T c)
 {
 #ifdef ZLE_UNICODE_SUPPORT
Index: Src/Zle/zle_utils.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_utils.c,v
retrieving revision 1.22
diff -u -r1.22 zle_utils.c
--- Src/Zle/zle_utils.c	25 Feb 2005 10:21:02 -0000	1.22
+++ Src/Zle/zle_utils.c	25 Feb 2005 14:33:52 -0000
@@ -578,14 +578,14 @@
     if (yesno) {
 	if (c == ZWC('\t'))
 	    c = ZWC('y');
-	else if (icntrl(c) || c == ZLEEOF) /* TODO iswcntrl */
+	else if (ZS_icntrl(c) || c == ZLEEOF)
 	    c = ZWC('n');
 	else
-	    c = tulower(c);	/* TODO tulower doesn't handle wint_t */
+	    c = ZS_tolower(c);
     }
     /* echo response and return */
     if (c != ZWC('\n'))
-	putc(c, shout);		/* TODO: convert to multibyte */
+	zwcputc(c);
     return c;
 }
 
Index: Src/Zle/zle_vi.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_vi.c,v
retrieving revision 1.7
diff -u -r1.7 zle_vi.c
--- Src/Zle/zle_vi.c	24 Feb 2005 15:32:57 -0000	1.7
+++ Src/Zle/zle_vi.c	25 Feb 2005 14:33:52 -0000
@@ -51,9 +51,10 @@
 int vichgbufsz, vichgbufptr, vichgflag;
 
 /*
- * TODO: need consistent handling of vichgbuf: ZLE_STRING_T or
- * char *?  Consequently, use of lastchar in this file needs fixing
- * too.
+ * Examination of the code suggests vichgbuf is consistently tied
+ * to raw byte input, so it is left as a character array rather
+ * than turned into wide characters.  In particular, when we replay
+ * it we use ungetbytes().
  */
 /**/
 char *vichgbuf;
@@ -117,11 +118,6 @@
     else
 	cmd = t_undefinedkey;
 
-    /*
-     * TODO: if this was bound to self-insert, we may
-     * be on the first character of a multibyte string
-     * and need to acquire the rest.
-     */
     if (!cmd || cmd == Th(z_sendbreak)) {
 	return ZLEEOF;
     } else if (cmd == Th(z_quotedinsert)) {
@@ -575,9 +571,9 @@
 	/* swap the case of all letters within range */
 	while (zlecs < c2) {
 	    if (islower(zleline[zlecs]))
-		zleline[zlecs] = tuupper(zleline[zlecs]);
+		zleline[zlecs] = ZS_toupper(zleline[zlecs]);
 	    else if (isupper(zleline[zlecs]))
-		zleline[zlecs] = tulower(zleline[zlecs]);
+		zleline[zlecs] = ZS_tolower(zleline[zlecs]);
 	    zlecs++;
 	}
 	/* go back to the first line of the range */
@@ -815,9 +811,9 @@
     eol = findeol();
     while (zlecs < eol && n--) {
 	if (islower(zleline[zlecs]))
-	    zleline[zlecs] = tuupper(zleline[zlecs]);
+	    zleline[zlecs] = ZS_toupper(zleline[zlecs]);
 	else if (isupper(zleline[zlecs]))
-	    zleline[zlecs] = tulower(zleline[zlecs]);
+	    zleline[zlecs] = ZS_tolower(zleline[zlecs]);
 	zlecs++;
     }
     if (zlecs && zlecs == eol)
@@ -862,8 +858,11 @@
 	zmod.flags |= MOD_VIAPP;
     else
 	zmod.flags &= ~MOD_VIAPP;
-    /* TODO tulower, idigit doen't handle wint_t */
-    zmod.vibuf = tulower(ch) + (idigit(ch) ? - ZWC('1') + 26 : -ZWC('a'));
+    zmod.vibuf = ZS_tolower(ch);
+    if (ch >= ZWC('1') && ch <= ZWC('9'))
+	zmod.vibuf += - (int)ZWC('1') + 26;
+    else
+	zmod.vibuf += - (int)ZWC('a');
     zmod.flags |= MOD_VIBUF;
     prefixflag = 1;
     return 0;
Index: Src/Zle/zle_word.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_word.c,v
retrieving revision 1.3
diff -u -r1.3 zle_word.c
--- Src/Zle/zle_word.c	14 Jan 2005 13:05:25 -0000	1.3
+++ Src/Zle/zle_word.c	25 Feb 2005 14:33:52 -0000
@@ -30,6 +30,11 @@
 #include "zle.mdh"
 #include "zle_word.pro"
 
+/*
+ * TODO: use of iword needs completely rethinking for Unicode
+ * since we can't base it on a table lookup.
+ */
+
 /**/
 int
 forwardword(char **args)
@@ -354,7 +359,7 @@
 	while (zlecs != zlell && !iword(zleline[zlecs]))
 	    zlecs++;
 	while (zlecs != zlell && iword(zleline[zlecs])) {
-	    zleline[zlecs] = tuupper(zleline[zlecs]);
+	    zleline[zlecs] = ZS_toupper(zleline[zlecs]);
 	    zlecs++;
 	}
     }
@@ -376,7 +381,7 @@
 	while (zlecs != zlell && !iword(zleline[zlecs]))
 	    zlecs++;
 	while (zlecs != zlell && iword(zleline[zlecs])) {
-	    zleline[zlecs] = tulower(zleline[zlecs]);
+	    zleline[zlecs] = ZS_tolower(zleline[zlecs]);
 	    zlecs++;
 	}
     }
@@ -401,7 +406,8 @@
 	while (zlecs != zlell && iword(zleline[zlecs]) && !isalpha(zleline[zlecs]))
 	    zlecs++;
 	while (zlecs != zlell && iword(zleline[zlecs])) {
-	    zleline[zlecs] = (first) ? tuupper(zleline[zlecs]) : tulower(zleline[zlecs]);
+	    zleline[zlecs] = (first) ? ZS_toupper(zleline[zlecs]) :
+		ZS_tolower(zleline[zlecs]);
 	    first = 0;
 	    zlecs++;
 	}

-- 
Peter Stephenson <pws@csr.com>                  Software Engineer
CSR PLC, Churchill House, Cambridge Business Park, Cowley Road
Cambridge, CB4 0WZ, UK                          Tel: +44 (0)1223 692070


**********************************************************************
This email and any files transmitted with it are confidential and
intended solely for the use of the individual or entity to whom they
are addressed. If you have received this email in error please notify
the system manager.

**********************************************************************

Re: PATCH: zle TODOs etc.

[Clint Adams]

On Mon, Mar 21, 2005 at 02:09:13AM +0000, Bart Schaefer wrote:
> Some discussion of this is in zsh-workers/20869.

Oops.  Thanks.

On Fri, Feb 25, 2005 at 02:46:17PM +0000, Peter Stephenson wrote:
> Word stuff: this will need a whole new way of doing iword().  Probably
> that's going to have to be a function call.  The obvious thing to do is
> to have it on each call check that either the character iswalnum() or
> the multibyte representation of the character is in the string set by
> $WORDCHARS (which we could cache as wchar_t in zle).  That's slower, but
> probably not prohibitively so.  (The functions supplied with the shell
> can in principle do much more, but the underlying pattern matching tests
> there don't no about multibyte characters either.)

Is this the right way to go about this?

M  Src/Zle/zle_main.c
M  Src/init.c
M  Src/params.c

* modified files

--- orig/Src/Zle/zle_main.c
+++ mod/Src/Zle/zle_main.c
@@ -101,6 +101,11 @@
 /**/
 mod_export int
 lastchar_wide_valid;
+
+/**/
+mod_export ZLE_STRING_T zle_wordchars;
+#else
+# define zle_wordchars wordchars;
 #endif
 
 /* the bindings for the previous and for this key */
@@ -1507,6 +1512,15 @@
 	kungetct = 0;
 }
 
+/**/
+mod_export void
+wordcharshook(void)
+{
+#ifdef ZLE_UNICODE_SUPPORT
+    /* TODO: convert wordchars to wchar zle_wordchars */
+#endif
+}
+
 /* Hook functions. Used to allow access to zle parameters if zle is
  * active. */
 
@@ -1560,6 +1574,7 @@
     zlegetlineptr = zlegetline;
     zlereadptr = zleread;
     zlesetkeymapptr = zlesetkeymap;
+    wordcharshookptr = wordcharshook;
 
     getkeyptr = getbyte;
 
@@ -1645,6 +1660,7 @@
     zlegetlineptr = NULL;
     zlereadptr = fallback_zleread;
     zlesetkeymapptr= noop_function_int;
+    wordcharshookptr = noop_function;
 
     getkeyptr = NULL;
 


--- orig/Src/init.c
+++ mod/Src/init.c
@@ -1150,6 +1150,9 @@
 #endif /* !LINKED_XMOD_zshQszle */
 
 /**/
+mod_export ZleVoidFn wordcharshookptr = noop_function;
+
+/**/
 unsigned char *
 autoload_zleread(char **lp, char **rp, int ha, int con)
 {


--- orig/Src/params.c
+++ mod/Src/params.c
@@ -3316,6 +3316,7 @@
     zsfree(wordchars);
     wordchars = x;
     inittyptab();
+    wordcharshookptr();
 }
 
 /* Function to get value for special parameter `_' */

Re: PATCH: zle TODOs etc.

[Bart Schaefer]

On Mar 22,  7:36pm, Clint Adams wrote:
}
} Is this the right way to go about this?
} 
} +mod_export void
} +wordcharshook(void)

It looks OK to me, though you should probably wait for another opinion,
as I'm not really the expert on this part of the code (maybe no one is).

Two comments:

I suggest avoiding the term "hook" in this context, because that has a
specific meaning in the zsh code (see the HOOKDEF macro) and tweaking
the value of wordchars really doesn't fit that usage.

I'd move the assignment from the /* Set up editor entry points */ section
down to the /* miscellaneous initialisations */ section, as you're not
really defining a new entry point.

Re: PATCH: zle TODOs etc.

[Clint Adams]

> Two comments:

Changed to trigger, for lack of a better idea.


M  Src/Zle/zle_main.c
M  Src/init.c
M  Src/params.c

* modified files

--- orig/Src/Zle/zle_main.c
+++ mod/Src/Zle/zle_main.c
@@ -101,6 +101,11 @@
 /**/
 mod_export int
 lastchar_wide_valid;
+
+/**/
+mod_export ZLE_STRING_T zle_wordchars;
+#else
+# define zle_wordchars wordchars;
 #endif
 
 /* the bindings for the previous and for this key */
@@ -1507,6 +1512,15 @@
 	kungetct = 0;
 }
 
+/**/
+mod_export void
+wordcharstrigger(void)
+{
+#ifdef ZLE_UNICODE_SUPPORT
+    /* TODO: convert wordchars to wchar zle_wordchars */
+#endif
+}
+
 /* Hook functions. Used to allow access to zle parameters if zle is
  * active. */
 
@@ -1572,6 +1586,7 @@
     kungetbuf = (char *) zalloc(kungetsz = 32);
     comprecursive = 0;
     rdstrs = NULL;
+    wordcharstriggerptr = wordcharstrigger;
 
     /* initialise the keymap system */
     init_keymaps();
@@ -1645,6 +1660,7 @@
     zlegetlineptr = NULL;
     zlereadptr = fallback_zleread;
     zlesetkeymapptr= noop_function_int;
+    wordcharstriggerptr = noop_function;
 
     getkeyptr = NULL;
 


--- orig/Src/init.c
+++ mod/Src/init.c
@@ -1152,6 +1152,9 @@
 #endif /* !LINKED_XMOD_zshQszle */
 
 /**/
+mod_export ZleVoidFn wordcharstriggerptr = noop_function;
+
+/**/
 unsigned char *
 autoload_zleread(char **lp, char **rp, int ha, int con)
 {


--- orig/Src/params.c
+++ mod/Src/params.c
@@ -3316,6 +3316,7 @@
     zsfree(wordchars);
     wordchars = x;
     inittyptab();
+    wordcharstriggerptr();
 }
 
 /* Function to get value for special parameter `_' */