PATCH: alternate views on .zle.hlgroups

PATCH: alternate views on .zle.hlgroups

[Oliver Kiddle]

This patch adds a hlgroup module which defines two extra special
variables - .zle.esc and .zle.sgr. These are readonly associative
arrays.

The purpose of these is to make is easier to apply the attributes
defined in .zle.hlgroups to other commands. The same effect can already
be achieved using ${(%):-%H\{...\}} but this seemed like a friendlier
approach.

.zle.esc contains literal escape sequences
.zle.sgr is stripped down to just the "Select Graphic Rendition" number
sequence. This is useful, with, e.g. GREP_COLORS, LSCOLORS, the
list-colors style and some other tools like jq and ag.

This implementation uses dupstring() to return memory from the parameter
string get function. I don't see existing cases to confirm that this
is safe but it appears to work fine in testing. Would be good to have
confirmation from someone who has a clearer understanding of what the
lifetime of the memory pools is.

Naming (hlgroup module in singular, .zle.hlgroups in plural) is chosen
for consistency with the existing parameter module. I might otherwise
have opted for singular in both cases. But I'm open to other suggestions
or completely different ideas on naming.

I had two further ideas for things that might be included in the module.
I briefly considered what it would involve to support a hook to allow
dynamic updates to variables like GREP_COLORS in response to changes to
.zle.hlgroups. This didn't seem trivial, especially given that it is
currently useful to define it as a nameref (to a variable that works in
older zsh).

My second thought was support for parsing terminal responses to the
escape sequence for getting the background colour. This is just about
doable in shell code but not trivial and error-prone. But it can be
very helpful to avoid unreadable colour combinations. What form should
this best take? A builtin? Having a key bound to \e]11;rgb: may be
the best way to avoid interference with a type-ahead buffer but that
inconveniently move the setup code to a zle widget. ungetc() in C may
work better than my current print -z solution, especially where some of
the typeahead text is not even intended for zsh.

Oliver

diff --git a/Src/Modules/hlgroup.c b/Src/Modules/hlgroup.c
new file mode 100644
index 000000000..fcf8cfc81
--- /dev/null
+++ b/Src/Modules/hlgroup.c
@@ -0,0 +1,182 @@
+#include "hlgroup.mdh"
+#include "hlgroup.pro"
+
+#define GROUPVAR ".zle.hlgroups"
+
+static const struct gsu_scalar pmesc_gsu =
+{ strgetfn, nullstrsetfn, nullunsetfn };
+
+/**/
+static char *
+convertattr(char *attrstr, int sgr)
+{
+    zattr atr;
+    char *r, *s;
+    int len;
+
+    match_highlight(attrstr, &atr, NULL);
+    s = zattrescape(atr, sgr ? NULL : &len);
+
+    if (sgr) {
+	char *c = s, *t = s - 1;
+
+	while (c[0] == '\033' && c[1] == '[') {
+	    c += 2;
+	    while (isdigit(*c) || *c == ';')
+		*++t = *c++;
+	    t++;
+	    if (*c != 'm')
+		break;
+	    *t = ';';
+	    c++;
+	}
+	*t = '\0';
+	len = t - s;
+    }
+
+    r = dupstring_wlen(s, len);
+    free(s);
+    return r;
+}
+
+/**/
+static HashNode
+getgroup(const char *name, int sgr)
+{
+    Param pm = NULL;
+    HashNode hn;
+    HashTable hlg;
+    Value v;
+    struct value vbuf;
+    char *var = GROUPVAR;
+
+    pm = (Param) hcalloc(sizeof(struct param));
+    pm->gsu.s = &pmesc_gsu;
+    pm->node.nam = dupstring(name);
+    pm->node.flags = PM_SCALAR|PM_SPECIAL;
+
+    if (!(v = getvalue(&vbuf, &var, 0)) ||
+	     PM_TYPE(v->pm->node.flags) != PM_HASHED ||
+	     !(hlg = v->pm->gsu.h->getfn(v->pm)) ||
+	     !(hn = gethashnode2(hlg, name)))
+    {
+	pm->u.str = dupstring("");
+	pm->node.flags |= PM_UNSET;
+    } else {
+	pm->u.str = convertattr(((Param) hn)->u.str, sgr);
+    }
+
+    return &pm->node;
+}
+
+/**/
+static void
+scangroup(ScanFunc func, int flags, int sgr)
+{
+    struct param pm;
+    int i;
+    HashNode hn;
+    HashTable hlg;
+    Value v;
+    struct value vbuf;
+    char *var = GROUPVAR;
+
+    if (!(v = getvalue(&vbuf, &var, 0)) ||
+	     PM_TYPE(v->pm->node.flags) != PM_HASHED)
+	return;
+    hlg = v->pm->gsu.h->getfn(v->pm);
+
+    memset((void *)&pm, 0, sizeof(struct param));
+    pm.node.flags = PM_SCALAR;
+    pm.gsu.s = &pmesc_gsu;
+
+    for (i = 0; i < hlg->hsize; i++)
+	for (hn = hlg->nodes[i]; hn; hn = hn->next) {
+	    pm.u.str = convertattr(((Param) hn)->u.str, sgr);
+	    pm.node.nam = hn->nam;
+	    func(&pm.node, flags);
+	}
+}
+/**/
+static HashNode
+getpmesc(UNUSED(HashTable ht), const char *name)
+{
+    return getgroup(name, 0);
+}
+
+/**/
+static void
+scanpmesc(UNUSED(HashTable ht), ScanFunc func, int flags)
+{
+    return scangroup(func, flags, 0);
+}
+
+/**/
+static HashNode
+getpmsgr(UNUSED(HashTable ht), const char *name)
+{
+    return getgroup(name, 1);
+}
+
+/**/
+static void
+scanpmsgr(UNUSED(HashTable ht), ScanFunc func, int flags)
+{
+    return scangroup(func, flags, 1);
+}
+
+static struct paramdef partab[] = {
+    SPECIALPMDEF(".zle.esc", PM_READONLY_SPECIAL, 0, getpmesc, scanpmesc),
+    SPECIALPMDEF(".zle.sgr", PM_READONLY_SPECIAL, 0, getpmsgr, scanpmsgr)
+};
+
+static struct features module_features = {
+    NULL, 0,
+    NULL, 0,
+    NULL, 0,
+    partab, sizeof(partab)/sizeof(*partab),
+    0
+};
+
+/**/
+int
+setup_(UNUSED(Module m))
+{
+    return 0;
+}
+
+/**/
+int
+features_(Module m, char ***features)
+{
+    *features = featuresarray(m, &module_features);
+    return 0;
+}
+
+/**/
+int
+enables_(Module m, int **enables)
+{
+    return handlefeatures(m, &module_features, enables);
+}
+
+/**/
+int
+boot_(UNUSED(Module m))
+{
+    return 0;
+}
+
+/**/
+int
+cleanup_(Module m)
+{
+    return setfeatureenables(m, &module_features, NULL);
+}
+
+/**/
+int
+finish_(UNUSED(Module m))
+{
+    return 0;
+}
diff --git a/Src/Modules/hlgroup.mdd b/Src/Modules/hlgroup.mdd
new file mode 100644
index 000000000..ee3ba7260
--- /dev/null
+++ b/Src/Modules/hlgroup.mdd
@@ -0,0 +1,7 @@
+name=zsh/hlgroup
+link=either
+load=yes
+
+autofeatures="p:.zle.esc p:.zle.sgr"
+
+objects="hlgroup.o"
diff --git a/Src/prompt.c b/Src/prompt.c
index 0d674ceab..7acbe0e47 100644
--- a/Src/prompt.c
+++ b/Src/prompt.c
@@ -241,6 +241,34 @@ promptexpand(char *s, int ns, char *rs, char *Rs)
     return new_vars.buf;
 }
 
+/* Get the escape sequence for a given attribute. */
+/**/
+mod_export char *
+zattrescape(zattr atr, int *len)
+{
+    struct buf_vars new_vars;
+    zattr savecurrent = txtcurrentattrs;
+    zattr saveunknown = txtunknownattrs;
+
+    memset(&new_vars, 0, sizeof(new_vars));
+    new_vars.last = bv;
+    bv = &new_vars;
+    new_vars.bufspc = 256;
+    new_vars.bp = new_vars.bufline = new_vars.buf = zshcalloc(new_vars.bufspc);
+    new_vars.dontcount = 1;
+
+    txtunknownattrs = 0;
+    treplaceattrs(atr);
+    applytextattributes(TSC_PROMPT);
+
+    bv = new_vars.last;
+
+    txtpendingattrs = txtcurrentattrs = savecurrent;
+    txtunknownattrs = saveunknown;
+
+    return unmetafy(new_vars.buf, len);
+}
+
 /* Parse the argument for %H */
 static char *
 parsehighlight(char *arg, char endchar, zattr *atr)

Re: PATCH: alternate views on .zle.hlgroups

[Bart Schaefer]

I don't do much with zle_highlight so I have only general comments ...

On Sun, Feb 11, 2024 at 9:26 PM Oliver Kiddle <opk@zsh.org> wrote:
>
> This implementation uses dupstring() to return memory from the parameter
> string get function.

The unset functions generally expect to free the memory from the
parameter value, but since these are readonly specials it's probably
safe to use the heap (see below).  I don't know what oddities might
arise from declaring a "local +h" on these names.

> Would be good to have
> confirmation from someone who has a clearer understanding of what the
> lifetime of the memory pools is.

There can be a stack of pools so the one that gets used will be the
one that's active at the time the parameter is referenced.  Assigning
the value to another parameter will copy the value, not point to it,
so the only iffy bit is whether the getfn refreshes the u.str pointer
every time the variable is referenced.

I believe there's a new pool during zle widgets, for example, so if
you reference one of these both inside a call to "zle
user-defined-thing" and then again after zle returns, this could be
tickled.

> My second thought was support for parsing terminal responses to the
> escape sequence for getting the background colour. [...]
> very helpful to avoid unreadable colour combinations. What form should
> this best take? A builtin? Having a key bound to \e]11;rgb: may be
> the best way to avoid interference with a type-ahead buffer but that
> inconveniently move the setup code to a zle widget. ungetc() in C may
> work better than my current print -z solution, especially where some of
> the typeahead text is not even intended for zsh.

The shell input is unbuffered** and managed by the shell itself, so I
don't know if ungetc() works at all.  There are no calls to ungetc
remaining in Src/*.c, they all use either ihungetc() or inungetc() via
the hungetc code pointer.  So I think if there's input not intended
for zsh you, either mustn't read it at all or there's nothing you can
do about it.

I'd use "zle -U" rather than "print -z" (or choose one or the other
based on context if "zle -U" isn't always usable at the time).  I
suppose you do need to account for -U being LIFO, but -z is the buffer
stack so popped one entry at a time so probably also won't do the
right thing if called more than once successively.

**Where support-able, else line buffered.

Re: PATCH: alternate views on .zle.hlgroups

[Oliver Kiddle]

Bart Schaefer wrote:
> I don't do much with zle_highlight so I have only general comments ...

Thanks. That gave me a few more ideas of things to test, none of which
triggered outcomes that appeared iffy.

I've attached a doc patch though some things remain that may need
addressing in the code. In particular resetting attributes after %H
in a prompt and whether the layers and rules for merging attributes
are sufficient for the needs of plugins like zsh-syntax-highlighting.
Probing the tty for background colour is something that'll probably need
testing privately for a while first.

Oliver

diff --git a/Doc/Makefile.in b/Doc/Makefile.in
index dabe11fe3..d9be182e9 100644
--- a/Doc/Makefile.in
+++ b/Doc/Makefile.in
@@ -63,7 +63,8 @@ Zsh/mod_attr.yo Zsh/mod_cap.yo Zsh/mod_clone.yo \
 Zsh/mod_compctl.yo Zsh/mod_complete.yo Zsh/mod_complist.yo \
 Zsh/mod_computil.yo Zsh/mod_curses.yo \
 Zsh/mod_datetime.yo Zsh/mod_db_gdbm.yo Zsh/mod_deltochar.yo \
-Zsh/mod_example.yo Zsh/mod_files.yo Zsh/mod_langinfo.yo \
+Zsh/mod_example.yo Zsh/mod_files.yo \
+Zsh/mod_hlgroup.yo Zsh/mod_langinfo.yo \
 Zsh/mod_ksh93.yo Zsh/mod_mapfile.yo Zsh/mod_mathfunc.yo \
 Zsh/mod_nearcolor.yo Zsh/mod_newuser.yo \
 Zsh/mod_parameter.yo Zsh/mod_pcre.yo Zsh/mod_private.yo \
diff --git a/Doc/Zsh/mod_hlgroup.yo b/Doc/Zsh/mod_hlgroup.yo
new file mode 100644
index 000000000..efe8934a1
--- /dev/null
+++ b/Doc/Zsh/mod_hlgroup.yo
@@ -0,0 +1,25 @@
+COMMENT(!MOD!zsh/hlgroup
+Alternative views of highlighting groups
+!MOD!)
+The tt(zsh/hlgroup) module defines special parameters that represent
+highlighting groups in different forms to ease the use of the groups when
+configuring other tools.
+
+In each case, these are readonly associative arrays where accessing elements
+uses values from the underlying tt(.zle.hlgroups) variable.
+
+startitem()
+vindex(.zle.esc)
+item(tt(.zle.esc))(
+This associative array contains the literal escape sequences used to apply the
+highlighting for each group. An example use would be when setting the
+tt(LESS_TERMCAP_xx) environment variables for the tt(less) pager.
+)
+vindex(.zle.sgr)
+item(tt(.zle.sgr))(
+Where highlighting makes use of CSI escape sequences, this parameter contains
+the "Select Graphic Rendition" number sequence. This is useful with, for
+example the tt(GREP_COLORS) and tt(LSCOLORS) environment variables and the
+tt(list-colors) style.
+)
+enditem()
diff --git a/Doc/Zsh/prompt.yo b/Doc/Zsh/prompt.yo
index 909012c8e..de988ab7c 100644
--- a/Doc/Zsh/prompt.yo
+++ b/Doc/Zsh/prompt.yo
@@ -246,6 +246,14 @@ item(tt(%K) LPAR()tt(%k)RPAR())(
 Start (stop) using a different bacKground colour.  The syntax is
 identical to that for tt(%F) and tt(%f).
 )
+item(tt(%H))(
+Change all character visual attributes using a highlighting specification from
+the tt(.zle.hlgroups) associative array. The key is specified in following
+braces so, for example tt(%H{error}) will use the highlighting specification
+for the `error' group.  If the key is not found in the associative array then
+it has no effect.  Highlighting specifications are in the same format as for
+the tt(zle_highlight) parameter.
+)
 item(tt(%{)...tt(%}))(
 Include a string as a literal escape sequence.
 The string within the braces should not change the cursor
diff --git a/Doc/Zsh/zle.yo b/Doc/Zsh/zle.yo
index 495bd86a8..31eb3f3ba 100644
--- a/Doc/Zsh/zle.yo
+++ b/Doc/Zsh/zle.yo
@@ -2807,6 +2807,17 @@ item(tt(italic))(
 The characters in the given context are shown in a italic font.
 Not all terminals support italic fonts.
 )
+item(tt(hl=)var(group))(
+Use the specified highlighting group.  The var(group) is used as a key into
+the associative array tt(.zle.hlgroups) to determine the actual highlighting.
+)
+item(tt(layer=)var(layer))(
+The layer is used to determine precedence when multiple highlighting regions
+overlap. The var(layer) is a decimal integer, with higher numbers taking
+precedence over lower numbers. The default layer is 10 with 30 used as the
+default for tt(special), 20 for tt(region) and tt(isearch) and 15 for
+tt(paste).
+)
 enditem()
 
 The characters described above as `special' are as follows.  The
diff --git a/NEWS b/NEWS
index 4d4699f7e..d0a8584e2 100644
--- a/NEWS
+++ b/NEWS
@@ -10,6 +10,12 @@ Changes since 5.9
 In region_highlight and zle_highlight, italic and faint can be
 specified as font attributes for terminals that support them.
 
+Highlighting groups can be referenced in region_highlight and
+zle_highlight for common attribute combinations and a layer can be
+specified to indicate precedence where highlighted regions overlap.
+Highlighting groups are also supported in the prompt via a new %H
+prompt escape.
+
 Ellipsis markers shown by the line editor to indicate where the line
 doesn't fit in the terminal can be highlighted.