[PATCH 1/3] Extended ksh compatibility: namespace parameter syntax

[PATCH 1/3] Extended ksh compatibility: namespace parameter syntax

[Bart Schaefer]

[-- Attachment #1: Type: text/plain, Size: 570 bytes --]

The attached patch enables assignment of and reference to parameters
prefixed with ksh-style namespace syntax, e.g., ${.namespace.param}.
As yet, there's no special significance to use of this syntax, that
is, parameters having such a prefix are ordinary parameters like any
others, with the usual dynamic scoping rules, etc.  Each of namespace
and param must be an identifier in the original zsh semantics of
identifiers.

The braces are required when using this syntax in a parameter
substitution, so without braces $foo.bar continues to be interpreted
as ${foo}.bar.

[-- Attachment #2: ns-param-syntax.txt --]
[-- Type: text/plain, Size: 8029 bytes --]

diff --git a/Src/Zle/compcore.c b/Src/Zle/compcore.c
index 64a860fa3..77fce66e8 100644
--- a/Src/Zle/compcore.c
+++ b/Src/Zle/compcore.c
@@ -1230,14 +1230,14 @@ check_param(char *s, int set, int test)
 	else if (idigit(*e))
 	    while (idigit(*e))
 		e++;
-	else if ((ie = itype_end(e, IIDENT, 0)) != e) {
+	else if ((ie = itype_end(e, INAMESPC, 0)) != e) {
 	    do {
 		e = ie;
 		if (comppatmatch && *comppatmatch &&
 		    (*e == Star || *e == Quest))
 		    ie = e + 1;
 		else
-		    ie = itype_end(e, IIDENT, 0);
+		    ie = itype_end(e, INAMESPC, 0);
 	    } while (ie != e);
 	}
 
diff --git a/Src/Zle/zle_tricky.c b/Src/Zle/zle_tricky.c
index 07fac7144..690cf6efb 100644
--- a/Src/Zle/zle_tricky.c
+++ b/Src/Zle/zle_tricky.c
@@ -576,7 +576,7 @@ parambeg(char *s)
 	    while (idigit(*e))
 		e++;
 	else
-	    e = itype_end(e, IIDENT, 0);
+	    e = itype_end(e, INAMESPC, 0);
 
 	/* Now make sure that the cursor is inside the name. */
 	if (offs <= e - s && offs >= b - s && n <= 0) {
@@ -765,7 +765,7 @@ docomplete(int lst)
 			    else if (idigit(*q))
 				do q++; while (idigit(*q));
 			    else
-				q = itype_end(q, IIDENT, 0);
+				q = itype_end(q, INAMESPC, 0);
 			    sav = *q;
 			    *q = '\0';
 			    if (zlemetacs - wb == q - s &&
@@ -1497,7 +1497,7 @@ get_comp_string(void)
 	if (varq)
 	    tt = clwords[clwpos];
 
-	s = itype_end(tt, IIDENT, 0);
+	s = itype_end(tt, INAMESPC, 0);
 	sav = *s;
 	*s = '\0';
 	zsfree(varname);
diff --git a/Src/lex.c b/Src/lex.c
index 15da85a93..2f7937410 100644
--- a/Src/lex.c
+++ b/Src/lex.c
@@ -1230,7 +1230,7 @@ gettokstr(int c, int sub)
 		    else {
 			int sav = *lexbuf.ptr;
 			*lexbuf.ptr = '\0';
-			t = itype_end(t, IIDENT, 0);
+			t = itype_end(t, INAMESPC, 0);
 			if (t < lexbuf.ptr) {
 			    skipparens(Inbrack, Outbrack, &t);
 			} else {
diff --git a/Src/params.c b/Src/params.c
index 90302b1b0..d3b6a7d43 100644
--- a/Src/params.c
+++ b/Src/params.c
@@ -1223,7 +1223,7 @@ isident(char *s)
 		break;
     } else {
 	/* Find the first character in `s' not in the iident type table */
-	ss = itype_end(s, IIDENT, 0);
+	ss = itype_end(s, INAMESPC, 0);
     }
 
     /* If the next character is not [, then it is *
@@ -2086,6 +2086,7 @@ fetchvalue(Value v, char **pptr, int bracks, int flags)
     char *s, *t, *ie;
     char sav, c;
     int ppar = 0;
+    int itype = (flags & SCANPM_NONAMESPC) ? IIDENT : INAMESPC;
 
     s = t = *pptr;
 
@@ -2095,7 +2096,7 @@ fetchvalue(Value v, char **pptr, int bracks, int flags)
 	else
 	    ppar = *s++ - '0';
     }
-    else if ((ie = itype_end(s, IIDENT, 0)) != s)
+    else if ((ie = itype_end(s, itype, 0)) != s)
 	s = ie;
     else if (c == Quest)
 	*s++ = '?';
@@ -2183,7 +2184,7 @@ fetchvalue(Value v, char **pptr, int bracks, int flags)
 		return v;
 	    }
 	} else if (!(flags & SCANPM_ASSIGNING) && v->isarr &&
-		   itype_end(t, IIDENT, 1) != t && isset(KSHARRAYS))
+		   itype_end(t, INAMESPC, 1) != t && isset(KSHARRAYS))
 	    v->end = 1, v->isarr = 0;
     }
     if (!bracks && *s)
@@ -6196,7 +6197,7 @@ setscope(Param pm)
     if (pm->node.flags & PM_NAMEREF) {
 	Param basepm;
 	struct asgment stop;
-	char *t = pm->u.str ? itype_end(pm->u.str, IIDENT, 0) : NULL;
+	char *t = pm->u.str ? itype_end(pm->u.str, INAMESPC, 0) : NULL;
 
 	/* Temporarily change nameref to array parameter itself */
 	if (t && *t == '[')
@@ -6277,7 +6278,7 @@ upscope(Param pm, int reflevel)
 mod_export int
 valid_refname(char *val)
 {
-    char *t = itype_end(val, IIDENT, 0);
+    char *t = itype_end(val, INAMESPC, 0);
 
     if (*t != 0) {
 	if (*t == '[') {
diff --git a/Src/subst.c b/Src/subst.c
index 05bfcc03b..7a4b433bc 100644
--- a/Src/subst.c
+++ b/Src/subst.c
@@ -1870,7 +1870,7 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int pf_flags,
      * these later on, too.
      */
     c = *s;
-    if (itype_end(s, IIDENT, 1) == s && *s != '#' && c != Pound &&
+    if (itype_end(s, INAMESPC, 1) == s && *s != '#' && c != Pound &&
 	!IS_DASH(c) &&
 	c != '!' && c != '$' && c != String && c != Qstring &&
 	c != '?' && c != Quest &&
@@ -2332,7 +2332,7 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int pf_flags,
 	    }
 	} else if ((c == '#' || c == Pound) &&
 		   (inbrace || !isset(POSIXIDENTIFIERS)) &&
-		   (itype_end(s+1, IIDENT, 0) != s + 1
+		   (itype_end(s+1, INAMESPC, 0) != s + 1
 		    || (cc = s[1]) == '*' || cc == Star || cc == '@'
 		    || cc == '?' || cc == Quest
 		    || cc == '$' || cc == String || cc == Qstring
@@ -2369,8 +2369,9 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int pf_flags,
 	     * Try to handle this when parameter is named
 	     * by (P) (second part of test).
 	     */
-	    if (itype_end(s+1, IIDENT, 0) != s+1 || (aspar && isstring(s[1]) &&
-				 (s[2] == Inbrace || s[2] == Inpar)))
+	    if (itype_end(s+1, INAMESPC, 0) != s+1 ||
+		(aspar && isstring(s[1]) &&
+		 (s[2] == Inbrace || s[2] == Inpar)))
 		chkset = 1, s++;
 	    else if (!inbrace) {
 		/* Special case for `$+' on its own --- leave unmodified */
@@ -2531,6 +2532,8 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int pf_flags,
 	    scanflags |= SCANPM_DQUOTED;
 	if (chkset)
 	    scanflags |= SCANPM_CHECKING;
+	if (!inbrace)
+	    scanflags |= SCANPM_NONAMESPC;
 	/*
 	 * Second argument: decide whether to use the subexpression or
 	 *   the string next on the line as the parameter name.
@@ -3211,7 +3214,7 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int pf_flags,
 	    shortest = 0;
 	    ++s;
 	}
-	if (*itype_end(s, IIDENT, 0)) {
+	if (*itype_end(s, INAMESPC, 0)) {
 	    untokenize(s);
 	    zerr("not an identifier: %s", s);
 	    return NULL;
@@ -3271,7 +3274,7 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int pf_flags,
 	int intersect = (*s == '*' || *s == Star);
 	char **compare, **ap, **apsrc;
 	++s;
-	if (*itype_end(s, IIDENT, 0)) {
+	if (*itype_end(s, INAMESPC, 0)) {
 	    untokenize(s);
 	    zerr("not an identifier: %s", s);
 	    return NULL;
diff --git a/Src/utils.c b/Src/utils.c
index 55f2d1ab0..1393ecb13 100644
--- a/Src/utils.c
+++ b/Src/utils.c
@@ -3123,7 +3123,7 @@ spckword(char **s, int hist, int cmd, int ask)
 
     if (**s == String && !*t) {
 	guess = *s + 1;
-	if (itype_end(guess, IIDENT, 1) == guess)
+	if (itype_end(guess, INAMESPC, 1) == guess)
 	    return;
 	ic = String;
 	d = 100;
@@ -4310,13 +4310,27 @@ wcsitype(wchar_t c, int itype)
  * If "once" is set, just test the first character, i.e. (outptr !=
  * inptr) tests whether the first character is valid in an identifier.
  *
- * Currently this is only called with itype IIDENT, IUSER or ISEP.
+ * Currently called only with itype INAMESPC, IIDENT, IUSER or ISEP.
  */
 
 /**/
 mod_export char *
 itype_end(const char *ptr, int itype, int once)
 {
+    if (itype == INAMESPC) {
+	itype = IIDENT;
+	if (once == 0 && !isset(POSIXIDENTIFIERS)) {
+	    /* Special case for names containing ".", ksh93 namespaces */
+	    char *t = itype_end(ptr + (*ptr == '.'), itype, 0);
+	    if (t > ptr+1) {
+		if (*t == '.')
+		    return itype_end(t+1, itype, 0);
+		else
+		    return t;
+	    }
+	}
+    }
+
 #ifdef MULTIBYTE_SUPPORT
     if (isset(MULTIBYTE) &&
 	(itype != IIDENT || !isset(POSIXIDENTIFIERS))) {
diff --git a/Src/zsh.h b/Src/zsh.h
index 96b4b06bd..0de1f7afb 100644
--- a/Src/zsh.h
+++ b/Src/zsh.h
@@ -1963,6 +1963,8 @@ struct tieddata {
 				  */
 #define SCANPM_CHECKING   (1<<10) /* Check if set, no need to create */
 #define SCANPM_NOEXEC     (1<<11) /* No command substitutions, etc. */
+#define SCANPM_NONAMESPC  (1<<12) /* namespace syntax not allowed */
+
 /* "$foo[@]"-style substitution
  * Only sign bit is significant
  */
diff --git a/Src/ztype.h b/Src/ztype.h
index 8757fc733..4675f73a9 100644
--- a/Src/ztype.h
+++ b/Src/ztype.h
@@ -43,6 +43,7 @@
 #define IWSEP    (1 << 13)
 #define INULL    (1 << 14)
 #define IPATTERN (1 << 15)
+#define INAMESPC (1 << 16)
 #define zistype(X,Y) (typtab[(unsigned char) (X)] & Y)
 #define idigit(X) zistype(X,IDIGIT)
 #define ialnum(X) zistype(X,IALNUM)

Re: [PATCH 1/3] Extended ksh compatibility: namespace parameter syntax

[Oliver Kiddle]

On 26 Feb, Bart Schaefer wrote:
> The attached patch enables assignment of and reference to parameters
> prefixed with ksh-style namespace syntax, e.g., ${.namespace.param}.
> As yet, there's no special significance to use of this syntax, that
> is, parameters having such a prefix are ordinary parameters like any
> others, with the usual dynamic scoping rules, etc.  Each of namespace
> and param must be an identifier in the original zsh semantics of
> identifiers.

I've been running this patch since you posted it and haven't found it to
cause any problems as such. It also holds up to testing but with no way
to assign to variables with a dot in their name, there's a limit to
what's possible to test.

My main concern is that in the absence of special significance to the
syntax up-front it could be harder to add later. At least the
documentation should warn so users should not be surprised if e.g, in the
future, `local foo` will hide a ${foo.bar}. But every new module like ksh93
could need reworking if the implementation changes so that this is a bar
entry in a foo hash table instead of a foo.bar entry in a global hash
table. I'm not sure how ksh implements the initial dot. It effectively
blocks consecutive dots because you have to declare the parent first.

Ksh uses dots both for namespaces and compound variables. Of the two,
compound variables are definitely the most useful. The namespace { ... }
syntax of ksh was not designed with the interactive nature of the shell
being considered foremost. I like the aspect of hiding shell specific
variables away below .sh - could also be good for .zle.

I think it is a mistake to not make compound variables, namespaces and
associative arrays merely different facets of the exact same underlying
thing. I'd like to be able to dump variables out as json/xml/yaml and
reimport them back without the loss of detail of which type they started
out as.

Oliver

Re: [PATCH 1/3] Extended ksh compatibility: namespace parameter syntax

[Sebastian Gniazdowski]

[-- Attachment #1: Type: text/plain, Size: 2188 bytes --]

I wonder, how far is this from supporting a nested arrays? Like arr[2][3]
or assoc[a][c]?

On Sun, 5 Mar 2023 at 09:11, Oliver Kiddle <opk@zsh.org> wrote:

> On 26 Feb, Bart Schaefer wrote:
> > The attached patch enables assignment of and reference to parameters
> > prefixed with ksh-style namespace syntax, e.g., ${.namespace.param}.
> > As yet, there's no special significance to use of this syntax, that
> > is, parameters having such a prefix are ordinary parameters like any
> > others, with the usual dynamic scoping rules, etc.  Each of namespace
> > and param must be an identifier in the original zsh semantics of
> > identifiers.
>
> I've been running this patch since you posted it and haven't found it to
> cause any problems as such. It also holds up to testing but with no way
> to assign to variables with a dot in their name, there's a limit to
> what's possible to test.
>
> My main concern is that in the absence of special significance to the
> syntax up-front it could be harder to add later. At least the
> documentation should warn so users should not be surprised if e.g, in the
> future, `local foo` will hide a ${foo.bar}. But every new module like ksh93
> could need reworking if the implementation changes so that this is a bar
> entry in a foo hash table instead of a foo.bar entry in a global hash
> table. I'm not sure how ksh implements the initial dot. It effectively
> blocks consecutive dots because you have to declare the parent first.
>
> Ksh uses dots both for namespaces and compound variables. Of the two,
> compound variables are definitely the most useful. The namespace { ... }
> syntax of ksh was not designed with the interactive nature of the shell
> being considered foremost. I like the aspect of hiding shell specific
> variables away below .sh - could also be good for .zle.
>
> I think it is a mistake to not make compound variables, namespaces and
> associative arrays merely different facets of the exact same underlying
> thing. I'd like to be able to dump variables out as json/xml/yaml and
> reimport them back without the loss of detail of which type they started
> out as.
>
> Oliver
>
>

-- 
Best regards,
Sebastian Gniazdowski

[-- Attachment #2: Type: text/html, Size: 2870 bytes --]

Re: [PATCH 1/3] Extended ksh compatibility: namespace parameter syntax

[Bart Schaefer]

On Sun, Mar 5, 2023 at 1:10 AM Oliver Kiddle <opk@zsh.org> wrote:
>
> On 26 Feb, Bart Schaefer wrote:
> > The attached patch enables assignment of and reference to parameters
> > prefixed with ksh-style namespace syntax, e.g., ${.namespace.param}.
>
> I've been running this patch since you posted it and haven't found it to
> cause any problems as such. It also holds up to testing but with no way
> to assign to variables with a dot in their name, there's a limit to
> what's possible to test.

Assignment should be possible.

% .foo.bar=something
% print ${.foo.bar}
something
% print ${.foo.bar::=otherthing}
otherthing
% typeset -m .foo.\*
.foo.bar=otherthing

> My main concern is that in the absence of special significance to the
> syntax up-front it could be harder to add later. At least the
> documentation should warn so users should not be surprised if e.g, in the
> future, `local foo` will hide a ${foo.bar}.

Yes, I've been working on doc and tests.  One difference that I want
to add is that parameters starting with "." are omitted from the
default output of "set" and "typeset" (with no arguments).

> But every new module like ksh93
> could need reworking if the implementation changes so that this is a bar
> entry in a foo hash table instead of a foo.bar entry in a global hash
> table.

Exactly how far we (or I) want to go with this is still an open
question.  Namespaces do not nest, so anything with a leading "." (and
exactly one further ".") is in the global hash table.

> Ksh uses dots both for namespaces and compound variables. Of the two,
> compound variables are definitely the most useful.

In ksh (based on doc and some testing, I'm not a ksh expert)
${foo.bar} might be a reference to ${foo[bar]} or it might be a call
to the function named foo.bar to invoke the "bar" property of "foo"
(or it might even be that ksh implements the former as the latter).
In the foregoing patch I have allowed the syntax without the leading
"." to be parsed, but so far not ${foo.bar.baz} etc., so the doc I've
written so far does warn that that leaving off the first "." might do
something different later.  Patch following soon.

> I like the aspect of hiding shell specific
> variables away below .sh - could also be good for .zle.

Given zsh dynamic scoping, it works reasonably well to have ZLE
variables behave like locals -- you can't access them outside a shell
function (widget) anyway.  If there were controls we wanted to expose
higher up -- such as, perhaps, fiddling with keymaps without having to
invoke "zle -K" or "bindkey -[eva]") then yes, a ".zle" namespace
could be handy.

> I think it is a mistake to not make compound variables, namespaces and
> associative arrays merely different facets of the exact same underlying
> thing.

Agreed with respect to associative arrays and compound variables
(whatever that ends up meaning to us, I find the ksh implementation
rather grotty especially from a syntax standpoint, and there are some
significant conflicts with existing zsh idioms).  Namespaces actually
are a slightly different thing because the variety of "objects" in a
namespace isn't limited to parameters.  If we were already using a
global hash table to contain as its elements the parameters, commands,
aliases, functions, etc. hash tables, that might be a different
matter, but we're pretty far from that now.

> I'd like to be able to dump variables out as json/xml/yaml and
> reimport them back without the loss of detail of which type they started
> out as.

For that, there are other obstacles (not related to namespaces or
compound variables) remaining to be overcome, but it's worth keeping
in mind.

Re: [PATCH 1/3] Extended ksh compatibility: namespace parameter syntax

[Bart Schaefer]

On Sun, Mar 5, 2023 at 1:57 AM Sebastian Gniazdowski
<sgniazdowski@gmail.com> wrote:
>
> I wonder, how far is this from supporting a nested arrays? Like arr[2][3] or assoc[a][c]?

It's exactly as far as it has always been.  Zsh's internal
implementation of indexed arrays does not support
pointers-to-pointers, and associative arrays have always had most of
the plumbing for that but not what might be called the contextual
clues.  None of what's in these patches gets any closer to refactoring
for either case.

Re: [PATCH 1/3] Extended ksh compatibility: namespace parameter syntax

[Bart Schaefer]

On Sun, Mar 5, 2023 at 12:20 PM Bart Schaefer <schaefer@brasslantern.com> wrote:
>
> On Sun, Mar 5, 2023 at 1:57 AM Sebastian Gniazdowski
> <sgniazdowski@gmail.com> wrote:
> >
> > I wonder, how far is this from supporting a nested arrays? Like arr[2][3] or assoc[a][c]?
>
> It's exactly as far as it has always been.

The other unanswered question is, to what does arr[2] expand when
arr[2][3] is also an array?  That is, when you have to subscript 3+
times to reach a scalar?  There's no convenient text representation of
array-of-array; do we just throw an error if there are too few levels
of dereference?

Re: [PATCH 1/3] Extended ksh compatibility: namespace parameter syntax

[Oliver Kiddle]

Bart Schaefer wrote:
> Assignment should be possible.
>
> % .foo.bar=something

There must be an off-by-one error in a check:

  a.b=c
  zsh: command not found: a.b=c

But it does work for anything longer than one character in the first
component such as foo.bar. I was confused when this didn't work for me
this morning because I was sure I had done some tests of assignments.

> Yes, I've been working on doc and tests.  One difference that I want
> to add is that parameters starting with "." are omitted from the
> default output of "set" and "typeset" (with no arguments).

Yes, that would be good.

> Given zsh dynamic scoping, it works reasonably well to have ZLE
> variables behave like locals -- you can't access them outside a shell
> function (widget) anyway.  If there were controls we wanted to expose
> higher up -- such as, perhaps, fiddling with keymaps without having to
> invoke "zle -K" or "bindkey -[eva]") then yes, a ".zle" namespace
> could be handy.

I was thinking of things like zle_bracketed_paste, zle_highlight and
ZLE_RPROMPT_INDENT which it'd be nice to have more hidden.

Oliver

Re: [PATCH 1/3] Extended ksh compatibility: namespace parameter syntax

[Bart Schaefer]

On Sun, Mar 5, 2023 at 2:25 PM Oliver Kiddle <opk@zsh.org> wrote:
>
> There must be an off-by-one error in a check:
>
>   a.b=c
>   zsh: command not found: a.b=c

Ah, thanks.  It was assuming there would be a leading ".".

> I was thinking of things like zle_bracketed_paste, zle_highlight and
> ZLE_RPROMPT_INDENT which it'd be nice to have more hidden.

Hmm, but ... if we change these to .zle.bracketed_paste etc., at what
point to deprecate the current names?