RC_EXPAND_PARAM final patch

RC_EXPAND_PARAM final patch

[Zoltan Hidvegi]

Afther applying the included patch:

% a=(a b)
% x=(x y)
% echo ${x}1{a,b}2${x}
x y1a2x y1b2x y
% echo ${x}1${^a}2${x}
x y1a2x y1b2x y
% echo ${^x}1${a}2${x} 
x1a y1a b2x y
% echo ${^x}1${a}2${^x}
x1a y1a b2x b2y
% echo ${x}1{a,b}2${^x}
x y1a2x y1b2x y1a2y y1b2y
% echo ${x}1${^a}2${^x}
x y1a2x y1a2y y1b2x y1b2y
% echo ${^x}1{a,b}2${x} 
x1a2x x1b2x y1a2x y1b2x y
% echo ${^x}1${^a}2${x} 
x1a2x x1b2x y1a2x y1b2x y

This is consistent, predictable, and the implementation is simple.  The
patch applies to original zsh-3.0.4 and zsh-3.1.2.  If you have already
applied my previous two-line fix to subst.c, you will have to reverse
that patch before applying this.  It is enough to recompile utils.o and
subst.o.

There is still one point where the behaviour might be surprising: that's
when the (e) substitution flag is used:

% echo ${x}1${(e)^a}2${x}
x y1a2x y y1b2x y

With the (e) flag, the behaviour is the same as it was with
zsh-2.6-beta16 and earlier.  This is related to this behaviour:

% integer i=0
% foo=('$[i++]' '$[i++]')
% echo a${(e)^foo}b${(e)foo} 
a0b1 2 a3b4 5

Here a${(e)^foo}b${(e)foo} is substituted to two identical
a$[i++]b${(e)foo} arguments, and substitution for these arguments is
started from the beginning.  This is exatly the zsh-2.6-beta16 behaviour.
This can be (or maybe, should be) changed, but that would require much
more complicated changes.  Since most people probably do not even know
about the (e) flag, this would not affect too many scripts.  The (e) flag
can be used to get the value of a variable named by an other variable.
For example

% foo=ZSH_VERSION
% echo ${(e):-\$$foo}
3.1.2

Zoltan


*** Src/zsh.h	1997/06/30 04:33:16	3.1.3.1
--- Src/zsh.h	1997/05/04 05:18:36
***************
*** 69,74 ****
--- 69,80 ----
  # define STOUC(X)	((unsigned char)(X))
  #endif
  
+ /* Meta together with the character following Meta denotes the character *
+  * which is the exclusive or of 32 and the character following Meta.     *
+  * This is used to represent characters which otherwise has special      *
+  * meaning for zsh.  These are the characters for which the imeta() test *
+  * is true: the null character, and the characters from Meta to Marker.  */
+ 
  #define Meta		((char) 0x83)
  
  /* Note that the fourth character in DEFAULT_IFS is Meta *
***************
*** 103,108 ****
--- 109,117 ----
  #define Nularg		((char) 0x9b)
  
  #define INULL(x)	(((x) & 0xfc) == 0x98)
+ 
+ /* Marker used in paramsubst for rc_expand_param */
+ #define Marker		((char) 0x9c)
  
  /* chars that need to be quoted if meant literally */
  
*** Src/utils.c	1997/07/13 05:24:42	3.1.3.2
--- Src/utils.c	1997/08/02 04:36:54
***************
*** 2282,2288 ****
      typtab['\t'] |= IBLANK | INBLANK;
      typtab['\n'] |= INBLANK;
      typtab['\0'] |= IMETA;
!     typtab[STOUC(Meta)] |= IMETA;
      for (t0 = (int)STOUC(Pound); t0 <= (int)STOUC(Nularg); t0++)
  	typtab[t0] |= ITOK | IMETA;
      for (s = ifs ? ifs : DEFAULT_IFS; *s; s++) {
--- 2282,2289 ----
      typtab['\t'] |= IBLANK | INBLANK;
      typtab['\n'] |= INBLANK;
      typtab['\0'] |= IMETA;
!     typtab[STOUC(Meta)  ] |= IMETA;
!     typtab[STOUC(Marker)] |= IMETA;
      for (t0 = (int)STOUC(Pound); t0 <= (int)STOUC(Nularg); t0++)
  	typtab[t0] |= ITOK | IMETA;
      for (s = ifs ? ifs : DEFAULT_IFS; *s; s++) {
*** Src/subst.c	1997/06/17 05:53:19	3.1.3.2
--- Src/subst.c	1997/08/02 06:05:47
***************
*** 1366,1377 ****
  	int i;
  	LinkNode on = n;
  
! 	if (!aval[0]) {
! 	    if (plan9) {
! 		uremnode(l, n);
! 		*str = aptr;
! 		return n;
! 	    }
  	    if (aptr > (char *) getdata(n) &&
  		aptr[-1] == Dnull && *s == Dnull)
  		*--aptr = '\0', s++;
--- 1366,1372 ----
  	int i;
  	LinkNode on = n;
  
! 	if (!aval[0] && !plan9) {
  	    if (aptr > (char *) getdata(n) &&
  		aptr[-1] == Dnull && *s == Dnull)
  		*--aptr = '\0', s++;
***************
*** 1398,1433 ****
  
  	if (plan9) {
  	    LinkList tl = newlinklist();
  
  	    addlinknode(tl, s);
  	    if (!eval && !stringsubst(tl, firstnode(tl), ssub))
  		return NULL;
  	    *str = aptr;
! 	    if (empty(tl)) {
! 		uremnode(l, n);
! 		return n;
! 	    }
! 	    i = 0;
! 	    while (aval[i]) {
! 		LinkNode tn;
! 
! 		x = aval[i++];
  		if (prenum || postnum)
  		    x = dopadding(x, prenum, postnum, preone, postone,
  				  premul, postmul);
  		if (eval && parsestr(x))
  		    return NULL;
  		xlen = strlen(x);
! 		for (tn = firstnode(tl); tn; incnode(tn)) {
! 		    strcatsub(&y, ostr, aptr, x, xlen,
! 			      (char *) getdata(tn), globsubst);
  		    if (qt && !*y && isarr != 2)
  			y = dupstring(nulstring);
! 		    if (i == 1)
! 			setdata(n, (void *) y);
  		    else
  			insertlinknode(l, n, (void *) y), incnode(n);
  		}
  	    }
  	} else {
  	    x = aval[0];
--- 1393,1439 ----
  
  	if (plan9) {
  	    LinkList tl = newlinklist();
+ 	    LinkNode tn;
  
+ 	    *--s = Marker;
  	    addlinknode(tl, s);
  	    if (!eval && !stringsubst(tl, firstnode(tl), ssub))
  		return NULL;
  	    *str = aptr;
! 	    tn = firstnode(tl);
! 	    while ((x = *aval++)) {
  		if (prenum || postnum)
  		    x = dopadding(x, prenum, postnum, preone, postone,
  				  premul, postmul);
  		if (eval && parsestr(x))
  		    return NULL;
  		xlen = strlen(x);
! 		for (tn = firstnode(tl);
! 		     tn && *(y = (char *) getdata(tn)) == Marker;
! 		     incnode(tn)) {
! 		    strcatsub(&y, ostr, aptr, x, xlen, y + 1, globsubst);
  		    if (qt && !*y && isarr != 2)
  			y = dupstring(nulstring);
! 		    if (plan9)
! 			setdata(n, (void *) y), plan9 = 0;
  		    else
  			insertlinknode(l, n, (void *) y), incnode(n);
  		}
+ 	    }
+ 	    for (; tn; incnode(tn)) {
+ 		y = (char *) getdata(tn);
+ 		if (*y == Marker)
+ 		    continue;
+ 		if (qt && !*y && isarr != 2)
+ 		    y = dupstring(nulstring);
+ 		if (plan9)
+ 		    setdata(n, (void *) y), plan9 = 0;
+ 		else
+ 		    insertlinknode(l, n, (void *) y), incnode(n);
+ 	    }
+ 	    if (plan9) {
+ 		uremnode(l, n);
+ 		return NULL;
  	    }
  	} else {
  	    x = aval[0];

Re: RC_EXPAND_PARAM final patch

[Bart Schaefer]

On Aug 2,  3:27am, Zoltan Hidvegi wrote:
} Subject: RC_EXPAND_PARAM final patch
}
} There is still one point where the behaviour might be surprising: that's
} when the (e) substitution flag is used:
} 
} % echo ${x}1${(e)^a}2${x}
} x y1a2x y y1b2x y

I get very strange behavior from this.  Vis:

zagzig<25> a=('$[i++]' '$[i++]' '$[i++]')
zagzig<26> i=0
zagzig<27> echo ${^x}1${a}2${^x}
x1$[i++] y1$[i++] z1$[i++] $[i++] $[i++]2x $[i++]2y $[i++]2z

OK so far, but:

zagzig<28> i=0
zagzig<29> echo ${^x}1${(e)a}2${^x}  
x10 y10 z10 $[i++] $[i++]2${^x}

Huh?  Why didn't it evaluate the second ${^x}, which is unquoted and at
the same (re)interpretation level as the first ${^x} ?  I would at least
have expected to get

x10 y10 z10 $[i++] $[i++]2x $[i++]2y $[i++]2z

-- 
Bart Schaefer                                 Brass Lantern Enterprises
http://www.well.com/user/barts              http://www.brasslantern.com

Re: RC_EXPAND_PARAM final patch

[Andrej Borsenkow]

On Sat, 2 Aug 1997, Zoltan Hidvegi wrote:

> 
> This is consistent, predictable, and the implementation is simple.
> 

Totally agreed. If I get it right, the patch does following:
  make substitutions (converting $^a into {a[1],a[2],...}
  word-split the result (it can result in lists in {...} being extended
    if SH_WORD_SPLIT is set)
  make brace expansion in each word (with implicit word splitting)

Correct? Could anybody with better English ;) add description of how ZSH
relly works to zshexpn(1)? 

> There is still one point where the behaviour might be surprising: that's
> when the (e) substitution flag is used:
> 
> % echo ${x}1${(e)^a}2${x}
> x y1a2x y y1b2x y
> 
> With the (e) flag, the behaviour is the same as it was with
> zsh-2.6-beta16 and earlier.  This is related to this behaviour:
> 
> % integer i=0
> % foo=('$[i++]' '$[i++]')
> % echo a${(e)^foo}b${(e)foo} 
> a0b1 2 a3b4 5
> 
> Here a${(e)^foo}b${(e)foo} is substituted to two identical
> a$[i++]b${(e)foo} arguments, and substitution for these arguments is
> started from the beginning.

It is the most surprising thing I ever seen in ZSH!!! _NO_ word in manuals
ever suggested such interpretation. I was always shure that 
${(e)a} just gets recursivly expanded, which is what I call consistent and
predictable ;-) Consider

a=('$[i++]' '$[i++]')
i=0
echo ${(e)^a}${(e)a}
01 2 34 5
i=0
echo ${(e)a}${(e)^a}
0 12 13

I hardly can call it `consistent'. If every expression were evaluated
first, we would get '02 12 3' and '0 12 13' which _is_ predictable - every
${...} is evaluated and combined using well-known rules.

Honestly speaking, how would you describe current behaviour in plain
words?

                              This is exatly the zsh-2.6-beta16 behaviour.
> This can be (or maybe, should be) changed, but that would require much
> more complicated changes. 

I understand, why it was implemented this way, but what's wrong with just
calling stringsubst for the result of ${(e)...} expansion?

                             Since most people probably do not even know
> about the (e) flag, this would not affect too many scripts.  

But if anybody would like to use it, he needs clear picture what to expect
(I still have to find out most things by trial-and-error :(

-------------------------------------------------------------------------
Andrej Borsenkow 		Fax:   +7 (095) 252 01 05
SNI ITS Moscow			Tel:   +7 (095) 252 13 88

NERV:  borsenkow.msk		E-Mail: borsenkow.msk@sni.de
-------------------------------------------------------------------------

Re: RC_EXPAND_PARAM final patch

[Peter Stephenson]

> Totally agreed. If I get it right, the patch does following:
>   make substitutions (converting $^a into {a[1],a[2],...}
>   word-split the result (it can result in lists in {...} being extended
>     if SH_WORD_SPLIT is set)
>   make brace expansion in each word (with implicit word splitting)
> 
> Correct? Could anybody with better English ;) add description of how ZSH
> relly works to zshexpn(1)? 

*** Doc/Zsh/expn.yo~	Mon Jun  2 08:37:08 1997
--- Doc/Zsh/expn.yo	Mon Aug  4 17:59:38 1997
***************
*** 259,264 ****
--- 259,271 ----
  is set to tt(LPAR())var(a b c)tt(RPAR()), are substituted with
  `var(fooabar foobbar foocbar)' instead of the default
  `var(fooa b cbar)'.
+ 
+ In practice, each such expansion is converted into the
+ equivalent list for brace expansion, i.e. tt(${^)var(var)tt(}) becomes
+ tt({$var[1],$var[2],...}), and is processed as described in em(brace
+ expansion) below.  If word splitting is also in effect the
+ tt($var[)var(N)tt(]) may themselves be split into different list
+ elements.
  )
  item(tt(${=)var(spec)tt(}))(
  pindex(SH_WORD_SPLIT, use of)


-- 
Peter Stephenson <pws@ifh.de>       Tel: +49 33762 77366
WWW:  http://www.ifh.de/~pws/       Fax: +49 33762 77413
Deutsches Elektronen-Synchrotron --- Institut fuer Hochenergiephysik Zeuthen
DESY-IfH, Platanenallee 6, 15738 Zeuthen, Germany.

Re: RC_EXPAND_PARAM final patch

[Zoltan T. Hidvegi]

Andrej Borsenkow wrote:
> Totally agreed. If I get it right, the patch does following:
>   make substitutions (converting $^a into {a[1],a[2],...}
>   word-split the result (it can result in lists in {...} being extended
>     if SH_WORD_SPLIT is set)
>   make brace expansion in each word (with implicit word splitting)

Yes.  The current mechanism is not exactly like this, but I'm goint to
change it and this is exactly I'm going to do.  Replace $^x with a brace
expansion list, and expand it later in together with other braces.
Currently there are several problems:

belgium ~ % echo {a,b}{x,y}
ax ay bx by

That fine.

belgium ~ % x=(x y)
belgium ~ % echo {a,b}$^x
ax bx ay by

Do you see the difference?  This is going to change.  The other problem:

belgium ~ % echo {$x,z}a
{x y,z}a

It is probably not what you would expect, although ksh behaves this way.
I'm goint to change it so that the result will be 3 words:

x ya za

But this would mean an other change: ${x}{a,b} will expand to 4 words,

x ya x yb

instead of the current x ya yb.  The current behaviour is the way ksh
behaves, and the planned new one is similar to bash (you can use $* and $@
to study the behaviour of arrays in bash and ksh).

> > Here a${(e)^foo}b${(e)foo} is substituted to two identical
> > a$[i++]b${(e)foo} arguments, and substitution for these arguments is
> > started from the beginning.
>
> It is the most surprising thing I ever seen in ZSH!!! _NO_ word in manuals
> ever suggested such interpretation. I was always shure that
> ${(e)a} just gets recursivly expanded, which is what I call consistent and

The (e) stuff is completely broken, this will be fixed and it'll work like
you'd expect.

Zoltan

Re: RC_EXPAND_PARAM final patch

[Bart Schaefer]

On Aug 4,  1:21pm,  (Zoltan T. Hidvegi) wrote:
} Subject: Re: RC_EXPAND_PARAM final patch
}
} belgium ~ % echo {$x,z}a
} {x y,z}a
} 
} It is probably not what you would expect, although ksh behaves this way.

Actually, this is exactly what I'd expect.  The $x is inside the { },
so the entire value of $x should be treated as part of the brace
expansion.

} I'm goint to change it so that the result will be 3 words:
} 
} x ya za

What, then, is the result of `echo {w,$x,z}a' ?  I can't imagine that
anyone would expect that to be the same as `{w,x} {y,z}a'.  Who told
zsh to put the extra } { in there?

} But this would mean an other change: ${x}{a,b} will expand to 4 words,
} 
} x ya x yb

This is fine, assuming that {p,q}${x}{a,b} expands to 8 words

px ya qx ya px yb qx yb

That's how I've thought it should work all along.

-- 
Bart Schaefer                                 Brass Lantern Enterprises
http://www.well.com/user/barts              http://www.brasslantern.com

Re: RC_EXPAND_PARAM final patch

[Zoltan T. Hidvegi]

Bart Schaefer wrote:
> On Aug 4,  1:21pm,  (Zoltan T. Hidvegi) wrote:
> } Subject: Re: RC_EXPAND_PARAM final patch
> }
> } belgium ~ % echo {$x,z}a
> } {x y,z}a
> }
> } It is probably not what you would expect, although ksh behaves this way.
>
> Actually, this is exactly what I'd expect.  The $x is inside the { },
> so the entire value of $x should be treated as part of the brace
> expansion.

I do not understand you.  Yes, I agree, that the entire value of $x should
be treated as part of the brace expansion, but this is not what happens
right now, as the result is two words, `{x' and `y,z}a', with braces
appearing in the result and not used for brace expansion.  I thought that
this is not what you expect.  What I would expect is 3 word result,

x ya za

> What, then, is the result of `echo {w,$x,z}a' ?  I can't imagine that
> anyone would expect that to be the same as `{w,x} {y,z}a'.  Who told
> zsh to put the extra } { in there?

Noone.  The result will be 4 words,

wa x ya za

This is different from the result of {w,x} {y,z}a which is w x ya za.  Or
do you have some more reasonable alternative?

> } But this would mean an other change: ${x}{a,b} will expand to 4 words,
> }
> } x ya x yb
>
> This is fine, assuming that {p,q}${x}{a,b} expands to 8 words
>
> px ya qx ya px yb qx yb

Almost, but the order is different:

px ya px yb qx ya qx yb

Zoltan

Re: RC_EXPAND_PARAM final patch

[Bart Schaefer]

On Aug 4,  2:51pm,  (Zoltan T. Hidvegi) wrote:
} Subject: Re: RC_EXPAND_PARAM final patch
}
} Bart Schaefer wrote:
} > On Aug 4,  1:21pm,  (Zoltan T. Hidvegi) wrote:
} > } Subject: Re: RC_EXPAND_PARAM final patch
} > }
} > } belgium ~ % echo {$x,z}a
} > } {x y,z}a
} > }
} > } It is probably not what you would expect, although ksh behaves this way.
} >
} > Actually, this is exactly what I'd expect.
} 
} I do not understand you.

Ah, no, *I* did not understand -you-.  I didn't notice that `{x y,z}a'
was meant to be read as literal output from `echo'.  Sorry; I withdraw
my remarks.

-- 
Bart Schaefer                                 Brass Lantern Enterprises
http://www.well.com/user/barts              http://www.brasslantern.com

Re: RC_EXPAND_PARAM final patch

[Andrej Borsenkow]

On Mon, 4 Aug 1997 hzoli@VNET.IBM.COM wrote:

> 
> But this would mean an other change: ${x}{a,b} will expand to 4 words,
> 
> x ya x yb
> 
> instead of the current x ya yb.  The current behaviour is the way ksh
> behaves, and the planned new one is similar to bash (you can use $* and $@
> to study the behaviour of arrays in bash and ksh).
> 

Currently (3.1.2 + RC_EXPAND_PARAM final patch):

% a=(a "b ")
% x=(x "y z")
% unsetopt shwordsplit
% args ${^a}1
"a1" "b 1"
% args ${^x}1
"x1" "y z1"

It's O.K., but ...
% setopt shwordsplit
% args ${^a}1
"a1" "b1" "1"
% args ${^x}1
"x1" "y1" "z1"     ????

Shouldn't it be  "x1" "y" "z1"  ?

-------------------------------------------------------------------------
Andrej Borsenkow 		Fax:   +7 (095) 252 01 05
SNI ITS Moscow			Tel:   +7 (095) 252 13 88

NERV:  borsenkow.msk		E-Mail: borsenkow.msk@sni.de
-------------------------------------------------------------------------

Re: RC_EXPAND_PARAM final patch

[Zoltan T. Hidvegi]

Andrej Borsenkow wrote:
> % a=(a "b ")
> % x=(x "y z")
[...]
> % setopt shwordsplit
> % args ${^a}1
> "a1" "b1" "1"
> % args ${^x}1
> "x1" "y1" "z1"     ????
>
> Shouldn't it be  "x1" "y" "z1"  ?

Why? ${^x} in itself expands to 3 words, x y z, and that's combined with
`1'.  With sh_word_split, a word with spaces acts like an array.

The first example is a bit more hard to understand.  Here ${^a} expands to
3 words too, but the last word is a null word, which is removeded from the
argument list, unless it is contacenated with something real.  That's why
${^a}1 expands to a1 b1 1, just immagine an invisible zero-length word
before the last 1.

Zoltan