PATCH: expanding parameters like echo/print builtins

PATCH: expanding parameters like echo/print builtins

[Mikael Magnusson]

Is this already possible? I had an array with a lot of \u1234 escapes
in it, and couldn't find a way to do it easily. I came up with this:
${arr:gs/(#b)'\\\\u(????)/${(#):-$(( 16#$match[1] ))}'}
but it's a bit awkward to type.

The following patch adds the (g::) parameter flag which calls the
getkeystring function, and allows setting the _EMACS, _OCTAL_ESC and
_CTRL flags.

---
 Doc/Zsh/expn.yo |    8 ++++++++
 Src/subst.c     |   55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 0 deletions(-)

diff --git a/Doc/Zsh/expn.yo b/Doc/Zsh/expn.yo
index 2c3a571..2beebd7 100644
--- a/Doc/Zsh/expn.yo
+++ b/Doc/Zsh/expn.yo
@@ -863,6 +863,14 @@ item(tt(F))(
 Join the words of arrays together using newline as a separator.
 This is a shorthand for `tt(pj:\n:)'.
 )
+item(tt(g:opts:))(
+Expand words like the echo builtin.  With the tt(o) option, octal escapes
+don't take a leading zero.  With the tt(c) option, sequences like `tt(^X)'
+are interpreted.  With the tt(e) option, interprets `tt(\M-t)' and similar
+sequences like the print builtin.  With both tt(o) and tt(e) options, behaves
+like the print builtin with the exception of the next sentence.   In none
+of these modes is `tt(\c)' interpreted.
+)
 item(tt(i))(
 Sort case-insensitively.  May be combined with `tt(n)' or `tt(O)'.
 )
diff --git a/Src/subst.c b/Src/subst.c
index 617986c..a2af01b 100644
--- a/Src/subst.c
+++ b/Src/subst.c
@@ -1607,6 +1607,10 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
      */	
     int presc = 0;
     /*
+     * The (g) flag.  Expand words with various GETKEY_ flags.
+     */
+    int getkeys = -1;
+    /*
      * The (@) flag; interacts obscurely with qt and isarr.
      * This is one of the things that decides whether multsub
      * will produce an array, but in an extremely indirect fashion.
@@ -1932,6 +1936,36 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 		    presc++;
 		    break;
 
+		case 'z':
+		    t = get_strarg(++s, &arglen);
+		    if (getkeys < 0)
+			getkeys = 0;
+		    if (*t) {
+			sav = *t;
+			*t = 0;
+			while (*++s) {
+			    switch (*s) {
+			    case 'e':
+				getkeys |= GETKEY_EMACS;
+				break;
+			    case 'o':
+				getkeys |= GETKEY_OCTAL_ESC;
+				break;
+			    case 'c':
+				getkeys |= GETKEY_CTRL;
+				break;
+
+			    default:
+				*t = sav;
+				goto flagerr;
+			    }
+			}
+			*t = sav;
+			s = t + arglen - 1;
+		    } else
+			goto flagerr;
+                    break;
+
 		case 'z':
 		    shsplit = LEXFLAGS_ACTIVE;
 		    break;
@@ -3141,6 +3175,27 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 	opts[PROMPTPERCENT] = opp;
     }
     /*
+     * Perform GETKEY expansion
+     */
+    if (getkeys >= 0) {
+	if (isarr) {
+	    char **ap;
+
+	    if (!copied)
+		aval = arrdup(aval), copied = 1;
+	    ap = aval;
+	    for (; *ap; ap++) {
+		int len;
+		*ap = getkeystring(*ap, &len, getkeys, NULL);
+	    }
+	} else {
+	    int len;
+	    if (!copied)
+		val = dupstring(val), copied = 1;
+	    val = getkeystring(val, &len, getkeys, NULL);
+	}
+    }
+    /*
      * One of the possible set of quotes to apply, depending on
      * the repetitions of the (q) flag.
      */
-- 
1.7.4-rc1

Re: PATCH: expanding parameters like echo/print builtins

[Bart Schaefer]

On May 11,  3:02pm, Mikael Magnusson wrote:
} Subject: PATCH: expanding parameters like echo/print builtins
}
} Is this already possible? I had an array with a lot of \u1234 escapes
} in it, and couldn't find a way to do it easily.

I'd probably have done it with ${(%)arr:gs/%/%%} ... but I don't have
any objection to the (g) flag except to wonder if another choice for
the letter (g) might be better.  The fact that it mnemes to GETKEY_
isn't going to be very helpful to most users.

But I don't have the list of available letters handy. :-)

Re: PATCH: expanding parameters like echo/print builtins

[Mikael Magnusson]

On 11 May 2011 17:03, Bart Schaefer <schaefer@brasslantern.com> wrote:
> On May 11,  3:02pm, Mikael Magnusson wrote:
> } Subject: PATCH: expanding parameters like echo/print builtins
> }
> } Is this already possible? I had an array with a lot of \u1234 escapes
> } in it, and couldn't find a way to do it easily.
>
> I'd probably have done it with ${(%)arr:gs/%/%%}

Yeah, me too, except (%) doesn't expand \u304c or \0123, or any of the
other stuff that (g) expands :).

> [...] I don't have
> any objection to the (g) flag except to wonder if another choice for
> the letter (g) might be better.  The fact that it mnemes to GETKEY_
> isn't going to be very helpful to most users.
>
> But I don't have the list of available letters handy. :-)

I looked at the list of letters and the first free one was b, so i
picked that, then when i got as far as discover that the function is
called getkeystring and noticed g was also free, i went with that. e
is taken (for echo), and p which does the same for subsequent flags is
taken, and P is also taken. If anyone has a better mnemonic in mind,
feel free to suggest it. The following characters are taken:
#%@AacCDefFikLnoOPqQtuUvVwWXz0p~jlmrsZ_lBEMNR

-- 
Mikael Magnusson

Re: PATCH: expanding parameters like echo/print builtins

[Bart Schaefer]

On May 11,  5:14pm, Mikael Magnusson wrote:
} Subject: Re: PATCH: expanding parameters like echo/print builtins
}
} On 11 May 2011 17:03, Bart Schaefer <schaefer@brasslantern.com> wrote:
} >
} > I'd probably have done it with ${(%)arr:gs/%/%%}
} 
} Yeah, me too, except (%) doesn't expand \u304c or \0123, or any of the
} other stuff that (g) expands :).

Hrm.  What was I thinking of?  Obviously hadn't eaten breakfast yet.
 
} I looked at the list of letters and the first free one was b, so i
} picked that, then when i got as far as discover that the function is
} called getkeystring and noticed g was also free, i went with that.
} [...] The following characters are taken:
} #%@AacCDefFikLnoOPqQtuUvVwWXz0p~jlmrsZ_lBEMNR

So ...

x='#%@AacCDefFikLnoOPqQtuUvVwWXz0p~jlmrsZ_lBEMNR'
y=( {$[##a]..$[##z]} {$[##A]..$[##Z]} )
print ${${(#)y}:#[$x]}

b d g h x y G H I J K S T Y

I guess (g) is as good as anything, unless someone prefers to save the
lowercase letters and go with (G).

Re: PATCH: expanding parameters like echo/print builtins

[Mikael Magnusson]

On 11 May 2011 18:03, Bart Schaefer <schaefer@brasslantern.com> wrote:
> On May 11,  5:14pm, Mikael Magnusson wrote:
> } Subject: Re: PATCH: expanding parameters like echo/print builtins
> }
> } On 11 May 2011 17:03, Bart Schaefer <schaefer@brasslantern.com> wrote:
> } >
> } > I'd probably have done it with ${(%)arr:gs/%/%%}
> }
> } Yeah, me too, except (%) doesn't expand \u304c or \0123, or any of the
> } other stuff that (g) expands :).
>
> Hrm.  What was I thinking of?  Obviously hadn't eaten breakfast yet.
>
> } I looked at the list of letters and the first free one was b, so i
> } picked that, then when i got as far as discover that the function is
> } called getkeystring and noticed g was also free, i went with that.
> } [...] The following characters are taken:
> } #%@AacCDefFikLnoOPqQtuUvVwWXz0p~jlmrsZ_lBEMNR
>
> So ...
>
> x='#%@AacCDefFikLnoOPqQtuUvVwWXz0p~jlmrsZ_lBEMNR'
> y=( {$[##a]..$[##z]} {$[##A]..$[##Z]} )
> print ${${(#)y}:#[$x]}
>
> b d g h x y G H I J K S T Y
>
> I guess (g) is as good as anything, unless someone prefers to save the
> lowercase letters and go with (G).

Oops, I typed l twice, the second one was supposed to be I, so the list is
b d g h x y G H J K S T Y

Do I need to clarify in the docs that you write g:: to give neither of
the three flags? Just g by itself is an error, like for the other
flags that take options. There's not really any good reason for this
other than it is much easier to reuse the code used by s and j (and
Z). But for s/j you want to be able to specify any character, whereas
for this flag there are a limited number of options, so : could be
obligatory for the field separator, then you could say just (g). Does
anyone care about this? :) (It would also mean we can't use : as a new
flag though).

-- 
Mikael Magnusson

Re: PATCH: expanding parameters like echo/print builtins

[Bart Schaefer]

On May 11,  6:22pm, Mikael Magnusson wrote:
} Subject: Re: PATCH: expanding parameters like echo/print builtins
}
} Do I need to clarify in the docs that you write g:: to give neither of
} the three flags? Just g by itself is an error, like for the other
} flags that take options.

I don't think that's a problem, but yes it should be doc'd.

BTW, the construct "... the exception of the next sentence.  In none
of these modes ..." is a little clunky; probably OK to just say "...
except that in none of these modes ..."

} for this flag there are a limited number of options, so : could be
} obligatory for the field separator, then you could say just (g). Does
} anyone care about this? :) (It would also mean we can't use : as a new
} flag though).

I don't like the idea of limiting the delimiter to ":".

RE Oliver's suggestion of a (\) flag -- I considered that too but it
is not clear whether the backslash might be removed in some contexts
before zsh gets around to parsing the flags.

I suppose we could use (/) (to mean "reverse the backslashes"?) but ...

Re: PATCH: expanding parameters like echo/print builtins

[Mikael Magnusson]

On 11 May 2011 18:48, Bart Schaefer <schaefer@brasslantern.com> wrote:
> On May 11,  6:22pm, Mikael Magnusson wrote:
> } Subject: Re: PATCH: expanding parameters like echo/print builtins
> }
> } Do I need to clarify in the docs that you write g:: to give neither of
> } the three flags? Just g by itself is an error, like for the other
> } flags that take options.
>
> I don't think that's a problem, but yes it should be doc'd.
>
> BTW, the construct "... the exception of the next sentence.  In none
> of these modes ..." is a little clunky; probably OK to just say "...
> except that in none of these modes ..."

How's this?

item(tt(g:opts:))(
Expand words like the echo builtin when no options are given (tt(g::)).
With the tt(o) option, octal escapes don't take a leading zero.  With the
tt(c) option, sequences like `tt(^X)' are interpreted.  With the tt(e)
option, interprets `tt(\M-t)' and similar sequences like the print
builtin.  With both of the tt(o) and tt(e) options, behaves like the
print builtin except that in none of these modes is `tt(\c)' interpreted.
)

And this for s,

 separator var(string).  Note that a var(string) of two or more
 characters means that all of them must match in sequence; this differs from
 the treatment of two or more characters in the tt(IFS) parameter.
-See also the tt(=) flag and the tt(SH_WORD_SPLIT) option.
+See also the tt(=) flag and the tt(SH_WORD_SPLIT) option.  An empty
+string may also be given in which case every character will be a separate
+element.


> } for this flag there are a limited number of options, so : could be
> } obligatory for the field separator, then you could say just (g). Does
> } anyone care about this? :) (It would also mean we can't use : as a new
> } flag though).
>
> I don't like the idea of limiting the delimiter to ":".

Okay, I guess I don't really either.

> RE Oliver's suggestion of a (\) flag -- I considered that too but it
> is not clear whether the backslash might be removed in some contexts
> before zsh gets around to parsing the flags.
>
> I suppose we could use (/) (to mean "reverse the backslashes"?) but ...

${(/::) looks like an alien girl waving goodbye. :)

-- 
Mikael Magnusson

Re: PATCH: expanding parameters like echo/print builtins

[Oliver Kiddle]

Bart wrote:
> I'd probably have done it with ${(%)arr:gs/%/%%} ... but I don't have
> any objection to the (g) flag except to wonder if another choice for
> the letter (g) might be better.  The fact that it mnemes to GETKEY_
> isn't going to be very helpful to most users.

Given that % is used for the (%) flag it might be logical to use \ here.
Or would it just be too confusing. It'd probably need quoting, right?

As for existing solutions, ${(Q)${(qqqq)var}//\\\\/\\} might do it.

Oliver

Re: PATCH: expanding parameters like echo/print builtins

[Peter Stephenson]

On Wed, 11 May 2011 15:02:51 +0200
Mikael Magnusson <mikachu@gmail.com> wrote:
>       * The (@) flag; interacts obscurely with qt and isarr.
>       * This is one of the things that decides whether multsub
>       * will produce an array, but in an extremely indirect fashion.
> @@ -1932,6 +1936,36 @@ paramsubst(LinkList l, LinkNode n, char **str,
> int qt, int ssub) presc++;
>  		    break;
>  
> +		case 'z':

This might want to be 'g'.

-- 
Peter Stephenson <pws@csr.com>            Software Engineer
Tel: +44 (0)1223 692070                   Cambridge Silicon Radio Limited
Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, UK


Member of the CSR plc group of companies. CSR plc registered in England and Wales, registered number 4187346, registered office Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, United Kingdom

Re: PATCH: expanding parameters like echo/print builtins

[Mikael Magnusson]

On 11 May 2011 18:21, Peter Stephenson <Peter.Stephenson@csr.com> wrote:
> On Wed, 11 May 2011 15:02:51 +0200
> Mikael Magnusson <mikachu@gmail.com> wrote:
>>       * The (@) flag; interacts obscurely with qt and isarr.
>>       * This is one of the things that decides whether multsub
>>       * will produce an array, but in an extremely indirect fashion.
>> @@ -1932,6 +1936,36 @@ paramsubst(LinkList l, LinkNode n, char **str,
>> int qt, int ssub) presc++;
>>                   break;
>>
>> +             case 'z':
>
> This might want to be 'g'.

Indeed, I noticed this line had spaces instead of tabs, so while
editing the patch file I copied the z line from the context and added
a +, taking care to change the z to a g as I did so, but somehow
didn't actually. So if this made you worry that I hadn't tested the
patch, fear not :). Speaking of testing, should I add tests for this
too? The c and e flags will have tendencies to output unprintable
characters, can I write them in the expected test output somehow? Ie,
it is probably not a good idea to write
>${(g:c:):-^X}
when testing that feature, since it'll break in the same way in both
places. It is probably not likely that c specifically would break and
everything else work, so I would personally be happy with just testing
g:: and maybe g:o: too, since that's easy enough. In fact...

--- a/Test/D04parameter.ztst
+++ b/Test/D04parameter.ztst
@@ -298,6 +298,14 @@
 >Howzat
 >usay

+  foo='\u65\123'
+  print -r ${(g:o:)foo}
+  foo='\u65\0123'
+  print -r ${(g::)foo}
+0:${(g)...}
+>eS
+>eS
+
   foo='I'\''m nearly out of my mind with tedium'
   bar=foo
   print ${(P)bar}


-- 
Mikael Magnusson

Re: PATCH: expanding parameters like echo/print builtins

[Peter Stephenson]

On Wed, 11 May 2011 18:38:46 +0200
Mikael Magnusson <mikachu@gmail.com> wrote:
> The c and e flags will have tendencies to output unprintable
> characters, can I write them in the expected test output somehow?

The way to do this is to capture the output via a pipe or $(...) or
whatever and then make that output readable by using ${(q)...} or
something such.  There will be examples somewhere.  This does rely on
the shell getting the reverse conversion right, but we test the features
its relying on elsewhere so in theory we shouldn't get errors cancelling
each other out.  (I'm loath to rely on od, and certainly xxd isn't
standard enough.)

-- 
Peter Stephenson <pws@csr.com>            Software Engineer
Tel: +44 (0)1223 692070                   Cambridge Silicon Radio Limited
Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, UK


Member of the CSR plc group of companies. CSR plc registered in England and Wales, registered number 4187346, registered office Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, United Kingdom

Re: PATCH: expanding parameters like echo/print builtins

[Mikael Magnusson]

On 11 May 2011 19:07, Peter Stephenson <Peter.Stephenson@csr.com> wrote:
> On Wed, 11 May 2011 18:38:46 +0200
> Mikael Magnusson <mikachu@gmail.com> wrote:
>> The c and e flags will have tendencies to output unprintable
>> characters, can I write them in the expected test output somehow?
>
> The way to do this is to capture the output via a pipe or $(...) or
> whatever and then make that output readable by using ${(q)...} or
> something such.  There will be examples somewhere.  This does rely on
> the shell getting the reverse conversion right, but we test the features
> its relying on elsewhere so in theory we shouldn't get errors cancelling
> each other out.  (I'm loath to rely on od, and certainly xxd isn't
> standard enough.)

Hm, or could I use ${(V)${(g:c:):-'^X'}}? (actually putting V in the
same parens work, but the order isn't documented). (Should the order
parameter expansion flags are applied in be documented somewhere?)

  foo='\u65\123'
  print -r ${(g:o:)foo}
  foo='\u65\0123'
  print -r ${(g::)foo}
  foo='\u65^X'
  print -r ${(V)${(g:c:)foo}}
  foo='\u65\C-x\M-a'
  print -r ${(V)${(g:e:)foo}}
  foo='\u65\123\C-x'
  print -r ${(V)${(g:eo:)foo}}
0:${(g)...}
>eS
>eS
>e^X
>e^X\M-a
>eS^X

-- 
Mikael Magnusson

Re: PATCH: expanding parameters like echo/print builtins

[Peter Stephenson]

On Wed, 11 May 2011 19:19:51 +0200
Mikael Magnusson <mikachu@gmail.com> wrote:
> (Should the order
> parameter expansion flags are applied in be documented somewhere?)

It's documented in *immense* detail in the subsection "Rules" under
Parameter Expansion, but even the 23 existing rules don't cover
everything.  Feel free to add another one.  Actually, (V) is covered, at
rule 15, so maybe you just missed it.

-- 
Peter Stephenson <pws@csr.com>            Software Engineer
Tel: +44 (0)1223 692070                   Cambridge Silicon Radio Limited
Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, UK


Member of the CSR plc group of companies. CSR plc registered in England and Wales, registered number 4187346, registered office Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, United Kingdom

Re: PATCH: expanding parameters like echo/print builtins

[Mikael Magnusson]

On 11 May 2011 19:26, Peter Stephenson <Peter.Stephenson@csr.com> wrote:
> On Wed, 11 May 2011 19:19:51 +0200
> Mikael Magnusson <mikachu@gmail.com> wrote:
>> (Should the order
>> parameter expansion flags are applied in be documented somewhere?)
>
> It's documented in *immense* detail in the subsection "Rules" under
> Parameter Expansion, but even the 23 existing rules don't cover
> everything.  Feel free to add another one.  Actually, (V) is covered, at
> rule 15, so maybe you just missed it.

Actually this is a case of me being an idiot. I remember scrolling
past it, but I always thought I was scrolling past the list given in
the beginning of zshexpn giving the relation between
history/alias/process/etc substitutions.

-- 
Mikael Magnusson

Re: PATCH: expanding parameters like echo/print builtins

[Mikael Magnusson]

On 11 May 2011 19:26, Peter Stephenson <Peter.Stephenson@csr.com> wrote:
> On Wed, 11 May 2011 19:19:51 +0200
> Mikael Magnusson <mikachu@gmail.com> wrote:
>> (Should the order
>> parameter expansion flags are applied in be documented somewhere?)
>
> It's documented in *immense* detail in the subsection "Rules" under
> Parameter Expansion, but even the 23 existing rules don't cover
> everything.  Feel free to add another one.  Actually, (V) is covered, at
> rule 15, so maybe you just missed it.

What I meant was that the relation between V and g:: isn't documented,
but yeah, I did miss the whole list :). Here's a patch, minus the
renumbering of subsequent items. Any better names than echo-type
expansion? :)

--- a/Doc/Zsh/expn.yo
+++ b/Doc/Zsh/expn.yo
@@ -1305,6 +1305,9 @@ is applied.
 item(tt(12.) em(Prompt evaluation))(
 Any prompt-style formatting from the tt((%)) family of flags is applied.
 )
+item(tt(13.) em(Echo-type expansion))(
+Any expansion from the tt((g)) flag is applied.
+)
 item(tt(13.) em(Quote application))(
 Any quoting or unquoting using tt((q)) and tt((Q)) and related flags
 is applied.


-- 
Mikael Magnusson

Re: PATCH: expanding parameters like echo/print builtins

[Bart Schaefer]

On May 11,  7:43pm, Mikael Magnusson wrote:
} Subject: Re: PATCH: expanding parameters like echo/print builtins
}
} What I meant was that the relation between V and g:: isn't documented,
} but yeah, I did miss the whole list :). Here's a patch, minus the
} renumbering of subsequent items.

Renumbering the subsequent items is a nasty bit to leave out, because
there are cross-references among the items that always get forgotten;
I just fixed all of those up a few months ago.

It might be better just to add this to item 12 rather than introduce
a new number.

} Any better names than echo-type expansion? :)

Elsewhere these kinds of things are referred to as "escape sequences"
but then so are %-dohickeys for prompts.  Probably would have been
a good idea years ago to refer to the % forms as "expandos" or some
such word ...

But, given that they're all escapes, we could go with

item(tt(12.) em(Escape sequence replacement))(
First any prompt-style formatting from the tt((%)) family of flags is
applied, then any replacements from the tt((g)) flag are performed.
)

The (g) stuff usually involves making the string *shorter* so to call
it an "expansion" is a bit odd.  [Is there a potential conflict where
(%) inserts something that (g) then incorrectly replaces, if you use
both of them togher?]

-- 

Re: PATCH: expanding parameters like echo/print builtins

[Mikael Magnusson]

On 12 May 2011 12:10, Bart Schaefer <schaefer@brasslantern.com> wrote:
> On May 11,  7:43pm, Mikael Magnusson wrote:
> } Subject: Re: PATCH: expanding parameters like echo/print builtins
> }
> } What I meant was that the relation between V and g:: isn't documented,
> } but yeah, I did miss the whole list :). Here's a patch, minus the
> } renumbering of subsequent items.
>
> Renumbering the subsequent items is a nasty bit to leave out, because
> there are cross-references among the items that always get forgotten;
> I just fixed all of those up a few months ago.
>
> It might be better just to add this to item 12 rather than introduce
> a new number.

Ah, good point.

> } Any better names than echo-type expansion? :)
>
> Elsewhere these kinds of things are referred to as "escape sequences"
> but then so are %-dohickeys for prompts.  Probably would have been
> a good idea years ago to refer to the % forms as "expandos" or some
> such word ...
>
> But, given that they're all escapes, we could go with
>
> item(tt(12.) em(Escape sequence replacement))(
> First any prompt-style formatting from the tt((%)) family of flags is
> applied, then any replacements from the tt((g)) flag are performed.
> )
>
> The (g) stuff usually involves making the string *shorter* so to call
> it an "expansion" is a bit odd.

Also a good point, I can change it to "process escape sequences like
the echo builtin" also in the description of the g:: flag, that's what
the print description calls it.

> [Is there a potential conflict where
> (%) inserts something that (g) then incorrectly replaces, if you use
> both of them togher?]

Well, if (%) inserts something before (g) processes it, (g) will
replace it, but what would make it incorrect?
/tmp/\u3042% echo -E ${(g::%):-%~}
/tmp/あ

It is of course possible to move (g) somewhere else in the sequence if
wanted, I just figured it was closest logically to (%) and put it
there. (But if you don't like the order, all you have to do is
/tmp/\u3042% echo -E ${(%)${(g::):-%~}}
/tmp/\u3042
)

-- 
Mikael Magnusson

Re: PATCH: expanding parameters like echo/print builtins

[Bart Schaefer]

On May 12, 12:40pm, Mikael Magnusson wrote:
} Subject: Re: PATCH: expanding parameters like echo/print builtins
}
} On 12 May 2011 12:10, Bart Schaefer <schaefer@brasslantern.com> wrote:
} > [Is there a potential conflict where
} > (%) inserts something that (g) then incorrectly replaces, if you use
} > both of them togher?]
} 
} Well, if (%) inserts something before (g) processes it, (g) will
} replace it, but what would make it incorrect?

Specifically, are there any terminal control sequences that use a
backslash?  A quick grep of /etc/termcap seems to show that there
are some sequences (fs and ds usually, but some others) that use
a backslash.  To avoid those being interpreted as part of a (g)
escape, it might be best to have (g) occur before (%) unless the
user explicitly orders them the other way by nesting.

-- 

PATCH: Add g:: parameter expansion flag

[Mikael Magnusson]

Okay, how's this? I swapped the order so g happens first, and changed
the docs as discussed.

---
 Doc/Zsh/expn.yo        |   17 ++++++++++++--
 Src/subst.c            |   55 ++++++++++++++++++++++++++++++++++++++++++++++++
 Test/D04parameter.ztst |   17 ++++++++++++++
 3 files changed, 86 insertions(+), 3 deletions(-)

diff --git a/Doc/Zsh/expn.yo b/Doc/Zsh/expn.yo
index 1cad3d2..d3a08c5 100644
--- a/Doc/Zsh/expn.yo
+++ b/Doc/Zsh/expn.yo
@@ -863,6 +863,14 @@ item(tt(F))(
 Join the words of arrays together using newline as a separator.
 This is a shorthand for `tt(pj:\n:)'.
 )
+item(tt(g:opts:))(
+Process escape sequences like the echo builtin when no options are given
+(tt(g::)).  With the tt(o) option, octal escapes don't take a leading
+zero.  With the tt(c) option, sequences like `tt(^X)' are also processed.
+With the tt(e) option, processes `tt(\M-t)' and similar sequences like the
+print builtin.  With both of the tt(o) and tt(e) options, behaves like the
+print builtin except that in none of these modes is `tt(\c)' interpreted.
+)
 item(tt(i))(
 Sort case-insensitively.  May be combined with `tt(n)' or `tt(O)'.
 )
@@ -1108,7 +1116,9 @@ Force field splitting at the
 separator var(string).  Note that a var(string) of two or more
 characters means that all of them must match in sequence; this differs from
 the treatment of two or more characters in the tt(IFS) parameter.
-See also the tt(=) flag and the tt(SH_WORD_SPLIT) option.
+See also the tt(=) flag and the tt(SH_WORD_SPLIT) option.  An empty
+string may also be given in which case every character will be a separate
+element.
 
 For historical reasons, the usual behaviour that empty array elements
 are retained inside double quotes is disabled for arrays generated
@@ -1292,8 +1302,9 @@ item(tt(11.) em(Case modification))(
 Any case modification from one of the flags tt((L)), tt((U)) or tt((C))
 is applied.
 )
-item(tt(12.) em(Prompt evaluation))(
-Any prompt-style formatting from the tt((%)) family of flags is applied.
+item(tt(12.) em(Escape sequence replacement))(
+First any replacements from the tt((g)) flag are performed, then any
+prompt-style formatting from the tt((%)) family of flags is applied.
 )
 item(tt(13.) em(Quote application))(
 Any quoting or unquoting using tt((q)) and tt((Q)) and related flags
diff --git a/Src/subst.c b/Src/subst.c
index 5d7a8b4..7aa07b5 100644
--- a/Src/subst.c
+++ b/Src/subst.c
@@ -1607,6 +1607,10 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
      */	
     int presc = 0;
     /*
+     * The (g) flag.  Process escape sequences with various GETKEY_ flags.
+     */
+    int getkeys = -1;
+    /*
      * The (@) flag; interacts obscurely with qt and isarr.
      * This is one of the things that decides whether multsub
      * will produce an array, but in an extremely indirect fashion.
@@ -1932,6 +1936,36 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 		    presc++;
 		    break;
 
+		case 'g':
+		    t = get_strarg(++s, &arglen);
+		    if (getkeys < 0)
+			getkeys = 0;
+		    if (*t) {
+			sav = *t;
+			*t = 0;
+			while (*++s) {
+			    switch (*s) {
+			    case 'e':
+				getkeys |= GETKEY_EMACS;
+				break;
+			    case 'o':
+				getkeys |= GETKEY_OCTAL_ESC;
+				break;
+			    case 'c':
+				getkeys |= GETKEY_CTRL;
+				break;
+
+			    default:
+				*t = sav;
+				goto flagerr;
+			    }
+			}
+			*t = sav;
+			s = t + arglen - 1;
+		    } else
+			goto flagerr;
+                    break;
+
 		case 'z':
 		    shsplit = LEXFLAGS_ACTIVE;
 		    break;
@@ -3100,6 +3134,27 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 	}
     }
     /*
+     * Process echo- and print-style escape sequences.
+     */
+    if (getkeys >= 0) {
+	if (isarr) {
+	    char **ap;
+
+	    if (!copied)
+		aval = arrdup(aval), copied = 1;
+	    ap = aval;
+	    for (; *ap; ap++) {
+		int len;
+		*ap = getkeystring(*ap, &len, getkeys, NULL);
+	    }
+	} else {
+	    int len;
+	    if (!copied)
+		val = dupstring(val), copied = 1;
+	    val = getkeystring(val, &len, getkeys, NULL);
+	}
+    }
+    /*
      * Perform prompt-style modifications.
      */
     if (presc) {
diff --git a/Test/D04parameter.ztst b/Test/D04parameter.ztst
index b91caaa..7322872 100644
--- a/Test/D04parameter.ztst
+++ b/Test/D04parameter.ztst
@@ -298,6 +298,23 @@
 >Howzat
 >usay
 
+  foo='\u65\123'
+  print -r ${(g:o:)foo}
+  foo='\u65\0123'
+  print -r ${(g::)foo}
+  foo='\u65^X'
+  print -r ${(V)${(g:c:)foo}}
+  foo='\u65\C-x\M-a'
+  print -r ${(V)${(g:e:)foo}}
+  foo='\u65\123\C-x'
+  print -r ${(V)${(g:eo:)foo}}
+0:${(g)...}
+>eS
+>eS
+>e^X
+>e^X\M-a
+>eS^X
+
   foo='I'\''m nearly out of my mind with tedium'
   bar=foo
   print ${(P)bar}
-- 
1.7.4-rc1

Re: PATCH: Add g:: parameter expansion flag

[Mikael Magnusson]

On 12 May 2011 17:49, Mikael Magnusson <mikachu@gmail.com> wrote:
> Okay, how's this? I swapped the order so g happens first, and changed
> the docs as discussed.

Just as I was thinking about committing it, I found a bug. I have to
metafy the result of getkeystring().
% print -Rn ${(g:o:):-'\201\227\343\201\257'}|wc -c
zsh: command not found: は
1

Not exactly the intended result :).

So I am pretty sure I want to use META_HREALLOC, is that correct? Ie
	    if (!copied)
		val = dupstring(val), copied = 1;
	    val = getkeystring(val, &len, getkeys, NULL);
            val = metafy(val, len, META_HREALLOC);
(and same for the isarr case)

-- 
Mikael Magnusson

Re: PATCH: Add g:: parameter expansion flag

[Peter Stephenson]

On Thu, 12 May 2011 21:41:36 +0200
Mikael Magnusson <mikachu@gmail.com> wrote:
> On 12 May 2011 17:49, Mikael Magnusson <mikachu@gmail.com> wrote:
> > Okay, how's this? I swapped the order so g happens first, and
> > changed the docs as discussed.
> 
> Just as I was thinking about committing it, I found a bug. I have to
> metafy the result of getkeystring().
> % print -Rn ${(g:o:):-'\201\227\343\201\257'}|wc -c
> zsh: command not found: は
> 1
> 
> Not exactly the intended result :).
> 
> So I am pretty sure I want to use META_HREALLOC, is that correct? Ie
> 	    if (!copied)
> 		val = dupstring(val), copied = 1;
> 	    val = getkeystring(val, &len, getkeys, NULL);
>             val = metafy(val, len, META_HREALLOC);
> (and same for the isarr case)
> 

That would do, as it's what untok_and_escape() does, although
META_HEAPDUP would probably be OK at this point.  Reallocating what's
on the heap is a slightly strange thing to do, since the point of the
heap is to provide quick storage without the need to micromanage it; it
tends to be done when there's a long string on the heap that it would
be inefficient to keep duplicating, which isn't the case here.  If the
original heap chunk is surrounded by other allocations it has to
duplicate anyway, since the heap doesn't let you reuse memory (until
the whole heap is popped); you can't tell just be looking at the call
whether this will be the case.

-- 
Peter Stephenson <pws@csr.com>            Software Engineer
Tel: +44 (0)1223 692070                   Cambridge Silicon Radio Limited
Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, UK


Member of the CSR plc group of companies. CSR plc registered in England and Wales, registered number 4187346, registered office Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, United Kingdom

Re: PATCH: Add g:: parameter expansion flag

[Mikael Magnusson]

On 13 May 2011 10:54, Peter Stephenson <Peter.Stephenson@csr.com> wrote:
> On Thu, 12 May 2011 21:41:36 +0200
> Mikael Magnusson <mikachu@gmail.com> wrote:
>> On 12 May 2011 17:49, Mikael Magnusson <mikachu@gmail.com> wrote:
>> > Okay, how's this? I swapped the order so g happens first, and
>> > changed the docs as discussed.
>>
>> Just as I was thinking about committing it, I found a bug. I have to
>> metafy the result of getkeystring().
>> % print -Rn ${(g:o:):-'\201\227\343\201\257'}|wc -c
>> zsh: command not found: は
>> 1
>>
>> Not exactly the intended result :).
>>
>> So I am pretty sure I want to use META_HREALLOC, is that correct? Ie
>>           if (!copied)
>>               val = dupstring(val), copied = 1;
>>           val = getkeystring(val, &len, getkeys, NULL);
>>             val = metafy(val, len, META_HREALLOC);
>> (and same for the isarr case)
>>
>
> That would do, as it's what untok_and_escape() does, although
> META_HEAPDUP would probably be OK at this point.  Reallocating what's
> on the heap is a slightly strange thing to do, since the point of the
> heap is to provide quick storage without the need to micromanage it;

In that case, isn't USEHEAP better? It'll allocate on the heap if
there's anything to escape and otherwise do it in-place. Which is what
I thought HREALLOC would do, but I guess that one would do more stuff
to try and just grow. Won't it always succeed though, since the to-be
reallocated value was just allocated on the line before? (At least
when the heap isn't full).

> it
> tends to be done when there's a long string on the heap that it would
> be inefficient to keep duplicating, which isn't the case here.  If the
> original heap chunk is surrounded by other allocations it has to
> duplicate anyway, since the heap doesn't let you reuse memory (until
> the whole heap is popped); you can't tell just be looking at the call
> whether this will be the case.

Why can't the string be long? Not that I think anyone would really
notice anyway :).

-- 
Mikael Magnusson

Re: PATCH: Add g:: parameter expansion flag

[Peter Stephenson]

On Fri, 13 May 2011 11:51:13 +0200
Mikael Magnusson <mikachu@gmail.com> wrote:
> >> So I am pretty sure I want to use META_HREALLOC, is that correct?
> > That would do, as it's what untok_and_escape() does, although
> > META_HEAPDUP would probably be OK at this point.  Reallocating
> > what's on the heap is a slightly strange thing to do, since the
> > point of the heap is to provide quick storage without the need to
> > micromanage it;
> 
> In that case, isn't USEHEAP better? It'll allocate on the heap if
> there's anything to escape and otherwise do it in-place. Which is what
> I thought HREALLOC would do, but I guess that one would do more stuff
> to try and just grow. Won't it always succeed though, since the to-be
> reallocated value was just allocated on the line before? (At least
> when the heap isn't full).

USEHEAP is fine if you don't mind further modifications to the string
propagating back to the original.  In some places the string might not
be modifiable, or might refer to a permanently stored string, so you
wouldn't want to do that.  That may not be the case here; it needs
deciding case by case.

> > it
> > tends to be done when there's a long string on the heap that it
> > would be inefficient to keep duplicating, which isn't the case
> > here.  If the original heap chunk is surrounded by other
> > allocations it has to duplicate anyway, since the heap doesn't let
> > you reuse memory (until the whole heap is popped); you can't tell
> > just be looking at the call whether this will be the case.
> 
> Why can't the string be long? Not that I think anyone would really
> notice anyway :).

The real problem is the combination of a long string that's continually
being extended, typically before any other heap allocation has been
done; that's the case hrealloc() is best at handling.

-- 
Peter Stephenson <pws@csr.com>            Software Engineer
Tel: +44 (0)1223 692070                   Cambridge Silicon Radio Limited
Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, UK


Member of the CSR plc group of companies. CSR plc registered in England and Wales, registered number 4187346, registered office Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, United Kingdom

PATCH: Add g:: parameter expansion flag

[Mikael Magnusson]

On 13 May 2011 13:54, Peter Stephenson <Peter.Stephenson@csr.com> wrote:
> On Fri, 13 May 2011 11:51:13 +0200
> Mikael Magnusson <mikachu@gmail.com> wrote:
>> >> So I am pretty sure I want to use META_HREALLOC, is that correct?
>> > That would do, as it's what untok_and_escape() does, although
>> > META_HEAPDUP would probably be OK at this point.  Reallocating
>> > what's on the heap is a slightly strange thing to do, since the
>> > point of the heap is to provide quick storage without the need to
>> > micromanage it;
>>
>> In that case, isn't USEHEAP better? It'll allocate on the heap if
>> there's anything to escape and otherwise do it in-place. Which is what
>> I thought HREALLOC would do, but I guess that one would do more stuff
>> to try and just grow. Won't it always succeed though, since the to-be
>> reallocated value was just allocated on the line before? (At least
>> when the heap isn't full).
>
> USEHEAP is fine if you don't mind further modifications to the string
> propagating back to the original.  In some places the string might not
> be modifiable, or might refer to a permanently stored string, so you
> wouldn't want to do that.  That may not be the case here; it needs
> deciding case by case.

Right, it isn't the case here, so I'll use this one.

Actually, it turns out getkeystring() always performs a copy, so I can
skip the dupstring and always set copied = 1, like for casemodify above
it. I also added an array to the test case, since it turns out you can
write code that works in one branch and code that doesn't work in the
other branch ;).

-- 
Mikael Magnusson

---
 Doc/Zsh/expn.yo        |   17 ++++++++++++--
 Src/subst.c            |   56 ++++++++++++++++++++++++++++++++++++++++++++++++
 Test/D04parameter.ztst |   20 +++++++++++++++++
 3 files changed, 90 insertions(+), 3 deletions(-)

diff --git a/Doc/Zsh/expn.yo b/Doc/Zsh/expn.yo
index 1cad3d2..9d5c81b 100644
--- a/Doc/Zsh/expn.yo
+++ b/Doc/Zsh/expn.yo
@@ -863,6 +863,14 @@ item(tt(F))(
 Join the words of arrays together using newline as a separator.
 This is a shorthand for `tt(pj:\n:)'.
 )
+item(tt(g:opts:))(
+Process escape sequences like the echo builtin when no options are given
+(tt(g::)).  With the tt(o) option, octal escapes don't take a leading
+zero.  With the tt(c) option, sequences like `tt(^X)' are also processed.
+With the tt(e) option, processes `tt(\M-t)' and similar sequences like the
+print builtin.  With both of the tt(o) and tt(e) options, behaves like the
+print builtin except that in none of these modes is `tt(\c)' interpreted.
+)
 item(tt(i))(
 Sort case-insensitively.  May be combined with `tt(n)' or `tt(O)'.
 )
@@ -1108,7 +1116,9 @@ Force field splitting at the
 separator var(string).  Note that a var(string) of two or more
 characters means that all of them must match in sequence; this differs from
 the treatment of two or more characters in the tt(IFS) parameter.
-See also the tt(=) flag and the tt(SH_WORD_SPLIT) option.
+See also the tt(=) flag and the tt(SH_WORD_SPLIT) option.  An empty
+string may also be given in which case every character will be a separate
+element.
 
 For historical reasons, the usual behaviour that empty array elements
 are retained inside double quotes is disabled for arrays generated
@@ -1292,8 +1302,9 @@ item(tt(11.) em(Case modification))(
 Any case modification from one of the flags tt((L)), tt((U)) or tt((C))
 is applied.
 )
-item(tt(12.) em(Prompt evaluation))(
-Any prompt-style formatting from the tt((%)) family of flags is applied.
+item(tt(12.) em(Escape sequence replacement))(
+First any replacements from the tt((g)) flag are performed, then any
+prompt-style formatting from the tt((%)) family of flags is applied.
 )
 item(tt(13.) em(Quote application))(
 Any quoting or unquoting using tt((q)) and tt((Q)) and related flags
diff --git a/Src/subst.c b/Src/subst.c
index 5d7a8b4..6bc0a82 100644
--- a/Src/subst.c
+++ b/Src/subst.c
@@ -1607,6 +1607,10 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
      */	
     int presc = 0;
     /*
+     * The (g) flag.  Process escape sequences with various GETKEY_ flags.
+     */
+    int getkeys = -1;
+    /*
      * The (@) flag; interacts obscurely with qt and isarr.
      * This is one of the things that decides whether multsub
      * will produce an array, but in an extremely indirect fashion.
@@ -1932,6 +1936,36 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 		    presc++;
 		    break;
 
+		case 'g':
+		    t = get_strarg(++s, &arglen);
+		    if (getkeys < 0)
+			getkeys = 0;
+		    if (*t) {
+			sav = *t;
+			*t = 0;
+			while (*++s) {
+			    switch (*s) {
+			    case 'e':
+				getkeys |= GETKEY_EMACS;
+				break;
+			    case 'o':
+				getkeys |= GETKEY_OCTAL_ESC;
+				break;
+			    case 'c':
+				getkeys |= GETKEY_CTRL;
+				break;
+
+			    default:
+				*t = sav;
+				goto flagerr;
+			    }
+			}
+			*t = sav;
+			s = t + arglen - 1;
+		    } else
+			goto flagerr;
+		    break;
+
 		case 'z':
 		    shsplit = LEXFLAGS_ACTIVE;
 		    break;
@@ -3100,6 +3134,28 @@ paramsubst(LinkList l, LinkNode n, char **str, int qt, int ssub)
 	}
     }
     /*
+     * Process echo- and print-style escape sequences.
+     */
+    if (getkeys >= 0) {
+	int len;
+
+	copied = 1;		/* string is always copied */
+	if (isarr) {
+	    char **ap, **ap2;
+
+	    ap = aval;
+	    aval = (char **) zhalloc(sizeof(char *) * (arrlen(aval)+1));
+	    for (ap2 = aval; *ap; ap++, ap2++) {
+		*ap2 = getkeystring(*ap, &len, getkeys, NULL);
+		*ap2 = metafy(*ap2, len, META_USEHEAP);
+	    }
+	    *ap2++ = NULL;
+	} else {
+	    val = getkeystring(val, &len, getkeys, NULL);
+	    val = metafy(val, len, META_USEHEAP);
+	}
+    }
+    /*
      * Perform prompt-style modifications.
      */
     if (presc) {
diff --git a/Test/D04parameter.ztst b/Test/D04parameter.ztst
index b91caaa..6379c8c 100644
--- a/Test/D04parameter.ztst
+++ b/Test/D04parameter.ztst
@@ -298,6 +298,26 @@
 >Howzat
 >usay
 
+  foo='\u65\123'
+  print -r ${(g:o:)foo}
+  foo='\u65\0123'
+  print -r ${(g::)foo}
+  foo='\u65^X'
+  print -r ${(V)${(g:c:)foo}}
+  foo='\u65\C-x\M-a'
+  print -r ${(V)${(g:e:)foo}}
+  foo='\u65\123\C-x'
+  print -r ${(V)${(g:eo:)foo}}
+  foo=('\u65' '\0123' '^X\M-a')
+  print -r ${(V)${(g:e:)foo}}
+0:${(g)...}
+>eS
+>eS
+>e^X
+>e^X\M-a
+>eS^X
+>e S ^X\M-a
+
   foo='I'\''m nearly out of my mind with tedium'
   bar=foo
   print ${(P)bar}
-- 
1.7.4-rc1