Another ${(z)param} buglet

Another ${(z)param} buglet

[Bart Schaefer]

It doesn't understand about comments, and becomes confused by a comment
that contains an unbalance quote character.

-- 

Re: Another ${(z)param} buglet

[Peter Stephenson]

On Tue, 07 Dec 2010 20:34:36 -0800
Bart Schaefer <schaefer@brasslantern.com> wrote:
> It doesn't understand about comments, and becomes confused by a
> comment that contains an unbalance quote character.

I think you can fix this something like the following, although it's not
particularly elegant.  Could make it strip comments instead of reporting
them, otherwise you need to remove fields starting with a # by hand (if
it's an unquoted #, it must be a comment).

If you were doing this to parse history, you'd need separate options for
handling INTERACTIVE_COMMENTS on and off.  I could make it respect
the option, but it's far from obvious that's sensible.

(z) is heavily overloaded to do weird things, and underdocumented, so I
have no idea if this is the right thing to do or not.

I was... confident is not the right word... blatant enough to attempt to
add some documentation to some of this.

Index: Src/hist.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/hist.c,v
retrieving revision 1.106
diff -p -u -r1.106 hist.c
--- Src/hist.c	10 Oct 2010 17:51:29 -0000	1.106
+++ Src/hist.c	8 Dec 2010 17:46:01 -0000
@@ -2345,7 +2345,7 @@ readhistfile(char *fn, int err, int read
 		/*
 		 * Attempt to do this using the lexer.
 		 */
-		LinkList wordlist = bufferwords(NULL, pt, NULL);
+		LinkList wordlist = bufferwords(NULL, pt, NULL, 1);
 		LinkNode wordnode;
 		int nwords_max;
 		nwords_max = 2 * countlinknodes(wordlist);
@@ -2885,11 +2885,26 @@ histfileIsLocked(void)
  * which may not even be valid at this point.
  *
  * However, I'm so confused it could simply be baking Bakewell tarts.
+ *
+ * list may be an existing linked list (off the heap), in which case
+ * it will be appended to; otherwise it will be created.
+ *
+ * If buf is set we will take input from that string, else we will
+ * attempt to use ZLE directly in a way they tell you not to do on all
+ * programming courses.
+ *
+ * If index is non-NULL, and input is from a string in ZLE, *index
+ * is set to the position of the end of the current editor word.
+ *
+ * If comments is 1, and buf is used for the input (i.e. this is
+ * not a string from ZLE), comments will be retained and reported
+ * as a single word in the output.  Otherwise, comments are not
+ * treated specially (they are analysed as normal words).
  */
 
 /**/
 mod_export LinkList
-bufferwords(LinkList list, char *buf, int *index)
+bufferwords(LinkList list, char *buf, int *index, int comments)
 {
     int num = 0, cur = -1, got = 0, ne = noerrs;
     int owb = wb, owe = we, oadx = addedx, ozp = zleparse, onc = nocomments;
@@ -2906,13 +2921,13 @@ bufferwords(LinkList list, char *buf, in
      * string expression, we just turn the option off for this function.
      */
     opts[RCQUOTES] = 0;
-    zleparse = 1;
     addedx = 0;
     noerrs = 1;
     lexsave();
     if (buf) {
 	int l = strlen(buf);
 
+	zleparse = 2;
 	p = (char *) zhalloc(l + 2);
 	memcpy(p, buf, l);
 	/*
@@ -2928,11 +2943,12 @@ bufferwords(LinkList list, char *buf, in
 	inpush(p, 0, NULL);
 	zlemetall = strlen(p) ;
 	zlemetacs = zlemetall + 1;
-	nocomments = 1;
+	nocomments = !comments;
     } else {
 	int ll, cs;
 	char *linein;
 
+	zleparse = 1;
 	linein = zleentry(ZLE_CMD_GET_LINE, &ll, &cs);
 	zlemetall = ll + 1; /* length of line plus space added below */
 	zlemetacs = cs;
Index: Src/lex.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/lex.c,v
retrieving revision 1.57
diff -p -u -r1.57 lex.c
--- Src/lex.c	18 Nov 2010 10:07:56 -0000	1.57
+++ Src/lex.c	8 Dec 2010 17:46:01 -0000
@@ -116,7 +116,20 @@ mod_export int wb, we;
 /**/
 mod_export int noaliases;
 
-/* we are parsing a line sent to use by the editor */
+/*
+ * we are parsing a line sent to use by the editor, or some other string
+ * that's not part of standard command input (e.g. eval is part of
+ * normal command input).
+ *
+ * zleparse = 1 is the normal case.
+ * zleparse = 2 is used for word splitting; the difference is we
+ * preserve comments.
+ *
+ * Note that although it is passed into the lexer as an input, the
+ * lexer can set it to zero after finding the word it's searching for.
+ * This only happens if the line being parsed actually does come from
+ * ZLE.
+ */
 
 /**/
 mod_export int zleparse;
@@ -743,26 +756,48 @@ gettok(void)
 
     /* chars in initial position in word */
 
+    /*
+     * Handle comments.  There are some special cases when this
+     * is not normal command input: zleparse implies we are examining
+     * a line lexically without it being used for normal command input.
+     * If zleparse is 1 we treat comments as normal for interactive
+     * mode.
+     * If zleparse is 2 (which has actually got nothing to do with zle)
+     * we always handle comments.
+     */
     if (c == hashchar && !nocomments &&
 	(isset(INTERACTIVECOMMENTS) ||
-	 (!zleparse && !expanding &&
+	 ((zleparse != 1) && !expanding &&
 	  (!interact || unset(SHINSTDIN) || strin)))) {
 	/* History is handled here to prevent extra  *
 	 * newlines being inserted into the history. */
 
+	if (zleparse == 2) {
+	    len = 0;
+	    bptr = tokstr = (char *)hcalloc(bsiz = 32);
+	    add(c);
+	}
 	while ((c = ingetc()) != '\n' && !lexstop) {
 	    hwaddc(c);
 	    addtoline(c);
+	    if (zleparse == 2)
+		add(c);
 	}
 
 	if (errflag)
 	    peek = LEXERR;
 	else {
-	    hwend();
-	    hwbegin(0);
-	    hwaddc('\n');
-	    addtoline('\n');
-	    peek = NEWLIN;
+	    if (zleparse == 2) {
+		if (!lexstop)
+		    hungetc(c);
+		peek = STRING;
+	    } else {
+		hwend();
+		hwbegin(0);
+		hwaddc('\n');
+		addtoline('\n');
+		peek = NEWLIN;
+	    }
 	}
 	return peek;
     }
Index: Src/subst.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/subst.c,v
retrieving revision 1.112
diff -p -u -r1.112 subst.c
--- Src/subst.c	25 Nov 2010 10:34:32 -0000	1.112
+++ Src/subst.c	8 Dec 2010 17:46:01 -0000
@@ -3207,10 +3207,10 @@ paramsubst(LinkList l, LinkNode n, char 
 	if (isarr) {
 	    char **ap;
 	    for (ap = aval; *ap; ap++)
-		list = bufferwords(list, *ap, NULL);
+		list = bufferwords(list, *ap, NULL, 1);
 	    isarr = 0;
 	} else
-	    list = bufferwords(NULL, val, NULL);
+	    list = bufferwords(NULL, val, NULL, 1);
 
 	if (!list || !firstnode(list))
 	    val = dupstring("");
Index: Src/Modules/parameter.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Modules/parameter.c,v
retrieving revision 1.51
diff -p -u -r1.51 parameter.c
--- Src/Modules/parameter.c	11 Feb 2009 20:42:17 -0000	1.51
+++ Src/Modules/parameter.c	8 Dec 2010 17:46:01 -0000
@@ -1044,7 +1044,7 @@ histwgetfn(UNUSED(Param pm))
     int i = addhistnum(curhist, -1, HIST_FOREIGN), iw;
     Histent he = gethistent(i, GETHIST_UPWARD);
 
-    if ((ll = bufferwords(NULL, NULL, NULL)))
+    if ((ll = bufferwords(NULL, NULL, NULL, 0)))
         for (n = firstnode(ll); n; incnode(n))
             pushnode(l, getdata(n));
 
Index: Src/Zle/zle_hist.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_hist.c,v
retrieving revision 1.66
diff -p -u -r1.66 zle_hist.c
--- Src/Zle/zle_hist.c	6 Oct 2010 08:25:29 -0000	1.66
+++ Src/Zle/zle_hist.c	8 Dec 2010 17:46:01 -0000
@@ -677,7 +677,7 @@ insertlastword(char **args)
 	 * a deleted word, because that can only have come
 	 * from a non-empty line.  I think.
 	 */
-	if (!(l = bufferwords(NULL, NULL, NULL))) {
+	if (!(l = bufferwords(NULL, NULL, NULL, 0))) {
 	    unmetafy_line();
 	    return 1;
 	}
Index: Src/Zle/zle_misc.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_misc.c,v
retrieving revision 1.58
diff -p -u -r1.58 zle_misc.c
--- Src/Zle/zle_misc.c	24 Apr 2009 09:00:38 -0000	1.58
+++ Src/Zle/zle_misc.c	8 Dec 2010 17:46:01 -0000
@@ -843,7 +843,7 @@ copyprevshellword(UNUSED(char **args))
     if (zmult <= 0)
 	return 1;
 
-    if ((l = bufferwords(NULL, NULL, &i))) {
+    if ((l = bufferwords(NULL, NULL, &i, 0))) {
 	i -= (zmult-1);
 	if (i < 0)
 	    return 1;
Index: Test/D04parameter.ztst
===================================================================
RCS file: /cvsroot/zsh/zsh/Test/D04parameter.ztst,v
retrieving revision 1.47
diff -p -u -r1.47 D04parameter.ztst
--- Test/D04parameter.ztst	25 Nov 2010 10:34:32 -0000	1.47
+++ Test/D04parameter.ztst	8 Dec 2010 17:46:02 -0000
@@ -417,6 +417,21 @@
 >5:i++ :
 >6:)):
 
+  line=$'A line with # someone\'s comment\nanother line # (1 more\nanother one'
+  print -l ${(z)line}
+0:Comments with (z)
+>A
+>line
+>with
+># someone's comment
+>;
+>another
+>line
+># (1 more
+>;
+>another
+>one
+
   psvar=(dog)
   setopt promptsubst
   foo='It shouldn'\''t $(happen) to a %1v.'


-- 
Peter Stephenson <pws@csr.com>            Software Engineer
Tel: +44 (0)1223 692070                   Cambridge Silicon Radio Limited
Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, UK


Member of the CSR plc group of companies. CSR plc registered in England and Wales, registered number 4187346, registered office Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, United Kingdom

Re: Another ${(z)param} buglet

[Bart Schaefer]

On Dec 8,  5:51pm, Peter Stephenson wrote:
} Subject: Re: Another ${(z)param} buglet
}
} On Tue, 07 Dec 2010 20:34:36 -0800
} Bart Schaefer <schaefer@brasslantern.com> wrote:
} > It doesn't understand about comments, and becomes confused by a
} > comment that contains an unbalance quote character.
} 
} I think you can fix this something like the following, although it's not
} particularly elegant.  Could make it strip comments instead of reporting
} them, otherwise you need to remove fields starting with a # by hand (if
} it's an unquoted #, it must be a comment).

Thanks for putting in this effort, but on reflection I think it's
important to be able to specify whether comments are handled or not.
There isn't yet a (Z) flag; perhaps we could make that the equivalent
of turning on interactivecomments, and leave (z) as it was?

I'm undecided on stripping comments vs. leaving them as "tokens" for
manual removal, but I lean to leaving them, as it's possible to remove
them once they've been parsed but it's practically impossible to put
them back once they're gone.

} I was... confident is not the right word... blatant enough to attempt
} to add some documentation to some of this.

Maybe this gives someone [*] enough to go on to be able to fix that
here-document problem I encountered.

[*] It won't be me, at least not until after the next 2 weeks are over.

Re: Another ${(z)param} buglet

[Peter Stephenson]

On Thu, 09 Dec 2010 07:42:33 -0800
Bart Schaefer <schaefer@brasslantern.com> wrote:
> Thanks for putting in this effort, but on reflection I think it's
> important to be able to specify whether comments are handled or not.
> There isn't yet a (Z) flag; perhaps we could make that the equivalent
> of turning on interactivecomments, and leave (z) as it was?

Wondered about that, but there are so many possible tweaks to word
splitting I'm trying to think of a more flexible way than simply two
letters (that doesn't involve anything as gross as a shell option).  The
best I can think of is extending the syntax to append options, a bit
like (q) can have the q multiple or a - added, but maybe less gross
e.g. explicit option flags like z+c+ to turn on comment handling (and
possibly z+C+ to strip comments). '+' appears not to be taken so would
work without complications and there's a mnemonic that there's more to
come (compared with "(q-)").

> I'm undecided on stripping comments vs. leaving them as "tokens" for
> manual removal, but I lean to leaving them, as it's possible to remove
> them once they've been parsed but it's practically impossible to put
> them back once they're gone.

That was my thinking, too.

> Maybe this gives someone [*] enough to go on to be able to fix that
> here-document problem I encountered.

That's not easily fixable down here.  You need some parsing to tell
you where the here documents are and how they fit together.  You could do
it within bufferwords() by assuming a << token is a here document and
being clever with multiple lines, but it's a real mess; they even need
to stack because of multios.

-- 
Peter Stephenson <pws@csr.com>            Software Engineer
Tel: +44 (0)1223 692070                   Cambridge Silicon Radio Limited
Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, UK


Member of the CSR plc group of companies. CSR plc registered in England and Wales, registered number 4187346, registered office Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, United Kingdom

Re: Another ${(z)param} buglet

[Peter Stephenson]

On Thu, 9 Dec 2010 18:16:32 +0000
Peter Stephenson <Peter.Stephenson@csr.com> wrote:
> The best I can think of is extending the syntax to append options, a bit
> like (q) can have the q multiple or a - added, but maybe less gross
> e.g. explicit option flags like z+c+ to turn on comment handling (and
> possibly z+C+ to strip comments). '+' appears not to be taken so would
> work without complications and there's a mnemonic that there's more to
> come (compared with "(q-)").

Here it is implemented and tested, so you can tell me it's not good
enough.

Luckily, after ten years in the electronics industry I never get bitter
and am entirely used to constant failure.

Note I've made the new parsed history reading option handle comments as
the option least likely to screw up: you can always parse a comment, you
just read to the end of the line.

Index: Doc/Zsh/expn.yo
===================================================================
RCS file: /cvsroot/zsh/zsh/Doc/Zsh/expn.yo,v
retrieving revision 1.126
diff -p -u -r1.126 expn.yo
--- Doc/Zsh/expn.yo	6 Dec 2010 13:53:15 -0000	1.126
+++ Doc/Zsh/expn.yo	9 Dec 2010 20:13:19 -0000
@@ -1006,6 +1006,17 @@ errors are silently ignored.
 item(tt(z))(
 Split the result of the expansion into words using shell parsing to
 find the words, i.e. taking into account any quoting in the value.
+Comments are not treated specially but as ordinary strings, similar
+to interactive shells with the tt(INTERACTIVE_COMMENTS) option unset.
+
+The flag can take option letters between a following pair of
+`tt(PLUS())' characters.  tt(LPAR()z+PLUS()c+PLUS()RPAR()) causes
+comments to be parsed as a string and retained; any field in the
+resulting array beginning with an unquoted comment character is a
+comment.  tt(LPAR()z+PLUS()C+PLUS()RPAR()) causes comments to be parsed
+and removed.  The rule for comments is standard: anything between a word
+starting with the third charcter of tt($HISTCHARS), default tt(#), up to
+the next newline is a comment.
 
 Note that this is done very late, as for the `tt((s))' flag. So to
 access single words in the result, one has to use nested expansions as 
Index: Src/hist.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/hist.c,v
retrieving revision 1.106
diff -p -u -r1.106 hist.c
--- Src/hist.c	10 Oct 2010 17:51:29 -0000	1.106
+++ Src/hist.c	9 Dec 2010 20:13:20 -0000
@@ -2345,7 +2345,7 @@ readhistfile(char *fn, int err, int read
 		/*
 		 * Attempt to do this using the lexer.
 		 */
-		LinkList wordlist = bufferwords(NULL, pt, NULL);
+		LinkList wordlist = bufferwords(NULL, pt, NULL, 1);
 		LinkNode wordnode;
 		int nwords_max;
 		nwords_max = 2 * countlinknodes(wordlist);
@@ -2885,11 +2885,27 @@ histfileIsLocked(void)
  * which may not even be valid at this point.
  *
  * However, I'm so confused it could simply be baking Bakewell tarts.
+ *
+ * list may be an existing linked list (off the heap), in which case
+ * it will be appended to; otherwise it will be created.
+ *
+ * If buf is set we will take input from that string, else we will
+ * attempt to use ZLE directly in a way they tell you not to do on all
+ * programming courses.
+ *
+ * If index is non-NULL, and input is from a string in ZLE, *index
+ * is set to the position of the end of the current editor word.
+ *
+ * comments is used if buf is non-NULL (i.e. this is not a string
+ * from ZLE).
+ * If it is 0, comments are not parsed; they are treated as ordinary words.
+ * If it is 1, comments are treated as single strings, one per line.
+ * If it is 2, comments are removed.
  */
 
 /**/
 mod_export LinkList
-bufferwords(LinkList list, char *buf, int *index)
+bufferwords(LinkList list, char *buf, int *index, int comments)
 {
     int num = 0, cur = -1, got = 0, ne = noerrs;
     int owb = wb, owe = we, oadx = addedx, ozp = zleparse, onc = nocomments;
@@ -2906,7 +2922,6 @@ bufferwords(LinkList list, char *buf, in
      * string expression, we just turn the option off for this function.
      */
     opts[RCQUOTES] = 0;
-    zleparse = 1;
     addedx = 0;
     noerrs = 1;
     lexsave();
@@ -2928,11 +2943,18 @@ bufferwords(LinkList list, char *buf, in
 	inpush(p, 0, NULL);
 	zlemetall = strlen(p) ;
 	zlemetacs = zlemetall + 1;
-	nocomments = 1;
+
+	/*
+	 * If comments is non-zero we are handling comments.
+	 * zleparse indicates the mode to the lexer.
+	 */
+	zleparse = 1 + comments;
+	nocomments = !comments;
     } else {
 	int ll, cs;
 	char *linein;
 
+	zleparse = 1;
 	linein = zleentry(ZLE_CMD_GET_LINE, &ll, &cs);
 	zlemetall = ll + 1; /* length of line plus space added below */
 	zlemetacs = cs;
Index: Src/lex.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/lex.c,v
retrieving revision 1.57
diff -p -u -r1.57 lex.c
--- Src/lex.c	18 Nov 2010 10:07:56 -0000	1.57
+++ Src/lex.c	9 Dec 2010 20:13:21 -0000
@@ -116,7 +116,22 @@ mod_export int wb, we;
 /**/
 mod_export int noaliases;
 
-/* we are parsing a line sent to use by the editor */
+/*
+ * we are parsing a line sent to use by the editor, or some other string
+ * that's not part of standard command input (e.g. eval is part of
+ * normal command input).
+ *
+ * zleparse = 1 is the normal case.
+ * zleparse = 2 is used for word splitting; the difference is we
+ *              preserve comments.
+ * zleparse = 3 is also for word splitting, here handling comments
+ *              but stripping them.
+ *
+ * Note that although it is passed into the lexer as an input, the
+ * lexer can set it to zero after finding the word it's searching for.
+ * This only happens if the line being parsed actually does come from
+ * ZLE.
+ */
 
 /**/
 mod_export int zleparse;
@@ -743,26 +758,50 @@ gettok(void)
 
     /* chars in initial position in word */
 
+    /*
+     * Handle comments.  There are some special cases when this
+     * is not normal command input: zleparse implies we are examining
+     * a line lexically without it being used for normal command input.
+     * If zleparse is 1 we treat comments as normal for interactive
+     * mode.
+     * If zleparse is 2 (which has actually got nothing to do with zle)
+     * we always handle comments and retain them.
+     * If zleparse is 3 we always handle comments and discard them.
+     */
     if (c == hashchar && !nocomments &&
 	(isset(INTERACTIVECOMMENTS) ||
-	 (!zleparse && !expanding &&
+	 ((zleparse != 1) && !expanding &&
 	  (!interact || unset(SHINSTDIN) || strin)))) {
 	/* History is handled here to prevent extra  *
 	 * newlines being inserted into the history. */
 
+	if (zleparse == 2) {
+	    len = 0;
+	    bptr = tokstr = (char *)hcalloc(bsiz = 32);
+	    add(c);
+	}
 	while ((c = ingetc()) != '\n' && !lexstop) {
 	    hwaddc(c);
 	    addtoline(c);
+	    if (zleparse == 2)
+		add(c);
 	}
 
 	if (errflag)
 	    peek = LEXERR;
 	else {
-	    hwend();
-	    hwbegin(0);
-	    hwaddc('\n');
-	    addtoline('\n');
-	    peek = NEWLIN;
+	    if (zleparse == 2) {
+		*bptr = '\0';
+		if (!lexstop)
+		    hungetc(c);
+		peek = STRING;
+	    } else {
+		hwend();
+		hwbegin(0);
+		hwaddc('\n');
+		addtoline('\n');
+		peek = NEWLIN;
+	    }
 	}
 	return peek;
     }
Index: Src/subst.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/subst.c,v
retrieving revision 1.112
diff -p -u -r1.112 subst.c
--- Src/subst.c	25 Nov 2010 10:34:32 -0000	1.112
+++ Src/subst.c	9 Dec 2010 20:13:21 -0000
@@ -1556,6 +1556,10 @@ paramsubst(LinkList l, LinkNode n, char 
      * The (z) flag, nothing to do with SH_WORD_SPLIT which is tied
      * spbreak, see above; fairly straighforward in use but c.f.
      * the comment for mods.
+     *
+     * This ultimately becomes zleparse during lexical analysis, via
+     * the comments argument to bufferwords(). It's got nothing
+     * to do with zle.
      */
     int shsplit = 0;
     /*
@@ -1934,6 +1938,27 @@ paramsubst(LinkList l, LinkNode n, char 
 
 		case 'z':
 		    shsplit = 1;
+		    if (s[1] == '+') {
+			s += 2;
+			while (*s && *s != '+' && *s != ')' && *s != Outpar) {
+			    switch (*s++) {
+			    case 'c':
+				/* Parse and keep comments */
+				shsplit = 2;
+				break;
+
+			    case 'C':
+				/* Parse and remove comments */
+				shsplit = 3;
+				break;
+
+			    default:
+				goto flagerr;
+			    }
+			}
+			if (*s != '+')
+			    goto flagerr;
+		    }
 		    break;
 
 		case 'u':
@@ -3207,10 +3232,10 @@ paramsubst(LinkList l, LinkNode n, char 
 	if (isarr) {
 	    char **ap;
 	    for (ap = aval; *ap; ap++)
-		list = bufferwords(list, *ap, NULL);
+		list = bufferwords(list, *ap, NULL, shsplit-1);
 	    isarr = 0;
 	} else
-	    list = bufferwords(NULL, val, NULL);
+	    list = bufferwords(NULL, val, NULL, shsplit-1);
 
 	if (!list || !firstnode(list))
 	    val = dupstring("");
Index: Src/Modules/parameter.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Modules/parameter.c,v
retrieving revision 1.51
diff -p -u -r1.51 parameter.c
--- Src/Modules/parameter.c	11 Feb 2009 20:42:17 -0000	1.51
+++ Src/Modules/parameter.c	9 Dec 2010 20:13:21 -0000
@@ -1044,7 +1044,7 @@ histwgetfn(UNUSED(Param pm))
     int i = addhistnum(curhist, -1, HIST_FOREIGN), iw;
     Histent he = gethistent(i, GETHIST_UPWARD);
 
-    if ((ll = bufferwords(NULL, NULL, NULL)))
+    if ((ll = bufferwords(NULL, NULL, NULL, 0)))
         for (n = firstnode(ll); n; incnode(n))
             pushnode(l, getdata(n));
 
Index: Src/Zle/zle_hist.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_hist.c,v
retrieving revision 1.66
diff -p -u -r1.66 zle_hist.c
--- Src/Zle/zle_hist.c	6 Oct 2010 08:25:29 -0000	1.66
+++ Src/Zle/zle_hist.c	9 Dec 2010 20:13:22 -0000
@@ -677,7 +677,7 @@ insertlastword(char **args)
 	 * a deleted word, because that can only have come
 	 * from a non-empty line.  I think.
 	 */
-	if (!(l = bufferwords(NULL, NULL, NULL))) {
+	if (!(l = bufferwords(NULL, NULL, NULL, 0))) {
 	    unmetafy_line();
 	    return 1;
 	}
Index: Src/Zle/zle_misc.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_misc.c,v
retrieving revision 1.58
diff -p -u -r1.58 zle_misc.c
--- Src/Zle/zle_misc.c	24 Apr 2009 09:00:38 -0000	1.58
+++ Src/Zle/zle_misc.c	9 Dec 2010 20:13:22 -0000
@@ -843,7 +843,7 @@ copyprevshellword(UNUSED(char **args))
     if (zmult <= 0)
 	return 1;
 
-    if ((l = bufferwords(NULL, NULL, &i))) {
+    if ((l = bufferwords(NULL, NULL, &i, 0))) {
 	i -= (zmult-1);
 	if (i < 0)
 	    return 1;
Index: Test/D04parameter.ztst
===================================================================
RCS file: /cvsroot/zsh/zsh/Test/D04parameter.ztst,v
retrieving revision 1.47
diff -p -u -r1.47 D04parameter.ztst
--- Test/D04parameter.ztst	25 Nov 2010 10:34:32 -0000	1.47
+++ Test/D04parameter.ztst	9 Dec 2010 20:13:22 -0000
@@ -417,6 +417,45 @@
 >5:i++ :
 >6:)):
 
+  line=$'A line with # someone\'s comment\nanother line # (1 more\nanother one'
+  print "*** Normal ***"
+  print -l ${(z)line}
+  print "*** Kept ***"
+  print -l ${(z+c+)line}
+  print "*** Removed ***"
+  print -l ${(z+C+)line}
+0:Comments with (z)
+>*** Normal ***
+>A
+>line
+>with
+>#
+>someone's comment
+>another line # (1 more
+>another one
+>*** Kept ***
+>A
+>line
+>with
+># someone's comment
+>;
+>another
+>line
+># (1 more
+>;
+>another
+>one
+>*** Removed ***
+>A
+>line
+>with
+>;
+>another
+>line
+>;
+>another
+>one
+
   psvar=(dog)
   setopt promptsubst
   foo='It shouldn'\''t $(happen) to a %1v.'
-- 
Peter Stephenson <p.w.stephenson@ntlworld.com>
Web page now at http://homepage.ntlworld.com/p.w.stephenson/

Re: Another ${(z)param} buglet

[Bart Schaefer]

On Dec 9,  6:16pm, Peter Stephenson wrote:
}
} On Thu, 09 Dec 2010 07:42:33 -0800
} Bart Schaefer <schaefer@brasslantern.com> wrote:
} > There isn't yet a (Z) flag; perhaps we could make that the equivalent
} > of turning on interactivecomments, and leave (z) as it was?
} 
} Wondered about that, but there are so many possible tweaks to word
} splitting I'm trying to think of a more flexible way than simply two
} letters (that doesn't involve anything as gross as a shell option).

Perhaps leave (z) alone and introduce (Z:stuff:) where stuff can be
a set of controls for how the parse is applied?

Another possibility would be to introduce a builtin ala zparseopts that
dismantles the string and populates a hash, or something.

Re: Another ${(z)param} buglet

[Peter Stephenson]

On Thu, 9 Dec 2010 20:19:13 +0000
Peter Stephenson <p.w.stephenson@ntlworld.com> wrote:
> On Thu, 9 Dec 2010 18:16:32 +0000
> Peter Stephenson <Peter.Stephenson@csr.com> wrote:
> > The best I can think of is extending the syntax to append options, a bit
> > like (q) can have the q multiple or a - added, but maybe less gross
> > e.g. explicit option flags like z+c+ to turn on comment handling (and
> > possibly z+C+ to strip comments). '+' appears not to be taken so would
> > work without complications and there's a mnemonic that there's more to
> > come (compared with "(q-)").
> 
> Here it is implemented and tested, so you can tell me it's not good
> enough.

As there were no further comments (funny, that...) I've committed it.

-- 
Peter Stephenson <p.w.stephenson@ntlworld.com>
Web page now at http://homepage.ntlworld.com/p.w.stephenson/

Re: Another ${(z)param} buglet

[Bart Schaefer]

On Dec 12, 10:45pm, Peter Stephenson wrote:
}
} > Here it is implemented and tested, so you can tell me it's not good
} > enough.
} 
} As there were no further comments (funny, that...) I've committed it.

I'd already made my suggestion about (Z:...:) as an alternative, so it
didn't seem necessary to repeat myself.  You've now done the difficult
bits, so ...

Here's a patch that leaves (z+...+) and the rest of PWS's excellent
bufferwords tricks, but adds (Z:...:) where ":" can be an arbitrary
character or matching braces as with (s:...:) et al.

I'm thinking future possible enhancments for (Z) include locally altering
the value of $histchars (which is why one might want a delimiter other
than "+"), and performing history substitutions on the string.  None of
that has been done here, yet; also I made only a minimal change to the
test script to prove it works, and haven't done doc yet either.

What say?  This could also easily be tweaked to make (z) and (z:...:),
with the :...: having become optional, but we don't have any other cases
where the delimited arg string is completely optional [although (l) and
(r) have optional sub-parts].

Index: Src/subst.c
--- zsh-forge/current/Src/subst.c	2010-12-12 16:22:56.000000000 -0800
+++ Src/subst.c	2010-12-12 16:50:38.000000000 -0800
@@ -1938,10 +1938,16 @@
 
 		case 'z':
 		    shsplit = 1;
-		    if (s[1] == '+') {
-			s += 2;
-			while (*s && *s != '+' && *s != ')' && *s != Outpar) {
-			    switch (*s++) {
+		    if (s[1] != '+')
+			break;
+
+		case 'Z':
+		    t = get_strarg(++s, &arglen);
+		    if (*t) {
+			sav = *t;
+			*t = 0;
+			while (*++s) {
+			    switch (*s) {
 			    case 'c':
 				/* Parse and keep comments */
 				shsplit = 2;
@@ -1956,9 +1962,10 @@
 				goto flagerr;
 			    }
 			}
-			if (*s != '+')
-			    goto flagerr;
-		    }
+			*t = sav;
+			s = t + arglen - 1;
+		    } else
+			goto flagerr;
 		    break;
 
 		case 'u':
--- zsh-forge/current/Test/D04parameter.ztst	2010-12-12 16:22:56.000000000 -0800
+++ Test/D04parameter.ztst	2010-12-12 16:50:37.000000000 -0800
@@ -423,7 +423,7 @@
   print "*** Kept ***"
   print -l ${(z+c+)line}
   print "*** Removed ***"
-  print -l ${(z+C+)line}
+  print -l ${(Z+C+)line}
 0:Comments with (z)
 >*** Normal ***
 >A

Re: Another ${(z)param} buglet

[Bart Schaefer]

On Dec 12, 10:45pm, Peter Stephenson wrote:
}
} > > possibly z+C+ to strip comments
} 
} As there were no further comments (funny, that...)

I really need to get some more sleep.

Re: Another ${(z)param} buglet

[Peter Stephenson]

On Sun, 12 Dec 2010 17:26:52 -0800
Bart Schaefer <schaefer@brasslantern.com> wrote:
> What say?  This could also easily be tweaked to make (z) and (z:...:),
> with the :...: having become optional, but we don't have any other
> cases where the delimited arg string is completely optional [although
> (l) and (r) have optional sub-parts].

Right, that's why I stuck with + there; you've got to guess if
something's a delimiter, which adds a non-quite-obvious bit to the
existing delimiter rule.

If you've got Z you don't necessarily need z+ as well.  I was trying to
avoid using up yet another letter, but once you do there's no great need
to do anything with the original one.

-- 
Peter Stephenson <pws@csr.com>            Software Engineer
Tel: +44 (0)1223 692070                   Cambridge Silicon Radio Limited
Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, UK


Member of the CSR plc group of companies. CSR plc registered in England and Wales, registered number 4187346, registered office Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, United Kingdom

Re: Another ${(z)param} buglet

[Peter Stephenson]

On Thu, 9 Dec 2010 20:19:13 +0000
Peter Stephenson <p.w.stephenson@ntlworld.com> wrote:
> On Thu, 9 Dec 2010 18:16:32 +0000
> Peter Stephenson <Peter.Stephenson@csr.com> wrote:
> > The best I can think of is extending the syntax to append options,
> > a bit like (q) can have the q multiple or a - added, but maybe less
> > gross e.g. explicit option flags like z+c+ to turn on comment
> > handling (and possibly z+C+ to strip comments). '+' appears not to
> > be taken so would work without complications and there's a mnemonic
> > that there's more to come (compared with "(q-)").
> 
> Here it is implemented and tested, so you can tell me it's not good
> enough.

One tweak to this version of the patch: if you stripped comments and
there was one right at the end of the string, the end of line was
treated as a newline so you got a ';', which is inconsistent with what
you get without the comment, and indeed with the comment retained as a
string.

Index: Src/lex.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/lex.c,v
retrieving revision 1.58
diff -p -u -r1.58 lex.c
--- Src/lex.c	12 Dec 2010 22:44:51 -0000	1.58
+++ Src/lex.c	13 Dec 2010 10:12:40 -0000
@@ -800,7 +800,15 @@ gettok(void)
 		hwbegin(0);
 		hwaddc('\n');
 		addtoline('\n');
-		peek = NEWLIN;
+		/*
+		 * If splitting a line and removing comments,
+		 * we don't want a newline token since it's
+		 * treated specially.
+		 */
+		if (zleparse == 3 && lexstop)
+		    peek = ENDINPUT;
+		else
+		    peek = NEWLIN;
 	    }
 	}
 	return peek;
Index: Test/D04parameter.ztst
===================================================================
RCS file: /cvsroot/zsh/zsh/Test/D04parameter.ztst,v
retrieving revision 1.48
diff -p -u -r1.48 D04parameter.ztst
--- Test/D04parameter.ztst	12 Dec 2010 22:44:51 -0000	1.48
+++ Test/D04parameter.ztst	13 Dec 2010 10:12:40 -0000
@@ -456,6 +456,12 @@
 >another
 >one
 
+  line='with comment # at the end'
+  print -l ${(z+C+)line}
+0:Test we don't get an additional newline token
+>with
+>comment
+
   psvar=(dog)
   setopt promptsubst
   foo='It shouldn'\''t $(happen) to a %1v.'

-- 
Peter Stephenson <pws@csr.com>            Software Engineer
Tel: +44 (0)1223 692070                   Cambridge Silicon Radio Limited
Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, UK


Member of the CSR plc group of companies. CSR plc registered in England and Wales, registered number 4187346, registered office Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, United Kingdom

Re: Another ${(z)param} buglet

[Bart Schaefer]

On Dec 13,  9:47am, Peter Stephenson wrote:
}
} Bart Schaefer <schaefer@brasslantern.com> wrote:
} > What say?  This could also easily be tweaked to make (z) and (z:...:),
} > with the :...: having become optional

It occurred to me later that, because any character can be a delimiter,
it's not really possible to have it be optional in this way.  That is,
${(lq8qq-q)RANDOM} only works because "l" is always followed by a
delimiter.

} If you've got Z you don't necessarily need z+ as well. I was trying to
} avoid using up yet another letter, but once you do there's no great
} need to do anything with the original one.

OK ... that still doesn't answer the question of whether to in fact
use up that extra letter, which is probably related to the question
of whether one ever might want to use a different delimiter.

As it's already possible to write confusing-looking expansions like
that RANDOM example, how about this suggestion:

Let's go ahead and use up Z for this so as not to tangle up z with a
new more restricted delimiter syntax.  Further let's immediately
reserve a character (maybe "+") to have no meaning of its own, but
instead to always introduce a delimited string into which we can put 
new options.  E.g. (C) and (+(C)) could have distinct meanings.
That gets us back the entire alphabet for new things we want to
throw into (+:...:), and permits the possibility of nesting (which
I hope we'd never need) such as (+{+{...}}).

Currently (before z+ or the above proposal) we have these letters
left over (I'm intentionally excluding $ as that would engender
_way_ too much confusion):

    ^ = _ * + b g G J K T x y Y Z

In the "probably to be avoided" category are:

    < > | - , ; : ! ? / .  [ ] { } &

And we've used these:

    ~ @ # % 0 a A B c C d D e E f F i I j k l L m M
    n N o O p P q Q r R s S t u U v V w W X z

Thoughts?  I'd actually lean towards "_" rather than "+" as the
introducer for the delimted extension syntax, but there may be things
I'm not considering like limitations of non-English keyboards.

-- 

Re: Another ${(z)param} buglet

[Peter Stephenson]

On Mon, 13 Dec 2010 09:35:12 -0800
Bart Schaefer <schaefer@brasslantern.com> wrote:
> Let's go ahead and use up Z for this so as not to tangle up z with a
> new more restricted delimiter syntax.  Further let's immediately
> reserve a character (maybe "+") to have no meaning of its own, but
> instead to always introduce a delimited string into which we can put 
> new options.  E.g. (C) and (+(C)) could have distinct meanings.
> That gets us back the entire alphabet for new things we want to
> throw into (+:...:), and permits the possibility of nesting (which
> I hope we'd never need) such as (+{+{...}}).

Can't see why that shouldn't work.

Here's a patch still using the previous delimiters, though it shouldn't
be any harder to fix up than before since it's parsed the same way even
though it's using the result more cleanly.

It tidies up the zleparse nonsense into a variable lexflags that
contains bit flags for this stuff.  I discovered one place in
zle_tricky.c where zleparse was being set to 2, predating this set of
patches; however zleparse was only ever tested and set as a boolean
elsewhere (as long ago as 4.2), so that must be cruft now.

This allows me to introduce a new flag, n, so that I can finally get my
lines split on all whitespace including newlines without ';' characters
appearing.

lexsave/lexrestore should probably save and restore lexflags: not done
that yet. Then we don't need to do it in bufferwords() (which does
it cleanly already, but goodness knows what ZLE does).

Index: Doc/Zsh/expn.yo
===================================================================
RCS file: /cvsroot/zsh/zsh/Doc/Zsh/expn.yo,v
retrieving revision 1.127
diff -p -u -r1.127 expn.yo
--- Doc/Zsh/expn.yo	12 Dec 2010 22:44:51 -0000	1.127
+++ Doc/Zsh/expn.yo	13 Dec 2010 18:00:21 -0000
@@ -1009,14 +1009,17 @@ find the words, i.e. taking into account
 Comments are not treated specially but as ordinary strings, similar
 to interactive shells with the tt(INTERACTIVE_COMMENTS) option unset.
 
-The flag can take option letters between a following pair of
-`tt(PLUS())' characters.  tt(LPAR()z+PLUS()c+PLUS()RPAR()) causes
-comments to be parsed as a string and retained; any field in the
+The flag can take a combination of option letters between a following
+pair of `tt(PLUS())' characters.  tt(LPAR()z+PLUS()c+PLUS()RPAR())
+causes comments to be parsed as a string and retained; any field in the
 resulting array beginning with an unquoted comment character is a
 comment.  tt(LPAR()z+PLUS()C+PLUS()RPAR()) causes comments to be parsed
 and removed.  The rule for comments is standard: anything between a word
 starting with the third charcter of tt($HISTCHARS), default tt(#), up to
-the next newline is a comment.
+the next newline is a comment.  tt(LPAR()z+PLUS()n+PLUS()RPAR()) causes
+unquoted newlines to be treated as ordinary whitespace, else they are
+treated as if they are shell code delimiters and converted to
+semicolons.
 
 Note that this is done very late, as for the `tt((s))' flag. So to
 access single words in the result, one has to use nested expansions as 
Index: Src/hist.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/hist.c,v
retrieving revision 1.107
diff -p -u -r1.107 hist.c
--- Src/hist.c	12 Dec 2010 22:44:51 -0000	1.107
+++ Src/hist.c	13 Dec 2010 18:00:21 -0000
@@ -2345,7 +2345,8 @@ readhistfile(char *fn, int err, int read
 		/*
 		 * Attempt to do this using the lexer.
 		 */
-		LinkList wordlist = bufferwords(NULL, pt, NULL, 1);
+		LinkList wordlist = bufferwords(NULL, pt, NULL,
+						LEXFLAGS_COMMENTS_KEEP);
 		LinkNode wordnode;
 		int nwords_max;
 		nwords_max = 2 * countlinknodes(wordlist);
@@ -2905,10 +2906,10 @@ histfileIsLocked(void)
 
 /**/
 mod_export LinkList
-bufferwords(LinkList list, char *buf, int *index, int comments)
+bufferwords(LinkList list, char *buf, int *index, int flags)
 {
     int num = 0, cur = -1, got = 0, ne = noerrs;
-    int owb = wb, owe = we, oadx = addedx, ozp = zleparse, onc = nocomments;
+    int owb = wb, owe = we, oadx = addedx, ozp = lexflags, onc = nocomments;
     int ona = noaliases, ocs = zlemetacs, oll = zlemetall;
     int forloop = 0, rcquotes = opts[RCQUOTES];
     char *p, *addedspaceptr;
@@ -2925,6 +2926,12 @@ bufferwords(LinkList list, char *buf, in
     addedx = 0;
     noerrs = 1;
     lexsave();
+    lexflags = flags | LEXFLAGS_ACTIVE;
+    /*
+     * Are we handling comments?
+     */
+    nocomments = !(flags & (LEXFLAGS_COMMENTS_KEEP|
+			    LEXFLAGS_COMMENTS_STRIP));
     if (buf) {
 	int l = strlen(buf);
 
@@ -2943,18 +2950,10 @@ bufferwords(LinkList list, char *buf, in
 	inpush(p, 0, NULL);
 	zlemetall = strlen(p) ;
 	zlemetacs = zlemetall + 1;
-
-	/*
-	 * If comments is non-zero we are handling comments.
-	 * zleparse indicates the mode to the lexer.
-	 */
-	zleparse = 1 + comments;
-	nocomments = !comments;
     } else {
 	int ll, cs;
 	char *linein;
 
-	zleparse = 1;
 	linein = zleentry(ZLE_CMD_GET_LINE, &ll, &cs);
 	zlemetall = ll + 1; /* length of line plus space added below */
 	zlemetacs = cs;
@@ -3096,7 +3095,7 @@ bufferwords(LinkList list, char *buf, in
 	    }
 	    forloop--;
 	}
-	if (!got && !zleparse) {
+	if (!got && !lexflags) {
 	    got = 1;
 	    cur = num - 1;
 	}
@@ -3121,7 +3120,7 @@ bufferwords(LinkList list, char *buf, in
     strinend();
     inpop();
     errflag = 0;
-    zleparse = ozp;
+    lexflags = ozp;
     nocomments = onc;
     noerrs = ne;
     lexrestore();
Index: Src/lex.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/lex.c,v
retrieving revision 1.59
diff -p -u -r1.59 lex.c
--- Src/lex.c	13 Dec 2010 11:20:47 -0000	1.59
+++ Src/lex.c	13 Dec 2010 18:00:21 -0000
@@ -117,15 +117,11 @@ mod_export int wb, we;
 mod_export int noaliases;
 
 /*
- * we are parsing a line sent to use by the editor, or some other string
- * that's not part of standard command input (e.g. eval is part of
- * normal command input).
+ * If non-zero, we are parsing a line sent to use by the editor, or some
+ * other string that's not part of standard command input (e.g. eval is
+ * part of normal command input).
  *
- * zleparse = 1 is the normal case.
- * zleparse = 2 is used for word splitting; the difference is we
- *              preserve comments.
- * zleparse = 3 is also for word splitting, here handling comments
- *              but stripping them.
+ * Set of bits from LEXFLAGS_*.
  *
  * Note that although it is passed into the lexer as an input, the
  * lexer can set it to zero after finding the word it's searching for.
@@ -134,7 +130,7 @@ mod_export int noaliases;
  */
 
 /**/
-mod_export int zleparse;
+mod_export int lexflags;
 
 /**/
 mod_export int wordbeg;
@@ -429,7 +425,7 @@ zshlex(void)
 	isnewlin = 0;
     else
 	isnewlin = (inbufct) ? -1 : 1;
-    if (tok == SEMI || tok == NEWLIN)
+    if (tok == SEMI || (tok == NEWLIN && !(lexflags & LEXFLAGS_NEWLINE)))
 	tok = SEPER;
 }
 
@@ -588,9 +584,9 @@ add(int c)
     }
 }
 
-#define SETPARBEGIN {if (zleparse && !(inbufflags & INP_ALIAS) && zlemetacs >= zlemetall+1-inbufct) parbegin = inbufct;}
+#define SETPARBEGIN {if (lexflags && !(inbufflags & INP_ALIAS) && zlemetacs >= zlemetall+1-inbufct) parbegin = inbufct;}
 #define SETPAREND {\
-	    if (zleparse && !(inbufflags & INP_ALIAS) && parbegin != -1 && parend == -1) {\
+	    if (lexflags && !(inbufflags & INP_ALIAS) && parbegin != -1 && parend == -1) {\
 		if (zlemetacs >= zlemetall + 1 - inbufct)\
 		    parbegin = -1;\
 		else\
@@ -760,22 +756,17 @@ gettok(void)
 
     /*
      * Handle comments.  There are some special cases when this
-     * is not normal command input: zleparse implies we are examining
+     * is not normal command input: lexflags implies we are examining
      * a line lexically without it being used for normal command input.
-     * If zleparse is 1 we treat comments as normal for interactive
-     * mode.
-     * If zleparse is 2 (which has actually got nothing to do with zle)
-     * we always handle comments and retain them.
-     * If zleparse is 3 we always handle comments and discard them.
      */
     if (c == hashchar && !nocomments &&
 	(isset(INTERACTIVECOMMENTS) ||
-	 ((zleparse != 1) && !expanding &&
+	 ((!lexflags || (lexflags & LEXFLAGS_COMMENTS)) && !expanding &&
 	  (!interact || unset(SHINSTDIN) || strin)))) {
 	/* History is handled here to prevent extra  *
 	 * newlines being inserted into the history. */
 
-	if (zleparse == 2) {
+	if (lexflags & LEXFLAGS_COMMENTS_KEEP) {
 	    len = 0;
 	    bptr = tokstr = (char *)hcalloc(bsiz = 32);
 	    add(c);
@@ -783,14 +774,14 @@ gettok(void)
 	while ((c = ingetc()) != '\n' && !lexstop) {
 	    hwaddc(c);
 	    addtoline(c);
-	    if (zleparse == 2)
+	    if (lexflags & LEXFLAGS_COMMENTS_KEEP)
 		add(c);
 	}
 
 	if (errflag)
 	    peek = LEXERR;
 	else {
-	    if (zleparse == 2) {
+	    if (lexflags & LEXFLAGS_COMMENTS_KEEP) {
 		*bptr = '\0';
 		if (!lexstop)
 		    hungetc(c);
@@ -805,7 +796,7 @@ gettok(void)
 		 * we don't want a newline token since it's
 		 * treated specially.
 		 */
-		if (zleparse == 3 && lexstop)
+		if ((lexflags & LEXFLAGS_COMMENTS_STRIP) && lexstop)
 		    peek = ENDINPUT;
 		else
 		    peek = NEWLIN;
@@ -1778,7 +1769,7 @@ gotword(void)
     we = zlemetall + 1 - inbufct + (addedx == 2 ? 1 : 0);
     if (zlemetacs <= we) {
 	wb = zlemetall - wordbeg + addedx;
-	zleparse = 0;
+	lexflags = 0;
     }
 }
 
@@ -1813,11 +1804,11 @@ exalias(void)
 	} else
 	    zshlextext = tokstr;
 
-	if (zleparse && !(inbufflags & INP_ALIAS)) {
-	    int zp = zleparse;
+	if (lexflags && !(inbufflags & INP_ALIAS)) {
+	    int zp = lexflags;
 
 	    gotword();
-	    if (zp == 1 && !zleparse) {
+	    if (zp == 1 && !lexflags) {
 		if (zshlextext == copy)
 		    zshlextext = tokstr;
 		return 0;
Index: Src/subst.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/subst.c,v
retrieving revision 1.113
diff -p -u -r1.113 subst.c
--- Src/subst.c	12 Dec 2010 22:44:51 -0000	1.113
+++ Src/subst.c	13 Dec 2010 18:00:21 -0000
@@ -1557,9 +1557,7 @@ paramsubst(LinkList l, LinkNode n, char 
      * spbreak, see above; fairly straighforward in use but c.f.
      * the comment for mods.
      *
-     * This ultimately becomes zleparse during lexical analysis, via
-     * the comments argument to bufferwords(). It's got nothing
-     * to do with zle.
+     * This gets set to one of the LEXFLAGS_* values.
      */
     int shsplit = 0;
     /*
@@ -1937,19 +1935,24 @@ paramsubst(LinkList l, LinkNode n, char 
 		    break;
 
 		case 'z':
-		    shsplit = 1;
+		    shsplit = LEXFLAGS_ACTIVE;
 		    if (s[1] == '+') {
 			s += 2;
 			while (*s && *s != '+' && *s != ')' && *s != Outpar) {
 			    switch (*s++) {
 			    case 'c':
 				/* Parse and keep comments */
-				shsplit = 2;
+				shsplit |= LEXFLAGS_COMMENTS_KEEP;
 				break;
 
 			    case 'C':
 				/* Parse and remove comments */
-				shsplit = 3;
+				shsplit |= LEXFLAGS_COMMENTS_STRIP;
+				break;
+
+			    case 'n':
+				/* Treat newlines as whitespace */
+				shsplit |= LEXFLAGS_NEWLINE;
 				break;
 
 			    default:
@@ -3232,10 +3235,10 @@ paramsubst(LinkList l, LinkNode n, char 
 	if (isarr) {
 	    char **ap;
 	    for (ap = aval; *ap; ap++)
-		list = bufferwords(list, *ap, NULL, shsplit-1);
+		list = bufferwords(list, *ap, NULL, shsplit);
 	    isarr = 0;
 	} else
-	    list = bufferwords(NULL, val, NULL, shsplit-1);
+	    list = bufferwords(NULL, val, NULL, shsplit);
 
 	if (!list || !firstnode(list))
 	    val = dupstring("");
Index: Src/zsh.h
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/zsh.h,v
retrieving revision 1.170
diff -p -u -r1.170 zsh.h
--- Src/zsh.h	22 Nov 2010 11:42:47 -0000	1.170
+++ Src/zsh.h	13 Dec 2010 18:00:21 -0000
@@ -1823,6 +1823,34 @@ struct histent {
 #define HFILE_NO_REWRITE	0x0020
 #define HFILE_USE_OPTIONS	0x8000
 
+/*
+ * Flags argument to bufferwords() used
+ * also by lexflags variable.
+ */
+/*
+ * Kick the lexer into special string-analysis
+ * mode without parsing.  Any bit set in
+ * the flags has this effect, but this
+ * has otherwise all the default effects.
+ */
+#define LEXFLAGS_ACTIVE		0x0001
+/*
+ * Parse comments and treat each comment as a single string
+ */
+#define LEXFLAGS_COMMENTS_KEEP	0x0002
+/*
+ * Parse comments and strip them.
+ */
+#define LEXFLAGS_COMMENTS_STRIP	0x0004
+/*
+ * Either of the above
+ */
+#define LEXFLAGS_COMMENTS (LEXFLAGS_COMMENTS_KEEP|LEXFLAGS_COMMENTS_STRIP)
+/*
+ * Treat newlines as whitespace
+ */
+#define LEXFLAGS_NEWLINE	0x0008
+
 /******************************************/
 /* Definitions for programable completion */
 /******************************************/
Index: Src/Zle/compcore.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/compcore.c,v
retrieving revision 1.102
diff -p -u -r1.102 compcore.c
--- Src/Zle/compcore.c	6 Dec 2010 10:49:51 -0000	1.102
+++ Src/Zle/compcore.c	13 Dec 2010 18:00:21 -0000
@@ -1481,7 +1481,7 @@ set_comp_sep(void)
 
     /* Put the string in the lexer buffer and call the lexer to *
      * get the words we have to expand.                        */
-    zleparse = 1;
+    lexflags = LEXFLAGS_ACTIVE;
     ocs = zlemetacs;
     oll = zlemetall;
     ol = zlemetaline;
@@ -1616,7 +1616,7 @@ set_comp_sep(void)
         }
 	else
 	    p = NULL;
-	if (!got && !zleparse) {
+	if (!got && !lexflags) {
 	    DPUTS(!p, "no current word in substr");
 	    got = 1;
 	    cur = i;
@@ -1634,7 +1634,7 @@ set_comp_sep(void)
     noaliases = ona;
     strinend();
     inpop();
-    errflag = zleparse = 0;
+    errflag = lexflags = 0;
     noerrs = ne;
     lexrestore();
     wb = owb;
Index: Src/Zle/compctl.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/compctl.c,v
retrieving revision 1.38
diff -p -u -r1.38 compctl.c
--- Src/Zle/compctl.c	11 Nov 2008 19:36:34 -0000	1.38
+++ Src/Zle/compctl.c	13 Dec 2010 18:00:21 -0000
@@ -2789,7 +2789,7 @@ sep_comp_string(char *ss, char *s, int n
 
     /* Put the string in the lexer buffer and call the lexer to *
      * get the words we have to expand.                        */
-    zleparse = 1;
+    lexflags = LEXFLAGS_ACTIVE;
     addedx = 1;
     noerrs = 1;
     lexsave();
@@ -2828,7 +2828,7 @@ sep_comp_string(char *ss, char *s, int n
 	    addlinknode(foo, (p = ztrdup(tokstr)));
 	else
 	    p = NULL;
-	if (!got && !zleparse) {
+	if (!got && !lexflags) {
 	    DPUTS(!p, "no current word in substr");
 	    got = 1;
 	    cur = i;
@@ -2843,7 +2843,7 @@ sep_comp_string(char *ss, char *s, int n
     noaliases = ona;
     strinend();
     inpop();
-    errflag = zleparse = 0;
+    errflag = lexflags = 0;
     noerrs = ne;
     lexrestore();
     wb = owb;
@@ -3703,7 +3703,7 @@ makecomplistflags(Compctl cc, char *s, i
 
 	/* Put the string in the lexer buffer and call the lexer to *
 	 * get the words we have to expand.                        */
-	zleparse = 1;
+	lexflags = LEXFLAGS_ACTIVE;
 	lexsave();
 	tmpbuf = (char *)zhalloc(strlen(cc->str) + 5);
 	sprintf(tmpbuf, "foo %s", cc->str); /* KLUDGE! */
@@ -3721,7 +3721,7 @@ makecomplistflags(Compctl cc, char *s, i
 	noaliases = ona;
 	strinend();
 	inpop();
-	errflag = zleparse = 0;
+	errflag = lexflags = 0;
 	lexrestore();
 	/* Fine, now do full expansion. */
 	prefork(foo, 0);
Index: Src/Zle/zle_tricky.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_tricky.c,v
retrieving revision 1.102
diff -p -u -r1.102 zle_tricky.c
--- Src/Zle/zle_tricky.c	22 Mar 2010 16:26:00 -0000	1.102
+++ Src/Zle/zle_tricky.c	13 Dec 2010 18:00:21 -0000
@@ -1140,7 +1140,7 @@ get_comp_string(void)
     zsfree(varname);
     varname = NULL;
     insubscr = 0;
-    zleparse = 1;
+    lexflags = LEXFLAGS_ACTIVE;
     clwpos = -1;
     lexsave();
     inpush(dupstrspace(linptr), 0, NULL);
@@ -1244,7 +1244,7 @@ get_comp_string(void)
 	    if (wordpos != redirpos)
 		wordpos = redirpos = 0;
 	}
-	if (!zleparse && !tt0) {
+	if (!lexflags && !tt0) {
 	    /* This is done when the lexer reached the word the cursor is on. */
 	    tt = tokstr ? dupstring(tokstr) : NULL;
 
@@ -1345,7 +1345,7 @@ get_comp_string(void)
 				 (sl - 1) : (zlemetacs_qsub - wb)]);
 	}
     } while (tok != LEXERR && tok != ENDINPUT &&
-	     (tok != SEPER || (zleparse && !tt0)));
+	     (tok != SEPER || (lexflags && !tt0)));
     /* Calculate the number of words stored in the clwords array. */
     clwnum = (tt || !wordpos) ? wordpos : wordpos - 1;
     zsfree(clwords[clwnum]);
@@ -1360,7 +1360,7 @@ get_comp_string(void)
     }
     strinend();
     inpop();
-    errflag = zleparse = 0;
+    errflag = lexflags = 0;
     if (parbegin != -1) {
 	/* We are in command or process substitution if we are not in
 	 * a $((...)). */
@@ -2707,7 +2707,7 @@ doexpandhist(void)
     noaliases = ona;
     strinend();
     inpop();
-    zleparse = 0;
+    lexflags = 0;
     lexrestore();
     expanding = 0;
 
@@ -2807,7 +2807,7 @@ getcurcmd(void)
     int curlincmd;
     char *s = NULL;
 
-    zleparse = 2;
+    lexflags = LEXFLAGS_ACTIVE;
     lexsave();
     metafy_line();
     inpush(dupstrspace(zlemetaline), 0, NULL);
@@ -2825,11 +2825,11 @@ getcurcmd(void)
 	    cmdwe = zlemetall + 1 - inbufct;
 	}
     }
-    while (tok != ENDINPUT && tok != LEXERR && zleparse);
+    while (tok != ENDINPUT && tok != LEXERR && lexflags);
     popheap();
     strinend();
     inpop();
-    errflag = zleparse = 0;
+    errflag = lexflags = 0;
     unmetafy_line();
     lexrestore();
 
Index: Test/D04parameter.ztst
===================================================================
RCS file: /cvsroot/zsh/zsh/Test/D04parameter.ztst,v
retrieving revision 1.49
diff -p -u -r1.49 D04parameter.ztst
--- Test/D04parameter.ztst	13 Dec 2010 11:20:47 -0000	1.49
+++ Test/D04parameter.ztst	13 Dec 2010 18:00:21 -0000
@@ -462,6 +462,17 @@
 >with
 >comment
 
+  line=$'echo one\necho two # with a comment\necho three'
+  print -l ${(z+nc+)line}
+0:Treating zplit newlines as ordinary whitespace
+>echo
+>one
+>echo
+>two
+># with a comment
+>echo
+>three
+
   psvar=(dog)
   setopt promptsubst
   foo='It shouldn'\''t $(happen) to a %1v.'

-- 
Peter Stephenson <pws@csr.com>            Software Engineer
Tel: +44 (0)1223 692070                   Cambridge Silicon Radio Limited
Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, UK


Member of the CSR plc group of companies. CSR plc registered in England and Wales, registered number 4187346, registered office Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, United Kingdom

Re: Another ${(z)param} buglet

[Peter Stephenson]

On Mon, 13 Dec 2010 18:12:03 +0000
Peter Stephenson <Peter.Stephenson@csr.com> wrote:
> lexsave/lexrestore should probably save and restore lexflags: not done
> that yet. Then we don't need to do it in bufferwords() (which does
> it cleanly already, but goodness knows what ZLE does).

This does that, which enables a small amount of tidying up in ZLE, and
also adds a special flag for ZLE mode which removes the chance of side
effects when using bufferwords() from elsewhere.

Index: Src/hist.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/hist.c,v
retrieving revision 1.108
diff -p -u -r1.108 hist.c
--- Src/hist.c	14 Dec 2010 09:59:04 -0000	1.108
+++ Src/hist.c	14 Dec 2010 10:17:10 -0000
@@ -2897,11 +2897,9 @@ histfileIsLocked(void)
  * If index is non-NULL, and input is from a string in ZLE, *index
  * is set to the position of the end of the current editor word.
  *
- * comments is used if buf is non-NULL (i.e. this is not a string
- * from ZLE).
- * If it is 0, comments are not parsed; they are treated as ordinary words.
- * If it is 1, comments are treated as single strings, one per line.
- * If it is 2, comments are removed.
+ * flags is passed directly to lexflags, see lex.c, except that
+ * we 'or' in the bit LEXFLAGS_ACTIVE to make sure the variable
+ * is set.
  */
 
 /**/
@@ -2909,7 +2907,7 @@ mod_export LinkList
 bufferwords(LinkList list, char *buf, int *index, int flags)
 {
     int num = 0, cur = -1, got = 0, ne = noerrs;
-    int owb = wb, owe = we, oadx = addedx, ozp = lexflags, onc = nocomments;
+    int owb = wb, owe = we, oadx = addedx, onc = nocomments;
     int ona = noaliases, ocs = zlemetacs, oll = zlemetall;
     int forloop = 0, rcquotes = opts[RCQUOTES];
     char *p, *addedspaceptr;
@@ -3120,7 +3118,6 @@ bufferwords(LinkList list, char *buf, in
     strinend();
     inpop();
     errflag = 0;
-    lexflags = ozp;
     nocomments = onc;
     noerrs = ne;
     lexrestore();
Index: Src/lex.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/lex.c,v
retrieving revision 1.60
diff -p -u -r1.60 lex.c
--- Src/lex.c	14 Dec 2010 09:59:04 -0000	1.60
+++ Src/lex.c	14 Dec 2010 10:17:10 -0000
@@ -126,7 +126,7 @@ mod_export int noaliases;
  * Note that although it is passed into the lexer as an input, the
  * lexer can set it to zero after finding the word it's searching for.
  * This only happens if the line being parsed actually does come from
- * ZLE.
+ * ZLE, and hence the bit LEXFLAGS_ZLE is set.
  */
 
 /**/
@@ -202,6 +202,7 @@ struct lexstack {
     int isfirstch;
     int histactive;
     int histdone;
+    int lexflags;
     int stophist;
     int hlinesz;
     char *hline;
@@ -258,6 +259,7 @@ lexsave(void)
     ls->isfirstch = isfirstch;
     ls->histactive = histactive;
     ls->histdone = histdone;
+    ls->lexflags = lexflags;
     ls->stophist = stophist;
     stophist = 0;
     if (!lstack) {
@@ -332,6 +334,7 @@ lexrestore(void)
     isfirstch = lstack->isfirstch;
     histactive = lstack->histactive;
     histdone = lstack->histdone;
+    lexflags = lstack->lexflags;
     stophist = lstack->stophist;
     chline = lstack->hline;
     hptr = lstack->hptr;
@@ -1804,7 +1807,7 @@ exalias(void)
 	} else
 	    zshlextext = tokstr;
 
-	if (lexflags && !(inbufflags & INP_ALIAS)) {
+	if ((lexflags & LEXFLAGS_ZLE) && !(inbufflags & INP_ALIAS)) {
 	    int zp = lexflags;
 
 	    gotword();
Index: Src/zsh.h
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/zsh.h,v
retrieving revision 1.171
diff -p -u -r1.171 zsh.h
--- Src/zsh.h	14 Dec 2010 09:59:04 -0000	1.171
+++ Src/zsh.h	14 Dec 2010 10:17:10 -0000
@@ -1835,13 +1835,19 @@ struct histent {
  */
 #define LEXFLAGS_ACTIVE		0x0001
 /*
+ * Being used from zle.  This is slightly more intrusive
+ * (=> grotesquely non-modular) than use from within
+ * the main shell, so it's a separate flag.
+ */
+#define LEXFLAGS_ZLE		0x0002
+/*
  * Parse comments and treat each comment as a single string
  */
-#define LEXFLAGS_COMMENTS_KEEP	0x0002
+#define LEXFLAGS_COMMENTS_KEEP	0x0004
 /*
  * Parse comments and strip them.
  */
-#define LEXFLAGS_COMMENTS_STRIP	0x0004
+#define LEXFLAGS_COMMENTS_STRIP	0x0008
 /*
  * Either of the above
  */
@@ -1849,7 +1855,7 @@ struct histent {
 /*
  * Treat newlines as whitespace
  */
-#define LEXFLAGS_NEWLINE	0x0008
+#define LEXFLAGS_NEWLINE	0x0010
 
 /******************************************/
 /* Definitions for programable completion */
Index: Src/Zle/compcore.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/compcore.c,v
retrieving revision 1.103
diff -p -u -r1.103 compcore.c
--- Src/Zle/compcore.c	14 Dec 2010 09:59:04 -0000	1.103
+++ Src/Zle/compcore.c	14 Dec 2010 10:17:10 -0000
@@ -1481,13 +1481,13 @@ set_comp_sep(void)
 
     /* Put the string in the lexer buffer and call the lexer to *
      * get the words we have to expand.                        */
-    lexflags = LEXFLAGS_ACTIVE;
     ocs = zlemetacs;
     oll = zlemetall;
     ol = zlemetaline;
     addedx = 1;
     noerrs = 1;
     lexsave();
+    lexflags = LEXFLAGS_ZLE;
     /*
      * tl is the length of the temporary string including
      * the space at the start and the x at the cursor position,
@@ -1634,7 +1634,7 @@ set_comp_sep(void)
     noaliases = ona;
     strinend();
     inpop();
-    errflag = lexflags = 0;
+    errflag = 0;
     noerrs = ne;
     lexrestore();
     wb = owb;
Index: Src/Zle/compctl.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/compctl.c,v
retrieving revision 1.39
diff -p -u -r1.39 compctl.c
--- Src/Zle/compctl.c	14 Dec 2010 09:59:04 -0000	1.39
+++ Src/Zle/compctl.c	14 Dec 2010 10:17:11 -0000
@@ -2789,10 +2789,10 @@ sep_comp_string(char *ss, char *s, int n
 
     /* Put the string in the lexer buffer and call the lexer to *
      * get the words we have to expand.                        */
-    lexflags = LEXFLAGS_ACTIVE;
     addedx = 1;
     noerrs = 1;
     lexsave();
+    lexflags = LEXFLAGS_ZLE;
     tmp = (char *) zhalloc(tl = sl + 3 + strlen(s));
     strcpy(tmp, ss);
     tmp[sl] = ' ';
@@ -2843,7 +2843,7 @@ sep_comp_string(char *ss, char *s, int n
     noaliases = ona;
     strinend();
     inpop();
-    errflag = lexflags = 0;
+    errflag = 0;
     noerrs = ne;
     lexrestore();
     wb = owb;
@@ -3703,8 +3703,8 @@ makecomplistflags(Compctl cc, char *s, i
 
 	/* Put the string in the lexer buffer and call the lexer to *
 	 * get the words we have to expand.                        */
-	lexflags = LEXFLAGS_ACTIVE;
 	lexsave();
+	lexflags = LEXFLAGS_ZLE;
 	tmpbuf = (char *)zhalloc(strlen(cc->str) + 5);
 	sprintf(tmpbuf, "foo %s", cc->str); /* KLUDGE! */
 	inpush(tmpbuf, 0, NULL);
@@ -3721,7 +3721,7 @@ makecomplistflags(Compctl cc, char *s, i
 	noaliases = ona;
 	strinend();
 	inpop();
-	errflag = lexflags = 0;
+	errflag = 0;
 	lexrestore();
 	/* Fine, now do full expansion. */
 	prefork(foo, 0);
Index: Src/Zle/zle_tricky.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_tricky.c,v
retrieving revision 1.103
diff -p -u -r1.103 zle_tricky.c
--- Src/Zle/zle_tricky.c	14 Dec 2010 09:59:04 -0000	1.103
+++ Src/Zle/zle_tricky.c	14 Dec 2010 10:17:11 -0000
@@ -1140,9 +1140,9 @@ get_comp_string(void)
     zsfree(varname);
     varname = NULL;
     insubscr = 0;
-    lexflags = LEXFLAGS_ACTIVE;
     clwpos = -1;
     lexsave();
+    lexflags = LEXFLAGS_ZLE;
     inpush(dupstrspace(linptr), 0, NULL);
     strinbeg(0);
     wordpos = tt0 = cp = rd = ins = oins = linarr = parct = ia = redirpos = 0;
@@ -2707,7 +2707,6 @@ doexpandhist(void)
     noaliases = ona;
     strinend();
     inpop();
-    lexflags = 0;
     lexrestore();
     expanding = 0;
 
@@ -2807,8 +2806,8 @@ getcurcmd(void)
     int curlincmd;
     char *s = NULL;
 
-    lexflags = LEXFLAGS_ACTIVE;
     lexsave();
+    lexflags = LEXFLAGS_ZLE;
     metafy_line();
     inpush(dupstrspace(zlemetaline), 0, NULL);
     strinbeg(1);
@@ -2829,7 +2828,7 @@ getcurcmd(void)
     popheap();
     strinend();
     inpop();
-    errflag = lexflags = 0;
+    errflag = 0;
     unmetafy_line();
     lexrestore();
 
-- 
Peter Stephenson <pws@csr.com>            Software Engineer
Tel: +44 (0)1223 692070                   Cambridge Silicon Radio Limited
Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, UK


Member of the CSR plc group of companies. CSR plc registered in England and Wales, registered number 4187346, registered office Churchill House, Cambridge Business Park, Cowley Road, Cambridge, CB4 0WZ, United Kingdom

Re: Another ${(z)param} buglet

[Bart Schaefer]

On Dec 13,  6:12pm, Peter Stephenson wrote:
} Subject: Re: Another ${(z)param} buglet
}
} On Mon, 13 Dec 2010 09:35:12 -0800
} Bart Schaefer <schaefer@brasslantern.com> wrote:
} > Let's go ahead and use up Z for this so as not to tangle up z with a
} > new more restricted delimiter syntax.  Further let's immediately
} > reserve a character (maybe "+") to have no meaning of its own, but
} > instead to always introduce a delimited string into which we can put 
} > new options.
} 
} Can't see why that shouldn't work.

Here's a patch (using underscore rather than plus for the reserved
meta-flag).

Index: Doc/Zsh/expn.yo
--- zsh-forge/current/Doc/Zsh/expn.yo	2010-12-14 08:13:59.000000000 -0800
+++ Doc/Zsh/expn.yo	2010-12-14 08:54:56.000000000 -0800
@@ -1009,18 +1009,6 @@
 Comments are not treated specially but as ordinary strings, similar
 to interactive shells with the tt(INTERACTIVE_COMMENTS) option unset.
 
-The flag can take a combination of option letters between a following
-pair of `tt(PLUS())' characters.  tt(LPAR()z+PLUS()c+PLUS()RPAR())
-causes comments to be parsed as a string and retained; any field in the
-resulting array beginning with an unquoted comment character is a
-comment.  tt(LPAR()z+PLUS()C+PLUS()RPAR()) causes comments to be parsed
-and removed.  The rule for comments is standard: anything between a word
-starting with the third charcter of tt($HISTCHARS), default tt(#), up to
-the next newline is a comment.  tt(LPAR()z+PLUS()n+PLUS()RPAR()) causes
-unquoted newlines to be treated as ordinary whitespace, else they are
-treated as if they are shell code delimiters and converted to
-semicolons.
-
 Note that this is done very late, as for the `tt((s))' flag. So to
 access single words in the result, one has to use nested expansions as 
 in `tt(${${(z)foo}[2]})'. Likewise, to remove the quotes in the
@@ -1129,6 +1117,25 @@
 empty field.  To override this behaviour, supply the "(@)" flag as well,
 i.e.  tt("${(@s.:.)line}").
 )
+item(tt(Z:)var(opts)tt(:))(
+As tt(z) but takes a combination of option letters between a following
+pair of delimiter characters.  tt(LPAR()Z+PLUS()c+PLUS()RPAR())
+causes comments to be parsed as a string and retained; any field in the
+resulting array beginning with an unquoted comment character is a
+comment.  tt(LPAR()Z+PLUS()C+PLUS()RPAR()) causes comments to be parsed
+and removed.  The rule for comments is standard: anything between a word
+starting with the third charcter of tt($HISTCHARS), default tt(#), up to
+the next newline is a comment.  tt(LPAR()Z+PLUS()n+PLUS()RPAR()) causes
+unquoted newlines to be treated as ordinary whitespace, else they are
+treated as if they are shell code delimiters and converted to
+semicolons.
+)
+item(tt(_:)var(flags)tt(:))(
+The underscore (tt(_)) flag is reserved for future use.  As of this
+revision of zsh, there are no valid var(flags); anything following an
+underscore, other than an empty pair of delimiters, is treated as an
+error, and the flag itself has no effect.
+)
 enditem()
 
 The following flags are meaningful with the tt(${)...tt(#)...tt(}) or
Index: Src/subst.c
--- zsh-forge/current/Src/subst.c	2010-12-14 08:13:59.000000000 -0800
+++ Src/subst.c	2010-12-14 08:28:56.000000000 -0800
@@ -1936,10 +1936,15 @@
 
 		case 'z':
 		    shsplit = LEXFLAGS_ACTIVE;
-		    if (s[1] == '+') {
-			s += 2;
-			while (*s && *s != '+' && *s != ')' && *s != Outpar) {
-			    switch (*s++) {
+		    break;
+
+		case 'Z':
+		    t = get_strarg(++s, &arglen);
+		    if (*t) {
+			sav = *t;
+			*t = 0;
+			while (*++s) {
+			    switch (*s) {
 			    case 'c':
 				/* Parse and keep comments */
 				shsplit |= LEXFLAGS_COMMENTS_KEEP;
@@ -1956,12 +1961,14 @@
 				break;
 
 			    default:
-				goto flagerr;
+				*t = sav;
+ 				goto flagerr;
 			    }
 			}
-			if (*s != '+')
-			    goto flagerr;
-		    }
+			*t = sav;
+			s = t + arglen - 1;
+		    } else
+			goto flagerr;
 		    break;
 
 		case 'u':
@@ -1973,6 +1980,25 @@
 		    evalchar = 1;
 		    break;
 
+		case '_':
+		    t = get_strarg(++s, &arglen);
+		    if (*t) {
+			sav = *t;
+			*t = 0;
+			while (*++s) {
+			    /* Reserved for future use */
+			    switch (*s) {
+			    default:
+				*t = sav;
+				goto flagerr;
+			    }
+			}
+			*t = sav;
+			s = t + arglen - 1;
+		    } else
+			goto flagerr;
+		    break;
+
 		default:
 		  flagerr:
 		    zerr("error in flags");

Index: Test/D04parameter.ztst
--- zsh-forge/current/Test/D04parameter.ztst	2010-12-14 08:14:00.000000000 -0800
+++ Test/D04parameter.ztst	2010-12-14 08:29:28.000000000 -0800
@@ -421,9 +421,9 @@
   print "*** Normal ***"
   print -l ${(z)line}
   print "*** Kept ***"
-  print -l ${(z+c+)line}
+  print -l ${(Z+c+)line}
   print "*** Removed ***"
-  print -l ${(z+C+)line}
+  print -l ${(Z+C+)line}
 0:Comments with (z)
 >*** Normal ***
 >A
@@ -457,13 +457,13 @@
 >one
 
   line='with comment # at the end'
-  print -l ${(z+C+)line}
+  print -l ${(Z+C+)line}
 0:Test we don't get an additional newline token
 >with
 >comment
 
   line=$'echo one\necho two # with a comment\necho three'
-  print -l ${(z+nc+)line}
+  print -l ${(Z+nc+)line}
 0:Treating zplit newlines as ordinary whitespace
 >echo
 >one

--