Global alias mangles input in 5.0.8

Global alias mangles input in 5.0.8

[fcook]

[-- Attachment #1: Type: text/plain, Size: 1201 bytes --]

I'm running on x86-64 Arch Linux, and noticing what looks like a bug doing the following:



> alias -g '$'='4<<('

> cat /proc/self/fd/4 $ ls )


The behavior of setting and using an alias like this in zsh 5.0.7 is relatively sane: the result of ls is piped through cat.



But under zsh 5.0.8, I get an error: "zsh: no such file or directory: s )".


I really don't understand why the first letter of "ls" is being removed or why the space and the closing paren are considered part of the same token as the 's'.


To further confuse matters, the precise filename in the error depends on how long the expansion of the alias is. If I add a space, and set it to '4 <<(' (or, to identical effect, put the space in front of the 4 instead of after), the file not found is " )" (without the 's'). Another space and the file is just ")", and with three spaces zsh is failing to find a file named the empty string.


Beyond three spaces I get "zsh: unterminated `<(...)'", which is surprising since the right paren is fairly clearly visible in the line to alias-expand.


For what it's worth, I see the same behavior with a non-special token being aliased, such as 'x' instead of '$'.


Thanks for your time.

Re: Global alias mangles input in 5.0.8

[Peter Stephenson]

On Wed, 1 Jul 2015 00:48:23 -0400
fcook <fcook@protonmail.com> wrote:
> I'm running on x86-64 Arch Linux, and noticing what looks like a bug doing the following:
> 
> > alias -g '$'='4<<('
> 
> > cat /proc/self/fd/4 $ ls )
> 
> But under zsh 5.0.8, I get an error: "zsh: no such file or directory: s )".

Looks like bad effects are specific to "<(...)", since I seem to be getting
sane results with "$(...)", but they're not specific to global aliases...

% alias foo='echo <('
% foo ls )
zsh: segmentation fault (core dumped)  ./zsh

I don't know if this is the same, but it seems likely to be related.

pws

Re: Global alias mangles input in 5.0.8

[Peter Stephenson]

On Wed, 01 Jul 2015 10:23:57 +0100
Peter Stephenson <p.stephenson@samsung.com> wrote:
> On Wed, 1 Jul 2015 00:48:23 -0400
> fcook <fcook@protonmail.com> wrote:
> > I'm running on x86-64 Arch Linux, and noticing what looks like a bug doing the following:
> > 
> > > alias -g '$'='4<<('
> > 
> > > cat /proc/self/fd/4 $ ls )
> > 
> > But under zsh 5.0.8, I get an error: "zsh: no such file or directory: s )".
> 
> Looks like bad effects are specific to "<(...)", since I seem to be getting
> sane results with "$(...)", but they're not specific to global aliases...
> 
> % alias foo='echo <('
> % foo ls )
> zsh: segmentation fault (core dumped)  ./zsh

I'm not sure why they don't happen with $(, but the reason is that we
have started to parse the complete (...) as a string so we can match
the correct closing parenthesis, and then we encounter the end of the
alias.  We backtrack because alias expansion occurs after the lexer,
while we're tracking the string at the level of input --- however, in
this case the alias was already being expanded when we found the "(",
so we backtrack too far.  The fix is not to do that.

Hope that's clear.

We didn't test this because it's an incredibly strange thing to do.

Luckily, there's absolutely no possibility of knock-on effects when
changing this code.  Ever.

Mikael...?

pws

diff --git a/Src/lex.c b/Src/lex.c
index baeed13..910773c 100644
--- a/Src/lex.c
+++ b/Src/lex.c
@@ -162,7 +162,7 @@ static int lex_add_raw;
 
 /* variables associated with the above */
 
-static char *tokstr_raw;
+static char *tokstr_raw, *lexbuf_ptr_start;
 static struct lexbufstate lexbuf_raw;
 
 /* text of punctuation tokens */
@@ -229,12 +229,13 @@ lex_context_save(struct lex_stack *ls, int toplevel)
     ls->lex_add_raw = lex_add_raw;
     ls->tokstr_raw = tokstr_raw;
     ls->lexbuf_raw = lexbuf_raw;
+    ls->lexbuf_ptr_start = lexbuf_ptr_start;
     ls->lexstop = lexstop;
     ls->toklineno = toklineno;
 
     tokstr = zshlextext = lexbuf.ptr = NULL;
     lexbuf.siz = 256;
-    tokstr_raw = lexbuf_raw.ptr = NULL;
+    tokstr_raw = lexbuf_raw.ptr = lexbuf_ptr_start = NULL;
     lexbuf_raw.siz = lexbuf_raw.len = lex_add_raw = 0;
 }
 
@@ -257,6 +258,7 @@ lex_context_restore(const struct lex_stack *ls, int toplevel)
     lex_add_raw = ls->lex_add_raw;
     tokstr_raw = ls->tokstr_raw;
     lexbuf_raw = ls->lexbuf_raw;
+    lexbuf_ptr_start = ls->lexbuf_ptr_start;
     lexstop = ls->lexstop;
     toklineno = ls->toklineno;
 }
@@ -1882,7 +1884,7 @@ zshlex_raw_add(int c)
 void
 zshlex_raw_back(void)
 {
-    if (!lex_add_raw)
+    if (!lex_add_raw || lexbuf_raw.ptr == lexbuf_ptr_start)
 	return;
     lexbuf_raw.ptr--;
     lexbuf_raw.len--;
@@ -1993,7 +1995,7 @@ skipcomm(void)
     cmdpop();
     return lexstop;
 #else
-    char *new_tokstr;
+    char *new_tokstr, *new_lexbuf_ptr_start;
     int new_lexstop, new_lex_add_raw;
     struct lexbufstate new_lexbuf;
 
@@ -2040,6 +2042,7 @@ skipcomm(void)
     }
     tokstr_raw = new_tokstr;
     lexbuf_raw = new_lexbuf;
+    lexbuf_ptr_start = lexbuf_raw.ptr;
     lex_add_raw = new_lex_add_raw;
     /*
      * Don't do any ZLE specials down here: they're only needed
@@ -2064,6 +2067,7 @@ skipcomm(void)
      */
     new_tokstr = tokstr_raw;
     new_lexbuf = lexbuf_raw;
+    new_lexbuf_ptr_start = lexbuf_ptr_start;
     /*
      * We're also going to propagate the lexical state:
      * if we couldn't parse the command substitution we
@@ -2079,6 +2083,7 @@ skipcomm(void)
 	 */
 	tokstr_raw = new_tokstr;
 	lexbuf_raw = new_lexbuf;
+	lexbuf_ptr_start = new_lexbuf_ptr_start;
     } else {
 	if (!new_lexstop) {
 	    /* Ignore the ')' added on input */
@@ -2093,6 +2098,7 @@ skipcomm(void)
 	tokstr = new_tokstr;
 	lexbuf = new_lexbuf;
 	lexstop = new_lexstop;
+	lexbuf_ptr_start = (char *)NULL;
 	hist_in_word(0);
     }
 
diff --git a/Src/zsh.h b/Src/zsh.h
index 183620f..d11d4fe 100644
--- a/Src/zsh.h
+++ b/Src/zsh.h
@@ -2801,6 +2801,7 @@ struct lex_stack {
     int lex_add_raw;
     char *tokstr_raw;
     struct lexbufstate lexbuf_raw;
+    char *lexbuf_ptr_start;
     int lexstop;
     zlong toklineno;
 };
diff --git a/Test/D03procsubst.ztst b/Test/D03procsubst.ztst
index c763f6e..07ec639 100644
--- a/Test/D03procsubst.ztst
+++ b/Test/D03procsubst.ztst
@@ -88,6 +88,7 @@
   print something=${:-=(echo 'C,D),(F,G)'}
 1: Graceful handling of bad substitution in enclosed context
 ?(eval):1: unterminated `=(...)'
+# '`
 
   () {
      print -n "first: "
@@ -115,3 +116,8 @@
 0:Process substitution as anonymous function argument
 >Execute a complicated order first
 >This line was brought to you by the letters F and D
+
+  alias foo='cat <('
+  eval 'foo echo this is bound to work)'
+0:backtacking within command string parsing with alias still pending
+>this is bound to work
diff --git a/Test/D08cmdsubst.ztst b/Test/D08cmdsubst.ztst
index a4c69a0..89e7259 100644
--- a/Test/D08cmdsubst.ztst
+++ b/Test/D08cmdsubst.ztst
@@ -148,3 +148,8 @@
  ) after
 0:Parsing of command substitution with ummatched parentheses: with frills
 >before start Universe began with u and ended with a crunch end after
+
+  alias foo='echo $('
+  eval 'foo echo this just works, OK\?)'
+0:backtracking within command string parsing with alias still pending
+>this just works, OK?

Re: Global alias mangles input in 5.0.8

[Peter Stephenson]

On Wed, 1 Jul 2015 21:26:12 +0100
Peter Stephenson <p.w.stephenson@ntlworld.com> wrote:
> On Wed, 01 Jul 2015 10:23:57 +0100
> Peter Stephenson <p.stephenson@samsung.com> wrote:
> > On Wed, 1 Jul 2015 00:48:23 -0400
> > fcook <fcook@protonmail.com> wrote:
> > > I'm running on x86-64 Arch Linux, and noticing what looks like a bug doing the following:
> > > 
> > > > alias -g '$'='4<<('
> > > 
> > > > cat /proc/self/fd/4 $ ls )
> > > 
> > > But under zsh 5.0.8, I get an error: "zsh: no such file or directory: s )".
> 
> I'm not sure why they don't happen with $(, but the reason is that we
> have started to parse the complete (...) as a string so we can match
> the correct closing parenthesis, and then we encounter the end of the
> alias.  We backtrack because alias expansion occurs after the lexer,
> while we're tracking the string at the level of input --- however, in
> this case the alias was already being expanded when we found the "(",
> so we backtrack too far.  The fix is not to do that.

OK, I've found the mistake.  One of the mistakes.

In this case we shouldn't actually backtrack at all, not only not beyond
the start, because our text is coming from the entire remainder of the
alias.  The previous fix failed in the case shown in the new test.

This one is slightly simpler anyway (single-line changes in three
places, not counting removing the old patch).

pws


diff --git a/Src/input.c b/Src/input.c
index 4a5bf89..613f89a 100644
--- a/Src/input.c
+++ b/Src/input.c
@@ -591,7 +591,7 @@ inpoptop(void)
 	     * history is before, but they're both pushed onto
 	     * the input stack.
 	     */
-	    if ((inbufflags & (INP_ALIAS|INP_HIST)) == INP_ALIAS)
+	    if ((inbufflags & (INP_ALIAS|INP_HIST|INP_RAW_KEEP)) == INP_ALIAS)
 		zshlex_raw_back();
 	}
     }
diff --git a/Src/lex.c b/Src/lex.c
index 910773c..b0cd963 100644
--- a/Src/lex.c
+++ b/Src/lex.c
@@ -162,7 +162,7 @@ static int lex_add_raw;
 
 /* variables associated with the above */
 
-static char *tokstr_raw, *lexbuf_ptr_start;
+static char *tokstr_raw;
 static struct lexbufstate lexbuf_raw;
 
 /* text of punctuation tokens */
@@ -229,13 +229,12 @@ lex_context_save(struct lex_stack *ls, int toplevel)
     ls->lex_add_raw = lex_add_raw;
     ls->tokstr_raw = tokstr_raw;
     ls->lexbuf_raw = lexbuf_raw;
-    ls->lexbuf_ptr_start = lexbuf_ptr_start;
     ls->lexstop = lexstop;
     ls->toklineno = toklineno;
 
     tokstr = zshlextext = lexbuf.ptr = NULL;
     lexbuf.siz = 256;
-    tokstr_raw = lexbuf_raw.ptr = lexbuf_ptr_start = NULL;
+    tokstr_raw = lexbuf_raw.ptr = NULL;
     lexbuf_raw.siz = lexbuf_raw.len = lex_add_raw = 0;
 }
 
@@ -258,7 +257,6 @@ lex_context_restore(const struct lex_stack *ls, int toplevel)
     lex_add_raw = ls->lex_add_raw;
     tokstr_raw = ls->tokstr_raw;
     lexbuf_raw = ls->lexbuf_raw;
-    lexbuf_ptr_start = ls->lexbuf_ptr_start;
     lexstop = ls->lexstop;
     toklineno = ls->toklineno;
 }
@@ -1884,7 +1882,7 @@ zshlex_raw_add(int c)
 void
 zshlex_raw_back(void)
 {
-    if (!lex_add_raw || lexbuf_raw.ptr == lexbuf_ptr_start)
+    if (!lex_add_raw)
 	return;
     lexbuf_raw.ptr--;
     lexbuf_raw.len--;
@@ -1995,7 +1993,7 @@ skipcomm(void)
     cmdpop();
     return lexstop;
 #else
-    char *new_tokstr, *new_lexbuf_ptr_start;
+    char *new_tokstr;
     int new_lexstop, new_lex_add_raw;
     struct lexbufstate new_lexbuf;
 
@@ -2023,6 +2021,18 @@ skipcomm(void)
 	new_tokstr = tokstr;
 	new_lexbuf = lexbuf;
 
+	/*
+	 * If we're expanding an alias at this point, we need the whole
+	 * remaining text as part of the string for the command in
+	 * parentheses, so don't backtrack.  This is different from the
+	 * usual case where the alias is fully within the command, where
+	 * we want the unexpanded text so that it will be expanded
+	 * again when the command in the parentheses is executed.
+	 *
+	 * I never wanted to be a software engineer, you know.
+	 */
+	if (inbufflags & INP_ALIAS)
+	    inbufflags |= INP_RAW_KEEP;
 	zcontext_save_partial(ZCONTEXT_LEX|ZCONTEXT_PARSE);
 	hist_in_word(1);
     } else {
@@ -2042,7 +2052,6 @@ skipcomm(void)
     }
     tokstr_raw = new_tokstr;
     lexbuf_raw = new_lexbuf;
-    lexbuf_ptr_start = lexbuf_raw.ptr;
     lex_add_raw = new_lex_add_raw;
     /*
      * Don't do any ZLE specials down here: they're only needed
@@ -2067,7 +2076,6 @@ skipcomm(void)
      */
     new_tokstr = tokstr_raw;
     new_lexbuf = lexbuf_raw;
-    new_lexbuf_ptr_start = lexbuf_ptr_start;
     /*
      * We're also going to propagate the lexical state:
      * if we couldn't parse the command substitution we
@@ -2083,7 +2091,6 @@ skipcomm(void)
 	 */
 	tokstr_raw = new_tokstr;
 	lexbuf_raw = new_lexbuf;
-	lexbuf_ptr_start = new_lexbuf_ptr_start;
     } else {
 	if (!new_lexstop) {
 	    /* Ignore the ')' added on input */
@@ -2098,7 +2105,6 @@ skipcomm(void)
 	tokstr = new_tokstr;
 	lexbuf = new_lexbuf;
 	lexstop = new_lexstop;
-	lexbuf_ptr_start = (char *)NULL;
 	hist_in_word(0);
     }
 
diff --git a/Src/zsh.h b/Src/zsh.h
index d11d4fe..69fef33 100644
--- a/Src/zsh.h
+++ b/Src/zsh.h
@@ -425,6 +425,7 @@ enum {
 #define INP_HISTCONT  (1<<5)	/* stack is continued from history expn.   */
 #define INP_LINENO    (1<<6)    /* update line number                      */
 #define INP_APPEND    (1<<7)    /* Append new lines to allow backup        */
+#define INP_RAW_KEEP  (1<<8)    /* Input needed in raw mode even if alias  */
 
 /* Flags for metafy */
 #define META_REALLOC	0
@@ -2801,7 +2802,6 @@ struct lex_stack {
     int lex_add_raw;
     char *tokstr_raw;
     struct lexbufstate lexbuf_raw;
-    char *lexbuf_ptr_start;
     int lexstop;
     zlong toklineno;
 };
diff --git a/Test/D03procsubst.ztst b/Test/D03procsubst.ztst
index 07ec639..7b87589 100644
--- a/Test/D03procsubst.ztst
+++ b/Test/D03procsubst.ztst
@@ -121,3 +121,8 @@
   eval 'foo echo this is bound to work)'
 0:backtacking within command string parsing with alias still pending
 >this is bound to work
+
+  alias foo='cat <( print'
+  eval 'foo here is some output)'
+0:full alias expanded when substitution starts in alias
+>here is some output