[BUG] General 256 colors bug – zle_highlight / fg_start_code, etc. is not respected (was: highlight test cases)

[Sebastian Gniazdowski <sgniazdowski@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 1254 bytes --]

On Fri, 30 Nov 2018 at 01:34, Sebastian Gniazdowski
<sgniazdowski@gmail.com> wrote:
> The output doesn't follow zsh_highlight replacements for start-code
> and end-code and still emits raw codes.
>
> Could this be fixed?

Turns out this doesn't involve near-color at all, only 256 color
codes. The new standout-targeted test returns:

-0m27m24mtr7mu27me word2 word3
+0m27m24m38;5;196mtr7m38;5;196mu27meCDE|39| word2 word3
Was testing: region highlight - standout overlapping on other
region_highlight entry

So again no CDE|3...|, but raw ^[38;5;196m. So this is a general Zsh
bug, not near-color bug, as zsh/near-color module isn't loaded in this
test, it's only the 256 code "fg=196" that is being used.

The setting: zle_highlight=( fg_start_code:"CDE|3" fg_end_code:"|"
bg_start_code:"BCDE|4" bg_end_code:"|" )

The file *-256.ztst.txt contains this test case. The file
*-standout-only.ztst.txt is the expected test case which shows no
problems with standout-overlaping other styles.

PS. I now also do not remove 2xm codes (underline, standaout) but the
test cases are all adapted to this change.

-- 
Sebastian Gniazdowski
News: https://twitter.com/ZdharmaI
IRC: https://kiwiirc.com/client/chat.freenode.net:+6697/#zplugin
Blog: http://zdharma.org

[-- Attachment #2: X04zlehighlight-256.ztst.txt --]
[-- Type: text/plain, Size: 6383 bytes --]

# Tests for region_highlight, true-color support, near-color support
# Version 0.7 2018-12-06
%prep

  export TERM=xterm-256color
  if [[ ${+termcap} != 1 || ${termcap[Co]} != <-> || ${termcap[Co]} -lt 256 ]]; then
    ZTST_unimplemented="no termcap module OR termcap doesn't support 256 or more colors"
  elif [[ $OSTYPE == cygwin ]]; then
    ZTST_unimplemented='the zsh/zpty module does not work on Cygwin'
  elif zmodload zsh/zpty 2> /dev/null; then
    zpty_start() {
      export PS1= PS2=
      zpty -d
      zpty zsh "${(q)ZTST_testdir}/../Src/zsh -fiV +Z"
      zpty -w zsh 'zle_highlight=( fg_start_code:"CDE|3" fg_end_code:"|" bg_start_code:"BCDE|4" bg_end_code:"|" )'
    }
    zpty_input() {
      zpty ${${(M)2:#nonl}:+-n} -w zsh "$1"
    }
    zpty_enable_zle() {
      zpty -w zsh "tcfunc() { REPLY=""; }"
      # This line will not be echoed back, behaving like ! -o zle
      zpty -w zsh "setopt zle; zle -T tc tcfunc; unset zle_bracketed_paste"
    }
    zpty_line() {
      setopt localoptions extendedglob noshwordsplit
      local REPLY cm=$'\r'
      integer i
      for (( i = 0; i < ${1:-1}; ++i )); do
        zpty -r zsh REPLY
        # P is for "preserve", and induces keeping some
        # expected color codes to test region_highlight,
        # etc. - the color codes are made a regular text.
        [[ "$2" = "p" ]] && {
            REPLY=${REPLY//(#b)$'\x1b'\[([0-9;]##m)/${match[1]}}
        } || {
            REPLY=${REPLY//$'\x1b'\[[0-9;]##m/}   # remove all [0-9]...m codes
        }
        # Fix e^Mexit - match ((?)\r(?)), if \2 == \3, then replace with \2
        # otherwise replace with \1 stripped out of leading/trailing [[:space:]]
        REPLY=${REPLY//(#b)((?(#c0,1))$cm(?(#c0,1)))/${${${(M)match[2]:#${match[3]}}:+${match[2]}}:-${${match[1]##[[:space:]]##}%%[[:space:]]##}}}
        [[ -n "$REPLY" ]] && print -r -- ${${REPLY%%[[:space:]]##}##[[:space:]]##}
      done
    }
    zpty_stop() {
      setopt localoptions extendedglob
      local REPLY cm=$'\r'
      # Zle is active, can use Ctrl-D to exit
      zpty -n -w zsh $'\C-d'
      # zpty gives no output when piped without these braces (?)
      # The while loop with // substitution is to convert `e^Mexit'
      # into `exit' (see zpty_line). The sed commands remove escapes
      { zpty -r zsh } | sed -e $'/[^\t\r ]/!d' -e $'s/\r$//' -e $'s/\x1b\\[[0-9;]*m//g' | while read REPLY; do REPLY=${REPLY//(#b)((?(#c0,1))$cm(?(#c0,1)))/${${${(M)match[2]:#${match[3]}}:+${match[2]}}:-${${match[1]##[[:space:]]##}%%[[:space:]]##}}}; print -rn -- "$REPLY"; done
      zpty -d
      :
    }
  else
    ZTST_unimplemented='the zsh/zpty module is not available'
  fi

%test

  zpty_start
  zpty_input 'rh_widget() { BUFFER="true word2 word3"; region_highlight+=( "0 4 fg=196" ); rh2; }'
  zpty_input 'rh2() { region_highlight+=( "2 3 standout" ); };'
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:region highlight - standout overlapping on other region_highlight entry
>0m27m24mtr7mu27me word2 word3

  zpty_start
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=green" ); }'
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:basic region_highlight with 8 colors
>0m27m24mCDE|32|trueCDE|39|

  zpty_start
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#040810" ); }'
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:basic region_highlight with true-color (hex-triplets)
>0m27m24mCDE|38;2;4;8;16|trueCDE|39|

  zpty_start
  zpty_input 'zmodload zsh/nearcolor'
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#040810" ); }'
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:basic region_highlight with near-color (hex-triplets at input)
>0m27m24mCDE|38;5;232|trueCDE|39|

  zpty_start
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=green" ); rh2; }'
  zpty_input 'rh2() { region_highlight+=( "1 2 fg=red" ); }' # `r' in red; the above line would be too long
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:overlapping region_highlight with 8 colors
>0m27m24mCDE|32|tCDE|31|rCDE|39|CDE|32|ueCDE|39|

  zpty_start
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#00cc00" ); rh2; }'
  zpty_input 'rh2() { region_highlight+=( "1 2 fg=#cc0000" ); }' # `r' in red; the above line would be too long
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:overlapping region_highlight with true-color
>0m27m24mCDE|38;2;0;204;0|tCDE|38;2;204;0;0|rCDE|39|CDE|38;2;0;204;0|ueCDE|39|

  zpty_start
  zpty_input 'zmodload zsh/nearcolor'
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#00cc00" ); rh2; }'
  zpty_input 'rh2() { region_highlight+=( "1 2 fg=#cc0000" ); }' # `r' in red; the above line would be too long
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:overlapping region_highlight with near-color (hex-triplets at input)
>0m27m24mCDE|38;5;40|tCDE|38;5;160|rCDE|39|CDE|38;5;40|ueCDE|39|

%clean

  zmodload -ui zsh/zpty

# vim:ft=zsh

[-- Attachment #3: X04zlehighlight-standout-only.ztst.txt --]
[-- Type: text/plain, Size: 6403 bytes --]

# Tests for region_highlight, true-color support, near-color support
# Version 0.7 2018-12-06
%prep

  export TERM=xterm-256color
  if [[ ${+termcap} != 1 || ${termcap[Co]} != <-> || ${termcap[Co]} -lt 256 ]]; then
    ZTST_unimplemented="no termcap module OR termcap doesn't support 256 or more colors"
  elif [[ $OSTYPE == cygwin ]]; then
    ZTST_unimplemented='the zsh/zpty module does not work on Cygwin'
  elif zmodload zsh/zpty 2> /dev/null; then
    zpty_start() {
      export PS1= PS2=
      zpty -d
      zpty zsh "${(q)ZTST_testdir}/../Src/zsh -fiV +Z"
      zpty -w zsh 'zle_highlight=( fg_start_code:"CDE|3" fg_end_code:"|" bg_start_code:"BCDE|4" bg_end_code:"|" )'
    }
    zpty_input() {
      zpty ${${(M)2:#nonl}:+-n} -w zsh "$1"
    }
    zpty_enable_zle() {
      zpty -w zsh "tcfunc() { REPLY=""; }"
      # This line will not be echoed back, behaving like ! -o zle
      zpty -w zsh "setopt zle; zle -T tc tcfunc; unset zle_bracketed_paste"
    }
    zpty_line() {
      setopt localoptions extendedglob noshwordsplit
      local REPLY cm=$'\r'
      integer i
      for (( i = 0; i < ${1:-1}; ++i )); do
        zpty -r zsh REPLY
        # P is for "preserve", and induces keeping some
        # expected color codes to test region_highlight,
        # etc. - the color codes are made a regular text.
        [[ "$2" = "p" ]] && {
            REPLY=${REPLY//(#b)$'\x1b'\[([0-9;]##m)/${match[1]}}
        } || {
            REPLY=${REPLY//$'\x1b'\[[0-9;]##m/}   # remove all [0-9]...m codes
        }
        # Fix e^Mexit - match ((?)\r(?)), if \2 == \3, then replace with \2
        # otherwise replace with \1 stripped out of leading/trailing [[:space:]]
        REPLY=${REPLY//(#b)((?(#c0,1))$cm(?(#c0,1)))/${${${(M)match[2]:#${match[3]}}:+${match[2]}}:-${${match[1]##[[:space:]]##}%%[[:space:]]##}}}
        [[ -n "$REPLY" ]] && print -r -- ${${REPLY%%[[:space:]]##}##[[:space:]]##}
      done
    }
    zpty_stop() {
      setopt localoptions extendedglob
      local REPLY cm=$'\r'
      # Zle is active, can use Ctrl-D to exit
      zpty -n -w zsh $'\C-d'
      # zpty gives no output when piped without these braces (?)
      # The while loop with // substitution is to convert `e^Mexit'
      # into `exit' (see zpty_line). The sed commands remove escapes
      { zpty -r zsh } | sed -e $'/[^\t\r ]/!d' -e $'s/\r$//' -e $'s/\x1b\\[[0-9;]*m//g' | while read REPLY; do REPLY=${REPLY//(#b)((?(#c0,1))$cm(?(#c0,1)))/${${${(M)match[2]:#${match[3]}}:+${match[2]}}:-${${match[1]##[[:space:]]##}%%[[:space:]]##}}}; print -rn -- "$REPLY"; done
      zpty -d
      :
    }
  else
    ZTST_unimplemented='the zsh/zpty module is not available'
  fi

%test

  zpty_start
  zpty_input 'rh_widget() { BUFFER="true word2 word3"; region_highlight+=( "0 4 fg=196" ); rh2; }'
  zpty_input 'rh2() { region_highlight=( "2 3 standout" ); };' # note the =, not +=
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:region highlight - standout overlapping on other region_highlight entry
>0m27m24mtr7mu27me word2 word3

  zpty_start
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=green" ); }'
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:basic region_highlight with 8 colors
>0m27m24mCDE|32|trueCDE|39|

  zpty_start
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#040810" ); }'
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:basic region_highlight with true-color (hex-triplets)
>0m27m24mCDE|38;2;4;8;16|trueCDE|39|

  zpty_start
  zpty_input 'zmodload zsh/nearcolor'
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#040810" ); }'
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:basic region_highlight with near-color (hex-triplets at input)
>0m27m24mCDE|38;5;232|trueCDE|39|

  zpty_start
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=green" ); rh2; }'
  zpty_input 'rh2() { region_highlight+=( "1 2 fg=red" ); }' # `r' in red; the above line would be too long
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:overlapping region_highlight with 8 colors
>0m27m24mCDE|32|tCDE|31|rCDE|39|CDE|32|ueCDE|39|

  zpty_start
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#00cc00" ); rh2; }'
  zpty_input 'rh2() { region_highlight+=( "1 2 fg=#cc0000" ); }' # `r' in red; the above line would be too long
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:overlapping region_highlight with true-color
>0m27m24mCDE|38;2;0;204;0|tCDE|38;2;204;0;0|rCDE|39|CDE|38;2;0;204;0|ueCDE|39|

  zpty_start
  zpty_input 'zmodload zsh/nearcolor'
  zpty_input 'rh_widget() { BUFFER="true"; region_highlight+=( "0 4 fg=#00cc00" ); rh2; }'
  zpty_input 'rh2() { region_highlight+=( "1 2 fg=#cc0000" ); }' # `r' in red; the above line would be too long
  zpty_input 'zle -N rh_widget'
  zpty_input 'bindkey "\C-a" rh_widget'
  zpty_enable_zle
  zpty_input $'\C-a'  # emits newline, which executes BUFFER="true" command
  zpty_line 1 p       # the line of interest, preserving escapes ("p")
  zpty_stop
0:overlapping region_highlight with near-color (hex-triplets at input)
>0m27m24mCDE|38;5;40|tCDE|38;5;160|rCDE|39|CDE|38;5;40|ueCDE|39|

%clean

  zmodload -ui zsh/zpty

# vim:ft=zsh