From b49f856b21eba759e503164a97b61343b9cbbf34 Mon Sep 17 00:00:00 2001
From: Daniel Shahaf <danielsh@apache.org>
Date: Tue, 31 Dec 2019 01:52:03 +0000
Subject: [PATCH 1/2] In the test suite, allow test cases to be marked as
 expected to fail.

See next commit for a use-case.
---
 Test/B01cd.ztst      |  4 +++-
 Test/ztst.zsh        | 30 ++++++++++++++++++++++++++++--
 Util/ztst-syntax.vim |  2 +-
 3 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/Test/B01cd.ztst b/Test/B01cd.ztst
index f79e18f..977cbdf 100644
--- a/Test/B01cd.ztst
+++ b/Test/B01cd.ztst
@@ -70,11 +70,13 @@
 # the expected status returned by the code when run, or - if it is
 # irrelevant.  An optional set of single-letter flags follows the status
 # or -.  The following are understood:
-#  . d   Don't diff stdout against the expected stdout.
+#  . d  Don't diff stdout against the expected stdout.
 #   D   Don't diff stderr against the expected stderr.
 #   q   All redirection lines given in the test script (not the lines
 #       actually produced by the test) are subject to ordinary quoted shell
 #       expansion (i.e. not globbing).
+#   f   Test is expected to fail. If the test's exit code, stdout, and stderr
+#       match, report a problem; otherwise, carry on to the next test case.
 # This can be followed by a `:' and a message describing the
 # test, which will be printed if the test fails, along with a
 # description of the failure that occurred.  The `:' and message are
diff --git a/Test/ztst.zsh b/Test/ztst.zsh
index a4c6252..0c71f9d 100755
--- a/Test/ztst.zsh
+++ b/Test/ztst.zsh
@@ -366,6 +366,7 @@ ZTST_test() {
   local last match mbegin mend found substlines
   local diff_out diff_err
   local ZTST_skip
+  integer expected_to_fail
 
   while true; do
     rm -f $ZTST_in $ZTST_out $ZTST_err
@@ -460,8 +461,21 @@ $ZTST_curline"
 	fi
       fi
 
+      if [[ $ZTST_flags = *f* ]]; then
+        expected_to_fail=1
+        ZTST_xfail_diff() { ZTST_diff "$@" > /dev/null }
+        ZTST_diff=ZTST_xfail_diff
+      else
+        expected_to_fail=0
+        ZTST_diff=ZTST_diff
+      fi
+
       # First check we got the right status, if specified.
       if [[ $ZTST_xstatus != - && $ZTST_xstatus != $ZTST_status ]]; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	ZTST_testfailed "bad status $ZTST_status, expected $ZTST_xstatus from:
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
@@ -480,7 +494,11 @@ $(<$ZTST_terr)"
 	rm -rf $ZTST_out
 	print -r -- "${(e)substlines}" >$ZTST_out
       fi
-      if [[ $ZTST_flags != *d* ]] && ! ZTST_diff $diff_out -u $ZTST_out $ZTST_tout; then
+      if [[ $ZTST_flags != *d* ]] && ! $ZTST_diff $diff_out -u $ZTST_out $ZTST_tout; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	ZTST_testfailed "output differs from expected as shown above for:
 $ZTST_code${$(<$ZTST_terr):+
 Error output:
@@ -488,15 +506,23 @@ $(<$ZTST_terr)}"
 	return 1
       fi
       if [[ $ZTST_flags = *q* && -s $ZTST_err ]]; then
+        if (( expected_to_fail )); then
+          ZTST_verbose 1 "Test failed, as expected."
+          continue
+        fi
 	substlines="$(<$ZTST_err)"
 	rm -rf $ZTST_err
 	print -r -- "${(e)substlines}" >$ZTST_err
       fi
-      if [[ $ZTST_flags != *D* ]] && ! ZTST_diff $diff_err -u $ZTST_err $ZTST_terr; then
+      if [[ $ZTST_flags != *D* ]] && ! $ZTST_diff $diff_err -u $ZTST_err $ZTST_terr; then
 	ZTST_testfailed "error output differs from expected as shown above for:
 $ZTST_code"
 	return 1
       fi
+      if (( expected_to_fail )); then
+        ZTST_testfailed "test was expected to fail, but passed."
+        return 1
+      fi
     fi
     ZTST_verbose 1 "Test successful."
     [[ -n $last ]] && break
diff --git a/Util/ztst-syntax.vim b/Util/ztst-syntax.vim
index 8d8becf..f0c82d7 100644
--- a/Util/ztst-syntax.vim
+++ b/Util/ztst-syntax.vim
@@ -28,7 +28,7 @@ syn include @zsh                   syntax/zsh.vim
 syn match  ztstPayload             /^\s\+\zs.*/ contains=@zsh
 
 syn match  ztstExitCode            /^\d\+\|^-/                nextgroup=ztstFlags
-syn match  ztstFlags               /[.dDq]*:/       contained nextgroup=ztstTestName contains=ztstColon
+syn match  ztstFlags               /[.dDqf]*:/      contained nextgroup=ztstTestName contains=ztstColon
 syn match  ztstColon               /:/              contained
 syn region ztstTestName            start=// end=/$/ contained 
 
-- 
2.20.1