[PATCH] improve error handling of ttyname_r and isatty

[PATCH] improve error handling of ttyname_r and isatty

[Benjamin Peterson]

POSIX allows ttyname(_r) and isatty to return EBADF if passed file descriptor is
invalid.
---
 src/unistd/isatty.c    | 6 +++++-
 src/unistd/ttyname_r.c | 5 ++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/unistd/isatty.c b/src/unistd/isatty.c
index c8badaf5..3d5213c6 100644
--- a/src/unistd/isatty.c
+++ b/src/unistd/isatty.c
@@ -1,9 +1,13 @@
 #include <unistd.h>
+#include <errno.h>
 #include <sys/ioctl.h>
 #include "syscall.h"
 
 int isatty(int fd)
 {
 	struct winsize wsz;
-	return !__syscall(SYS_ioctl, fd, TIOCGWINSZ, &wsz);
+	unsigned long r = __syscall_ret(__syscall(SYS_ioctl, fd, TIOCGWINSZ, &wsz));
+	if (r == 0) return 1;
+	if (errno != EBADF) errno = ENOTTY;
+	return 0;
 }
diff --git a/src/unistd/ttyname_r.c b/src/unistd/ttyname_r.c
index 33aa4ae1..3f836a6a 100644
--- a/src/unistd/ttyname_r.c
+++ b/src/unistd/ttyname_r.c
@@ -10,7 +10,10 @@ int ttyname_r(int fd, char *name, size_t size)
 	char procname[sizeof "/proc/self/fd/" + 3*sizeof(int) + 2];
 	ssize_t l;
 
-	if (!isatty(fd)) return ENOTTY;
+	if (!isatty(fd)) {
+		if (errno != EBADF) errno = ENOTTY;
+		return errno;
+	}
 
 	__procfdname(procname, fd);
 	l = readlink(procname, name, size);
-- 
2.17.1

Re: [PATCH] improve error handling of ttyname_r and isatty

[Rich Felker]

On Thu, Sep 13, 2018 at 09:27:55AM -0700, Benjamin Peterson wrote:
> POSIX allows ttyname(_r) and isatty to return EBADF if passed file descriptor is
> invalid.
> ---
>  src/unistd/isatty.c    | 6 +++++-
>  src/unistd/ttyname_r.c | 5 ++++-
>  2 files changed, 9 insertions(+), 2 deletions(-)
> 
> diff --git a/src/unistd/isatty.c b/src/unistd/isatty.c
> index c8badaf5..3d5213c6 100644
> --- a/src/unistd/isatty.c
> +++ b/src/unistd/isatty.c
> @@ -1,9 +1,13 @@
>  #include <unistd.h>
> +#include <errno.h>
>  #include <sys/ioctl.h>
>  #include "syscall.h"
>  
>  int isatty(int fd)
>  {
>  	struct winsize wsz;
> -	return !__syscall(SYS_ioctl, fd, TIOCGWINSZ, &wsz);
> +	unsigned long r = __syscall_ret(__syscall(SYS_ioctl, fd, TIOCGWINSZ, &wsz));
> +	if (r == 0) return 1;
> +	if (errno != EBADF) errno = ENOTTY;
> +	return 0;
>  }

__syscall_ret(__syscall(...)) is just an ugly way of writing
syscall(). :-)

But the whole reason __syscall_ret exists is for the sake of setting
errno via a tailcall after the __syscall returns. When you have to
patch up the value, rather than shuffling it in and out of errno and
breaking the tailcall, a cleaner method is something like:

	int r = __syscall(...);
	if (r && r!=-EBADF) r=-ENOTTY;
	return __syscall_ret(r);

> diff --git a/src/unistd/ttyname_r.c b/src/unistd/ttyname_r.c
> index 33aa4ae1..3f836a6a 100644
> --- a/src/unistd/ttyname_r.c
> +++ b/src/unistd/ttyname_r.c
> @@ -10,7 +10,10 @@ int ttyname_r(int fd, char *name, size_t size)
>  	char procname[sizeof "/proc/self/fd/" + 3*sizeof(int) + 2];
>  	ssize_t l;
>  
> -	if (!isatty(fd)) return ENOTTY;
> +	if (!isatty(fd)) {
> +		if (errno != EBADF) errno = ENOTTY;
> +		return errno;
> +	}
>  
>  	__procfdname(procname, fd);
>  	l = readlink(procname, name, size);
> -- 
> 2.17.1

With isatty setting errno reasonably, there's no need for errno fixup
here. Just::

	if (!isatty(fd)) return errno;

suffices.

Rich

Re: [PATCH] improve error handling of ttyname_r and isatty

[Benjamin Peterson]

I've made a new version with your comments addressed. I don't think we can
actually tail call through __syscall_ret in isatty(), since we need to return
nonzero if the syscall succeeded.

[PATCH v2] improve error handling of ttyname_r and isatty

[Benjamin Peterson]

POSIX allows ttyname(_r) and isatty to return EBADF if passed file descriptor is
invalid.
---
 src/unistd/isatty.c    | 6 +++++-
 src/unistd/ttyname_r.c | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/unistd/isatty.c b/src/unistd/isatty.c
index c8badaf5..75a9c186 100644
--- a/src/unistd/isatty.c
+++ b/src/unistd/isatty.c
@@ -1,9 +1,13 @@
 #include <unistd.h>
+#include <errno.h>
 #include <sys/ioctl.h>
 #include "syscall.h"
 
 int isatty(int fd)
 {
 	struct winsize wsz;
-	return !__syscall(SYS_ioctl, fd, TIOCGWINSZ, &wsz);
+	unsigned long r = syscall(SYS_ioctl, fd, TIOCGWINSZ, &wsz);
+	if (r == 0) return 1;
+	if (errno != EBADF) errno = ENOTTY;
+	return 0;
 }
diff --git a/src/unistd/ttyname_r.c b/src/unistd/ttyname_r.c
index 33aa4ae1..24106ad8 100644
--- a/src/unistd/ttyname_r.c
+++ b/src/unistd/ttyname_r.c
@@ -10,7 +10,7 @@ int ttyname_r(int fd, char *name, size_t size)
 	char procname[sizeof "/proc/self/fd/" + 3*sizeof(int) + 2];
 	ssize_t l;
 
-	if (!isatty(fd)) return ENOTTY;
+	if (!isatty(fd)) return errno;
 
 	__procfdname(procname, fd);
 	l = readlink(procname, name, size);
-- 
2.17.1

Re: [PATCH v2] improve error handling of ttyname_r and isatty

[Rich Felker]

On Thu, Sep 13, 2018 at 02:23:42PM -0700, Benjamin Peterson wrote:
> POSIX allows ttyname(_r) and isatty to return EBADF if passed file descriptor is
> invalid.
> ---
>  src/unistd/isatty.c    | 6 +++++-
>  src/unistd/ttyname_r.c | 2 +-
>  2 files changed, 6 insertions(+), 2 deletions(-)
> 
> diff --git a/src/unistd/isatty.c b/src/unistd/isatty.c
> index c8badaf5..75a9c186 100644
> --- a/src/unistd/isatty.c
> +++ b/src/unistd/isatty.c
> @@ -1,9 +1,13 @@
>  #include <unistd.h>
> +#include <errno.h>
>  #include <sys/ioctl.h>
>  #include "syscall.h"
>  
>  int isatty(int fd)
>  {
>  	struct winsize wsz;
> -	return !__syscall(SYS_ioctl, fd, TIOCGWINSZ, &wsz);
> +	unsigned long r = syscall(SYS_ioctl, fd, TIOCGWINSZ, &wsz);
> +	if (r == 0) return 1;
> +	if (errno != EBADF) errno = ENOTTY;
> +	return 0;
>  }
> diff --git a/src/unistd/ttyname_r.c b/src/unistd/ttyname_r.c
> index 33aa4ae1..24106ad8 100644
> --- a/src/unistd/ttyname_r.c
> +++ b/src/unistd/ttyname_r.c
> @@ -10,7 +10,7 @@ int ttyname_r(int fd, char *name, size_t size)
>  	char procname[sizeof "/proc/self/fd/" + 3*sizeof(int) + 2];
>  	ssize_t l;
>  
> -	if (!isatty(fd)) return ENOTTY;
> +	if (!isatty(fd)) return errno;
>  
>  	__procfdname(procname, fd);
>  	l = readlink(procname, name, size);
> -- 
> 2.17.1

Thanks, applying.

Rich