[musl] [PATCH V3 0/1] feat(x86_64): use wrfsbase if AT_HWCAP2 allows usage

[musl] [PATCH V3 0/1] feat(x86_64): use wrfsbase if AT_HWCAP2 allows usage

[Harald Hoyer]

V2: Forgot to add `arch/x86_64/bits/hwcap.h`
V3: Tabified src/thread/x86_64/__set_thread_area.c

Harald Hoyer (1):
  feat(x86_64): use wrfsbase if AT_HWCAP2 allows usage

 arch/x86_64/bits/hwcap.h              |  2 ++
 ldso/dynlink.c                        |  1 +
 src/env/__libc_start_main.c           |  1 +
 src/internal/libc.c                   |  1 +
 src/internal/libc.h                   |  1 +
 src/thread/x86_64/__set_thread_area.c | 14 ++++++++++++++
 src/thread/x86_64/__set_thread_area.s | 11 -----------
 7 files changed, 20 insertions(+), 11 deletions(-)
 create mode 100644 arch/x86_64/bits/hwcap.h
 create mode 100644 src/thread/x86_64/__set_thread_area.c
 delete mode 100644 src/thread/x86_64/__set_thread_area.s

-- 
2.35.1

[musl] [PATCH 1/1] feat(x86_64): use wrfsbase if AT_HWCAP2 allows usage

[Harald Hoyer]

If `AT_HWCAP2` has `HWCAP2_FSGSBASE` set, then instead of calling
`arch_prctl()`, the `wrfsbase` instruction will be used.

This is helpful in SGX contexts, where inside the enclave no other
mechanism is possible.

Tested against `libc-test`.

Signed-off-by: Harald Hoyer <harald@profian.com>
---
 arch/x86_64/bits/hwcap.h              |  2 ++
 ldso/dynlink.c                        |  1 +
 src/env/__libc_start_main.c           |  1 +
 src/internal/libc.c                   |  1 +
 src/internal/libc.h                   |  1 +
 src/thread/x86_64/__set_thread_area.c | 14 ++++++++++++++
 src/thread/x86_64/__set_thread_area.s | 11 -----------
 7 files changed, 20 insertions(+), 11 deletions(-)
 create mode 100644 arch/x86_64/bits/hwcap.h
 create mode 100644 src/thread/x86_64/__set_thread_area.c
 delete mode 100644 src/thread/x86_64/__set_thread_area.s

diff --git a/arch/x86_64/bits/hwcap.h b/arch/x86_64/bits/hwcap.h
new file mode 100644
index 00000000..eb962bce
--- /dev/null
+++ b/arch/x86_64/bits/hwcap.h
@@ -0,0 +1,2 @@
+#define HWCAP2_RING3MWAIT (1 << 0)
+#define HWCAP2_FSGSBASE   (1 << 1)
diff --git a/ldso/dynlink.c b/ldso/dynlink.c
index 5b9c8be4..2d38ec63 100644
--- a/ldso/dynlink.c
+++ b/ldso/dynlink.c
@@ -1710,6 +1710,7 @@ void __dls2b(size_t *sp, size_t *auxv)
 	 * use during dynamic linking. If possible it will also serve as the
 	 * thread pointer at runtime. */
 	search_vec(auxv, &__hwcap, AT_HWCAP);
+	search_vec(auxv, &__hwcap2, AT_HWCAP2);
 	libc.auxv = auxv;
 	libc.tls_size = sizeof builtin_tls;
 	libc.tls_align = tls_align;
diff --git a/src/env/__libc_start_main.c b/src/env/__libc_start_main.c
index c5b277bd..38772bdb 100644
--- a/src/env/__libc_start_main.c
+++ b/src/env/__libc_start_main.c
@@ -28,6 +28,7 @@ void __init_libc(char **envp, char *pn)
 	libc.auxv = auxv = (void *)(envp+i+1);
 	for (i=0; auxv[i]; i+=2) if (auxv[i]<AUX_CNT) aux[auxv[i]] = auxv[i+1];
 	__hwcap = aux[AT_HWCAP];
+	__hwcap2 = aux[AT_HWCAP2];
 	if (aux[AT_SYSINFO]) __sysinfo = aux[AT_SYSINFO];
 	libc.page_size = aux[AT_PAGESZ];
 
diff --git a/src/internal/libc.c b/src/internal/libc.c
index cb051810..16e47633 100644
--- a/src/internal/libc.c
+++ b/src/internal/libc.c
@@ -3,6 +3,7 @@
 struct __libc __libc;
 
 size_t __hwcap;
+size_t __hwcap2;
 char *__progname=0, *__progname_full=0;
 
 weak_alias(__progname, program_invocation_short_name);
diff --git a/src/internal/libc.h b/src/internal/libc.h
index 619bba86..7d239626 100644
--- a/src/internal/libc.h
+++ b/src/internal/libc.h
@@ -47,6 +47,7 @@ hidden void __libc_exit_fini(void);
 hidden void __fork_handler(int);
 
 extern hidden size_t __hwcap;
+extern hidden size_t __hwcap2;
 extern hidden size_t __sysinfo;
 extern char *__progname, *__progname_full;
 
diff --git a/src/thread/x86_64/__set_thread_area.c b/src/thread/x86_64/__set_thread_area.c
new file mode 100644
index 00000000..dcc5d116
--- /dev/null
+++ b/src/thread/x86_64/__set_thread_area.c
@@ -0,0 +1,14 @@
+#include <libc.h>
+#include <syscall.h>
+#include <bits/hwcap.h>
+
+hidden int __set_thread_area(void *p)
+{
+	if (__hwcap2 & HWCAP2_FSGSBASE) {
+		__asm__ ("wrfsbase %0" :: "r" (p) : "memory");
+		return 0;
+	}
+
+	// arch_prctl(SET_FS, arg)
+	return syscall(__NR_arch_prctl, 0x1002, p);
+}
diff --git a/src/thread/x86_64/__set_thread_area.s b/src/thread/x86_64/__set_thread_area.s
deleted file mode 100644
index 7347ff4d..00000000
--- a/src/thread/x86_64/__set_thread_area.s
+++ /dev/null
@@ -1,11 +0,0 @@
-/* Copyright 2011-2012 Nicholas J. Kain, licensed under standard MIT license */
-.text
-.global __set_thread_area
-.hidden __set_thread_area
-.type __set_thread_area,@function
-__set_thread_area:
-	mov %rdi,%rsi           /* shift for syscall */
-	movl $0x1002,%edi       /* SET_FS register */
-	movl $158,%eax          /* set fs segment to */
-	syscall                 /* arch_prctl(SET_FS, arg)*/
-	ret
-- 
2.35.1

Re: [musl] [PATCH 1/1] feat(x86_64): use wrfsbase if AT_HWCAP2 allows usage

[Rich Felker]

On Tue, Mar 29, 2022 at 02:24:16PM +0200, Harald Hoyer wrote:
> If `AT_HWCAP2` has `HWCAP2_FSGSBASE` set, then instead of calling
> `arch_prctl()`, the `wrfsbase` instruction will be used.
> 
> This is helpful in SGX contexts, where inside the enclave no other
> mechanism is possible.

Thanks for including this motivation, since otherwise I don't think it
makes any sense to use this feature. BTW what happens with other
syscalls in such a context (at least set_tid_address is called
unconditionally), and how does the process communicate any information
or even exit?

> diff --git a/src/thread/x86_64/__set_thread_area.c b/src/thread/x86_64/__set_thread_area.c
> new file mode 100644
> index 00000000..dcc5d116
> --- /dev/null
> +++ b/src/thread/x86_64/__set_thread_area.c
> @@ -0,0 +1,14 @@
> +#include <libc.h>
> +#include <syscall.h>
> +#include <bits/hwcap.h>
> +
> +hidden int __set_thread_area(void *p)
> +{
> +	if (__hwcap2 & HWCAP2_FSGSBASE) {
> +		__asm__ ("wrfsbase %0" :: "r" (p) : "memory");
> +		return 0;
> +	}
> +
> +	// arch_prctl(SET_FS, arg)
> +	return syscall(__NR_arch_prctl, 0x1002, p);
> +}

I'm guessing this breaks build on anything but recent assembler
versions, no? If so, it should probably be written with a .byte
directive or something.

There's also a question of whether the existence of the hwcap flag is
intended to document a contract for the kernel to permit the process
to perform this operation, and a contract for the kernel to accept it
being set this way (rather than creating a possible inconsistency
between the kernel's idea of the process's %fs base and the actual %fs
base that's active. Is this documented somewhere on the kernel side?
If so then this should be okay, but this needs checking before it can
be merged.

Rich

Re: [musl] [PATCH 1/1] feat(x86_64): use wrfsbase if AT_HWCAP2 allows usage

[Harald Hoyer]

Am 29.03.22 um 17:54 schrieb Rich Felker:
> On Tue, Mar 29, 2022 at 02:24:16PM +0200, Harald Hoyer wrote:
>> If `AT_HWCAP2` has `HWCAP2_FSGSBASE` set, then instead of calling
>> `arch_prctl()`, the `wrfsbase` instruction will be used.
>>
>> This is helpful in SGX contexts, where inside the enclave no other
>> mechanism is possible.
> 
> Thanks for including this motivation, since otherwise I don't think it
> makes any sense to use this feature. BTW what happens with other
> syscalls in such a context (at least set_tid_address is called
> unconditionally), and how does the process communicate any information
> or even exit?

In our project (enarx) we catch the `syscall` exception and handle it by proxying
it to the host (with filtering and sanity checks). Because `arch_prctl` sets a
hardware register, which is handled special in enclave context switching,
we can't do that for this syscall.

> 
>> diff --git a/src/thread/x86_64/__set_thread_area.c b/src/thread/x86_64/__set_thread_area.c
>> new file mode 100644
>> index 00000000..dcc5d116
>> --- /dev/null
>> +++ b/src/thread/x86_64/__set_thread_area.c
>> @@ -0,0 +1,14 @@
>> +#include <libc.h>
>> +#include <syscall.h>
>> +#include <bits/hwcap.h>
>> +
>> +hidden int __set_thread_area(void *p)
>> +{
>> +	if (__hwcap2 & HWCAP2_FSGSBASE) {
>> +		__asm__ ("wrfsbase %0" :: "r" (p) : "memory");
>> +		return 0;
>> +	}
>> +
>> +	// arch_prctl(SET_FS, arg)
>> +	return syscall(__NR_arch_prctl, 0x1002, p);
>> +}
> 
> I'm guessing this breaks build on anything but recent assembler
> versions, no? If so, it should probably be written with a .byte
> directive or something.

Is gcc-4.6.0 old enough?

commit 4ee89d5fb78b48b62b507a29d3a576c63ae22505
Author: H.J. Lu <hongjiu.lu@intel.com>
Date:   Mon Jul 5 21:57:55 2010 +0000

or llvm 3.1.0?

commit 228d9131aad9f3553c9c69abf010f41ce43c8423
Author: Craig Topper <craig.topper@gmail.com>
Date:   Sun Oct 30 19:57:21 2011 +0000

> 
> There's also a question of whether the existence of the hwcap flag is
> intended to document a contract for the kernel to permit the process
> to perform this operation, and a contract for the kernel to accept it
> being set this way (rather than creating a possible inconsistency
> between the kernel's idea of the process's %fs base and the actual %fs
> base that's active. Is this documented somewhere on the kernel side?
> If so then this should be okay, but this needs checking before it can
> be merged.
> 
> Rich

linux Documentation/x86/x86_64/fsgs.rst:

  The kernel provides reliable information about the enabled state in the
  ELF AUX vector. If the HWCAP2_FSGSBASE bit is set in the AUX vector, the
  kernel has FSGSBASE instructions enabled and applications can use them.

Re: [musl] [PATCH 1/1] feat(x86_64): use wrfsbase if AT_HWCAP2 allows usage

[Rich Felker]

On Wed, Mar 30, 2022 at 08:57:55AM +0200, Harald Hoyer wrote:
> Am 29.03.22 um 17:54 schrieb Rich Felker:
> >On Tue, Mar 29, 2022 at 02:24:16PM +0200, Harald Hoyer wrote:
> >>If `AT_HWCAP2` has `HWCAP2_FSGSBASE` set, then instead of calling
> >>`arch_prctl()`, the `wrfsbase` instruction will be used.
> >>
> >>This is helpful in SGX contexts, where inside the enclave no other
> >>mechanism is possible.
> >
> >Thanks for including this motivation, since otherwise I don't think it
> >makes any sense to use this feature. BTW what happens with other
> >syscalls in such a context (at least set_tid_address is called
> >unconditionally), and how does the process communicate any information
> >or even exit?
> 
> In our project (enarx) we catch the `syscall` exception and handle it by proxying
> it to the host (with filtering and sanity checks). Because `arch_prctl` sets a
> hardware register, which is handled special in enclave context switching,
> we can't do that for this syscall.

Is there no way to make it jump the guest to a host-provided (ala
vdso) helper that would juse use the instruction?

> >>diff --git a/src/thread/x86_64/__set_thread_area.c b/src/thread/x86_64/__set_thread_area.c
> >>new file mode 100644
> >>index 00000000..dcc5d116
> >>--- /dev/null
> >>+++ b/src/thread/x86_64/__set_thread_area.c
> >>@@ -0,0 +1,14 @@
> >>+#include <libc.h>
> >>+#include <syscall.h>
> >>+#include <bits/hwcap.h>
> >>+
> >>+hidden int __set_thread_area(void *p)
> >>+{
> >>+	if (__hwcap2 & HWCAP2_FSGSBASE) {
> >>+		__asm__ ("wrfsbase %0" :: "r" (p) : "memory");
> >>+		return 0;
> >>+	}
> >>+
> >>+	// arch_prctl(SET_FS, arg)
> >>+	return syscall(__NR_arch_prctl, 0x1002, p);
> >>+}
> >
> >I'm guessing this breaks build on anything but recent assembler
> >versions, no? If so, it should probably be written with a .byte
> >directive or something.
> 
> Is gcc-4.6.0 old enough?
> 
> commit 4ee89d5fb78b48b62b507a29d3a576c63ae22505
> Author: H.J. Lu <hongjiu.lu@intel.com>
> Date:   Mon Jul 5 21:57:55 2010 +0000

It's a matter of binutils not gcc. And I'm not sure. Previously I
don't think we've had any minimum required binutils version for
x86_64, so my leaning would be to do this a way that does not create a
dependency on the assembler knowing new insns from a -march greater
than what you're compiling for.

Using .byte does preclude letting the compiler pick an input register,
but "D"(p) instead of "r"(p) should be zero-cost anyway since p is
always going to be in the first argument register anyway.

> or llvm 3.1.0?
> 
> commit 228d9131aad9f3553c9c69abf010f41ce43c8423
> Author: Craig Topper <craig.topper@gmail.com>
> Date:   Sun Oct 30 19:57:21 2011 +0000
> 
> >
> >There's also a question of whether the existence of the hwcap flag is
> >intended to document a contract for the kernel to permit the process
> >to perform this operation, and a contract for the kernel to accept it
> >being set this way (rather than creating a possible inconsistency
> >between the kernel's idea of the process's %fs base and the actual %fs
> >base that's active. Is this documented somewhere on the kernel side?
> >If so then this should be okay, but this needs checking before it can
> >be merged.
> >
> >Rich
> 
> linux Documentation/x86/x86_64/fsgs.rst:
> 
>  The kernel provides reliable information about the enabled state in the
>  ELF AUX vector. If the HWCAP2_FSGSBASE bit is set in the AUX vector, the
>  kernel has FSGSBASE instructions enabled and applications can use them.

Thanks! I just found that myself, along with:

https://lwn.net/Articles/821723/

which explains some of the history and motivation.

Short of any concern others may raise that I'm not seeing, I think
this change is almost surely okay with care taken not to bump the
assembler version requirements. I'll plan to take this shortly after
release.

Rich