From mboxrd@z Thu Jan 1 00:00:00 1970 X-Spam-Checker-Version: SpamAssassin 3.4.4 (2020-01-24) on inbox.vuxu.org X-Spam-Level: X-Spam-Status: No, score=-1.5 required=5.0 tests=DKIM_INVALID,DKIM_SIGNED, MAILING_LIST_MULTI,RCVD_IN_DNSWL_LOW,T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.4 Received: (qmail 7176 invoked from network); 31 May 2023 14:02:14 -0000 Received: from second.openwall.net (193.110.157.125) by inbox.vuxu.org with ESMTPUTF8; 31 May 2023 14:02:14 -0000 Received: (qmail 3249 invoked by uid 550); 31 May 2023 14:02:10 -0000 Mailing-List: contact musl-help@lists.openwall.com; run by ezmlm Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-ID: Reply-To: musl@lists.openwall.com Received: (qmail 3208 invoked from network); 31 May 2023 14:02:09 -0000 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=inria.fr; s=dc; h=from:to:subject:date:message-id:in-reply-to:references: mime-version:content-transfer-encoding; bh=OetfAiI6GB3IiyLDbUq8kHN3cF7ZdwBKC+7tLHw2GSQ=; b=kj0LRHpHnQaKId0V3pnIMNaiglfI8y+BwIhT15SlB1aRn4MOidRnI7Sg /6PhimGwyIAcWszmi4fHxLdH3IRzH/Lr/Grd1ACdu6vV5Uvskm0lzFmT2 ZZ5hBr22u7AgzdEGTAurIk5dhLxr0M0+9ytuZ8p6CJrHNa1I9bLpkPYcY E=; Authentication-Results: mail3-relais-sop.national.inria.fr; dkim=none (message not signed) header.i=none; spf=SoftFail smtp.mailfrom=Jens.Gustedt@inria.fr; dmarc=fail (p=none dis=none) d=inria.fr X-IronPort-AV: E=Sophos;i="6.00,207,1681164000"; d="scan'208";a="57461908" From: Jens Gustedt To: musl@lists.openwall.com Date: Wed, 31 May 2023 16:01:43 +0200 Message-Id: <976b592b08c82bd6b1f8834904071db6602f3885.1685534402.git.Jens.Gustedt@inria.fr> X-Mailer: git-send-email 2.34.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [musl] [C23 string conversion 1/2] C23: implement the c8rtomb and mbrtoc8 functions The implementations each separate two cases: a fast path that corresponds to calls with state variables that are in the initial state and a leading input character that is ASCII; the other cases are handled by a function for the slow path. These functions are implemented such that the fast versions should not need stack variables of their own, and thus can tail call into the slow path with a jmp instruction if necessary. The fast versions could typically be also used as shallow inline wrapper, if we like to go that way. Only the implementation of mbrtoc8 is a bit tricky. This is because of the weird conventions for dripping the output bytes one call after the other. This is handled by using the state variable as a stack for the up to three characters that are still to be sent to the output. --- include/uchar.h | 6 +++++ src/multibyte/c8rtomb.c | 31 +++++++++++++++++++++++++ src/multibyte/mbrtoc8.c | 51 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 src/multibyte/c8rtomb.c create mode 100644 src/multibyte/mbrtoc8.c diff --git a/include/uchar.h b/include/uchar.h index 7e5c4d40..9f6a3706 100644 --- a/include/uchar.h +++ b/include/uchar.h @@ -9,6 +9,9 @@ extern "C" { typedef unsigned short char16_t; typedef unsigned char32_t; #endif +#if __cplusplus < 201811L +typedef unsigned char char8_t; +#endif #define __NEED_mbstate_t #define __NEED_size_t @@ -16,6 +19,9 @@ typedef unsigned char32_t; #include #include +size_t c8rtomb(char *__restrict, char8_t, mbstate_t *__restrict); +size_t mbrtoc8(char8_t *__restrict, const char *__restrict, size_t, mbstate_t *__restrict); + size_t c16rtomb(char *__restrict, char16_t, mbstate_t *__restrict); size_t mbrtoc16(char16_t *__restrict, const char *__restrict, size_t, mbstate_t *__restrict); diff --git a/src/multibyte/c8rtomb.c b/src/multibyte/c8rtomb.c new file mode 100644 index 00000000..8645e112 --- /dev/null +++ b/src/multibyte/c8rtomb.c @@ -0,0 +1,31 @@ +#include +#include + +__attribute__((__noinline__)) +static size_t c8rtomb_slow(char *__restrict s, unsigned char c8, mbstate_t *__restrict st) +{ + // We need an internal state different from mbrtowc. + static mbstate_t internal_state; + if (!st) st = &internal_state; + wchar_t wc; + + // mbrtowc has return values -2, -1, 0, 1. + size_t res = mbrtowc(&wc, (char const*)&c8, 1, st); + switch (res) { + // our implementation of wcrtomb ignores the state + case 1: res = wcrtomb(s, wc, 0); break; + case 0: res = 1; if (s) *s = 0; break; + } + return res; +} + +static size_t __c8rtomb(char *__restrict s, unsigned char c8, mbstate_t *__restrict st) +{ + if (st && !*(unsigned*)st && (c8 < 0x80)) { + if (s) *s = c8; + return 1; + } + return c8rtomb_slow(s, c8, st); +} + +weak_alias(__c8rtomb, c8rtomb); diff --git a/src/multibyte/mbrtoc8.c b/src/multibyte/mbrtoc8.c new file mode 100644 index 00000000..0c73fa10 --- /dev/null +++ b/src/multibyte/mbrtoc8.c @@ -0,0 +1,51 @@ +#include +#include +#include +#include +#include "internal.h" + +__attribute__((__noinline__)) +static size_t mbrtoc8_slow(unsigned char *__restrict pc8, const char *__restrict src, size_t n, mbstate_t *__restrict st) +{ + static unsigned internal_state; + wchar_t wc; + unsigned* pending = st ? (void*)st : &internal_state; + + // The high bit is set if there is still missing input. If it + // is not set and the value is not zero, a previous call has + // stacked the missing output bytes withing the state. + if ((-*pending) > INT_MIN) { + if (pc8) *pc8 = *pending; + (*pending) >>= 8; + return -3; + } + + // mbrtowc has return values -2, -1, 0, 1, ..., 4. + size_t res = mbrtowc(&wc, src, n, (void*)pending); + if (res <= 4) { + _Alignas(unsigned) unsigned char s[8] = { 0 }; + // Write the result bytes to s over the word boundary + // as we need it. Our wcrtomb implementation ignores + // the state variable, there will be no errors and the + // return value can be ignored. + wcrtomb((void*)(s+3), wc, 0); + // Depending on the endianess this maybe a single load + // instruction. We want to be sure that the bytes are + // in this order, and that the high-order byte is 0. + *pending = (0u|s[4])<<000 | (0u|s[5])<<010 | (0u|s[6])<<020 | (0u|s[7])<<030; + if (pc8) *pc8 = s[3]; + } + return res; +} + +static size_t __mbrtoc8(unsigned char *__restrict pc8, const char *__restrict src, size_t n, mbstate_t *__restrict st) +{ + unsigned char *s = (void*)src; + if (st && !*(unsigned*)st && (s[0] < 0x80u)) { + if (pc8) *pc8 = s[0]; + return s[0]>0; + } + return mbrtoc8_slow(pc8, src, n, st); +} + +weak_alias(__mbrtoc8, mbrtoc8); -- 2.34.1