Re: [PATCH] x86_64/memset: use "small block" code for blocks up to 30 bytes long

[Denys Vlasenko <vda.linux@googlemail.com>]

[-- Attachment #1: Type: text/plain, Size: 1466 bytes --]

On Sun, Feb 15, 2015 at 4:03 PM, Rich Felker <dalias@libc.org> wrote:
>> Just because we don't personally see a hit from 6-cycle imul of AMD CPUs,
>> it does not mean people who do use those CPUs don't exist. Have heart...
>
> Did you test the version I attached? I think there should be at least
> 4-5 cycles between when the imul is launched and when the result is
> used, so I'm failing to see how the latency is a big deal.

Okay, I won't insist.
Your version works good. The "rep stosq" setup time is still noticeable
even when we switch to it after 126:

129 byte block: 10.37 bytes/ns
128 byte block: 10.65 bytes/ns
127 byte block: 10.58 bytes/ns
126 byte block: 18.44 bytes/ns
125 byte block: 18.30 bytes/ns
124 byte block: 18.15 bytes/ns

but I don't think we should do anything about this.


Here

        lea -1(%rdx),%rcx
        cmp $126,%rcx
        jae 2f

you'd have a stall, since cmp needs the result of lea. why not this?

        lea -1(%rdx),%rcx
        cmp $127,%rdx
        jae 2f

then you can even move lea to "big buf" code part
(no point doing it in "small buf" code where it is not used).


Possible bug: this check seems misplaced:

2:      test %rdx,%rdx
        jz 1b

it should be before byte stores:
        mov %sil,(%rdi)
        mov %sil,-1(%rdi,%rdx)
        cmp $2,%edx
        jbe 1f
otherwise memset of zero length will fill two bytes, at buf[0] and buf[-1]


"sub $8,%rcx" can be folded into lea.

Please see attached file.

[-- Attachment #2: vda1.s --]
[-- Type: application/octet-stream, Size: 1056 bytes --]

.global memset
.type memset,@function
memset:
	movzbq %sil,%rax
	mov $0x101010101010101,%r8
	imul %r8,%rax

	cmp $127,%rdx
	jae 2f

	test %rdx,%rdx
	jz 1f

	mov %sil,(%rdi)
	mov %sil,-1(%rdi,%rdx)
	cmp $2,%edx
	jbe 1f

	mov %ax,1(%rdi)
	mov %ax,(-1-2)(%rdi,%rdx)
	cmp $6,%edx
	jbe 1f

	mov %eax,(1+2)(%rdi)
	mov %eax,(-1-2-4)(%rdi,%rdx)
	cmp $14,%edx
	jbe 1f

	mov %rax,(1+2+4)(%rdi)
	mov %rax,(-1-2-4-8)(%rdi,%rdx)
	cmp $30,%edx
	jbe 1f

	mov %rax,(1+2+4+8)(%rdi)
	mov %rax,(1+2+4+8+8)(%rdi)
	mov %rax,(-1-2-4-8-16)(%rdi,%rdx)
	mov %rax,(-1-2-4-8-8)(%rdi,%rdx)
	cmp $62,%edx
	jbe 1f

	mov %rax,(1+2+4+8+16)(%rdi)
	mov %rax,(1+2+4+8+16+8)(%rdi)
	mov %rax,(1+2+4+8+16+16)(%rdi)
	mov %rax,(1+2+4+8+16+24)(%rdi)
	mov %rax,(-1-2-4-8-16-32)(%rdi,%rdx)
	mov %rax,(-1-2-4-8-16-24)(%rdi,%rdx)
	mov %rax,(-1-2-4-8-16-16)(%rdi,%rdx)
	mov %rax,(-1-2-4-8-16-8)(%rdi,%rdx)

1:	mov %rdi,%rax
	ret

2:	lea -9(%rdx),%rcx
	mov %rdi,%r8
	shr $3,%rcx
	mov %rax,(%rdi)
	mov %rax,-16(%rdi,%rdx)
	mov %rax,-8(%rdi,%rdx)
	add $8,%rdi
	and $-8,%rdi
	rep
	stosq
	mov %r8,%rax
	ret