Skip to content

Commit 321a407

Browse files
TocarIPrandall77
TocarIP
authored and
randall77
committed
runtime: optimize indexbytebody on amd64
Use avx2 to compare 32 bytes per iteration. Results (haswell): name old time/op new time/op delta IndexByte32-6 15.5ns ± 0% 14.7ns ± 5% -4.87% (p=0.000 n=16+20) IndexByte4K-6 360ns ± 0% 183ns ± 0% -49.17% (p=0.000 n=19+20) IndexByte4M-6 384µs ± 0% 256µs ± 1% -33.41% (p=0.000 n=20+20) IndexByte64M-6 6.20ms ± 0% 4.18ms ± 1% -32.52% (p=0.000 n=19+20) IndexBytePortable32-6 73.4ns ± 5% 75.8ns ± 3% +3.35% (p=0.000 n=20+19) IndexBytePortable4K-6 5.15µs ± 0% 5.15µs ± 0% ~ (all samples are equal) IndexBytePortable4M-6 5.26ms ± 0% 5.25ms ± 0% -0.12% (p=0.000 n=20+18) IndexBytePortable64M-6 84.1ms ± 0% 84.1ms ± 0% -0.08% (p=0.012 n=18+20) Index32-6 352ns ± 0% 352ns ± 0% ~ (all samples are equal) Index4K-6 53.8µs ± 0% 53.8µs ± 0% -0.03% (p=0.000 n=16+18) Index4M-6 55.4ms ± 0% 55.4ms ± 0% ~ (p=0.149 n=20+19) Index64M-6 886ms ± 0% 886ms ± 0% ~ (p=0.108 n=20+20) IndexEasy32-6 80.3ns ± 0% 80.1ns ± 0% -0.21% (p=0.000 n=20+20) IndexEasy4K-6 426ns ± 0% 215ns ± 0% -49.53% (p=0.000 n=20+20) IndexEasy4M-6 388µs ± 0% 262µs ± 1% -32.42% (p=0.000 n=18+20) IndexEasy64M-6 6.20ms ± 0% 4.19ms ± 1% -32.47% (p=0.000 n=18+20) name old speed new speed delta IndexByte32-6 2.06GB/s ± 1% 2.17GB/s ± 5% +5.19% (p=0.000 n=18+20) IndexByte4K-6 11.4GB/s ± 0% 22.3GB/s ± 0% +96.45% (p=0.000 n=17+20) IndexByte4M-6 10.9GB/s ± 0% 16.4GB/s ± 1% +50.17% (p=0.000 n=20+20) IndexByte64M-6 10.8GB/s ± 0% 16.0GB/s ± 1% +48.19% (p=0.000 n=19+20) IndexBytePortable32-6 436MB/s ± 5% 422MB/s ± 3% -3.27% (p=0.000 n=20+19) IndexBytePortable4K-6 795MB/s ± 0% 795MB/s ± 0% ~ (p=0.940 n=17+18) IndexBytePortable4M-6 798MB/s ± 0% 799MB/s ± 0% +0.12% (p=0.000 n=20+18) IndexBytePortable64M-6 798MB/s ± 0% 798MB/s ± 0% +0.08% (p=0.011 n=18+20) Index32-6 90.9MB/s ± 0% 90.9MB/s ± 0% -0.00% (p=0.025 n=20+20) Index4K-6 76.1MB/s ± 0% 76.1MB/s ± 0% +0.03% (p=0.000 n=14+15) Index4M-6 75.7MB/s ± 0% 75.7MB/s ± 0% ~ (p=0.076 n=20+19) Index64M-6 75.7MB/s ± 0% 75.7MB/s ± 0% ~ (p=0.456 n=20+17) IndexEasy32-6 399MB/s ± 0% 399MB/s ± 0% +0.20% (p=0.000 n=20+19) IndexEasy4K-6 9.60GB/s ± 0% 19.02GB/s ± 0% +98.19% (p=0.000 n=20+20) IndexEasy4M-6 10.8GB/s ± 0% 16.0GB/s ± 1% +47.98% (p=0.000 n=18+20) IndexEasy64M-6 10.8GB/s ± 0% 16.0GB/s ± 1% +48.08% (p=0.000 n=18+20) Change-Id: I46075921dde9f3580a89544c0b3a2d8c9181ebc4 Reviewed-on: https://go-review.googlesource.com/16484 Reviewed-by: Keith Randall <[email protected]> Run-TryBot: Ilya Tocar <[email protected]> Reviewed-by: Klaus Post <[email protected]> TryBot-Result: Gobot Gobot <[email protected]>
1 parent ffb2063 commit 321a407

File tree

4 files changed

+59
-8
lines changed

4 files changed

+59
-8
lines changed

‎src/cmd/internal/obj/x86/a.out.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -749,6 +749,8 @@ const (
749749
AVPCMPEQB
750750
AVPMOVMSKB
751751
AVPAND
752+
AVPTEST
753+
AVPBROADCASTB
752754

753755
// from 386
754756
AJCXZW

‎src/cmd/internal/obj/x86/anames.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -690,6 +690,8 @@ var Anames = []string{
690690
"VPCMPEQB",
691691
"VPMOVMSKB",
692692
"VPAND",
693+
"VPTEST",
694+
"VPBROADCASTB",
693695
"JCXZW",
694696
"FCMOVCC",
695697
"FCMOVCS",

‎src/cmd/internal/obj/x86/asm6.go

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -219,8 +219,9 @@ const (
219219
Pf2 = 0xf2 /* xmm escape 1: f2 0f */
220220
Pf3 = 0xf3 /* xmm escape 2: f3 0f */
221221
Pq3 = 0x67 /* xmm escape 3: 66 48 0f */
222-
Pvex1 = 0xc5 /* 66 escape, vex encoding */
223-
Pvex2 = 0xc6 /* f3 escape, vex encoding */
222+
Pvex1 = 0xc5 /* 66.0f escape, vex encoding */
223+
Pvex2 = 0xc6 /* f3.0f escape, vex encoding */
224+
Pvex3 = 0xc7 /* 66.0f38 escape, vex encoding */
224225
Pw = 0x48 /* Rex.w */
225226
Pw8 = 0x90 // symbolic; exact value doesn't matter
226227
Py = 0x80 /* defaults to 64-bit mode */
@@ -631,6 +632,11 @@ var yxr_ml_vex = []ytab{
631632
{Yxr, Ynone, Yml, Zr_m_xm_vex, 1},
632633
}
633634

635+
var yml_xr_vex = []ytab{
636+
{Yml, Ynone, Yxr, Zm_r_xm_vex, 1},
637+
{Yxr, Ynone, Yxr, Zm_r_xm_vex, 1},
638+
}
639+
634640
var yxm_xm_xm = []ytab{
635641
{Yxr, Yxr, Yxr, Zr_r_r_vex, 1},
636642
{Yxm, Yxr, Yxr, Zr_r_r_vex, 1},
@@ -1510,6 +1516,8 @@ var optab =
15101516
{AVPCMPEQB, yxm_xm_xm, Pvex1, [23]uint8{0x74, 0x74}},
15111517
{AVPMOVMSKB, ymskb_vex, Pvex1, [23]uint8{0xd7}},
15121518
{AVPAND, yxm_xm_xm, Pvex1, [23]uint8{0xdb, 0xdb}},
1519+
{AVPBROADCASTB, yml_xr_vex, Pvex3, [23]uint8{0x78, 0x78}},
1520+
{AVPTEST, yml_xr_vex, Pvex3, [23]uint8{0x17, 0x17}},
15131521
{obj.AUSEFIELD, ynop, Px, [23]uint8{0, 0}},
15141522
{obj.ATYPE, nil, 0, [23]uint8{}},
15151523
{obj.AFUNCDATA, yfuncdata, Px, [23]uint8{0, 0}},
@@ -2965,13 +2973,13 @@ func vexprefix(ctxt *obj.Link, to *obj.Addr, from *obj.Addr, from3 *obj.Addr, pr
29652973
rexX := regrex[from.Index]
29662974
var prefBit uint8
29672975
// This will go into VEX.PP field.
2968-
if pref == Pvex1 {
2976+
if pref == Pvex1 || pref == Pvex3 {
29692977
prefBit = 1
29702978
} else if pref == Pvex2 {
29712979
prefBit = 2
2972-
} // TODO add Pvex0,Pvex3
2980+
} // TODO add Pvex0
29732981

2974-
if rexX == 0 && rexB == 0 { // 2-byte vex prefix
2982+
if rexX == 0 && rexB == 0 && pref != Pvex3 { // 2-byte vex prefix
29752983
// In 2-byte case, first byte is always C5
29762984
ctxt.Andptr[0] = 0xc5
29772985
ctxt.Andptr = ctxt.Andptr[1:]
@@ -2998,9 +3006,13 @@ func vexprefix(ctxt *obj.Link, to *obj.Addr, from *obj.Addr, from3 *obj.Addr, pr
29983006
ctxt.Andptr[0] = 0xc4
29993007
ctxt.Andptr = ctxt.Andptr[1:]
30003008

3001-
// Encode VEX.mmmmm with prefix value, for now assume 0F 38,
3002-
// which encodes as 1.
3003-
ctxt.Andptr[0] = 0x1 // TODO handle different prefix
3009+
// Encode VEX.mmmmm with prefix value, assume 0F,
3010+
// which encodes as 1, unless 0F38 was specified with pvex3.
3011+
ctxt.Andptr[0] = 0x1 // TODO handle 0F3A
3012+
if pref == Pvex3 {
3013+
ctxt.Andptr[0] = 0x2
3014+
}
3015+
30043016
// REX.[RXB] are inverted and encoded in 3 upper bits
30053017
if rexR == 0 {
30063018
ctxt.Andptr[0] |= 0x80

‎src/runtime/asm_amd64.s

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1940,6 +1940,9 @@ TEXT runtime·indexbytebody(SB),NOSPLIT,$0
19401940
CMPQ BX, $16
19411941
JLT small
19421942

1943+
CMPQ BX, $32
1944+
JA avx2
1945+
no_avx2:
19431946
// round up to first 16-byte boundary
19441947
TESTQ $15, SI
19451948
JZ aligned
@@ -2003,6 +2006,38 @@ small:
20032006
MOVQ $-1, (R8)
20042007
RET
20052008

2009+
avx2:
2010+
CMPB runtime·support_avx2(SB), $1
2011+
JNE no_avx2
2012+
MOVD AX, X0
2013+
LEAQ -32(SI)(BX*1), R11
2014+
VPBROADCASTB X0, X1
2015+
avx2_loop:
2016+
MOVHDU (DI), X2
2017+
VPCMPEQB X1, X2, X3
2018+
VPTEST X3, X3
2019+
JNZ avx2success
2020+
ADDQ $32, DI
2021+
CMPQ DI, R11
2022+
JLT avx2_loop
2023+
MOVQ R11, DI
2024+
MOVHDU (DI), X2
2025+
VPCMPEQB X1, X2, X3
2026+
VPTEST X3, X3
2027+
JNZ avx2success
2028+
VZEROUPPER
2029+
MOVQ $-1, (R8)
2030+
RET
2031+
2032+
avx2success:
2033+
VPMOVMSKB X3, DX
2034+
BSFL DX, DX
2035+
SUBQ SI, DI
2036+
ADDQ DI, DX
2037+
MOVQ DX, (R8)
2038+
VZEROUPPER
2039+
RET
2040+
20062041
// we've found the chunk containing the byte
20072042
// now just figure out which specific byte it is
20082043
ssesuccess:

0 commit comments

Comments
 (0)