File: count_pair_amd64.s

package info (click to toggle)
golang-github-segmentio-asm 1.2.0%2Bgit20231107.1cfacc8-1
links: PTS, VCS
area: main
in suites: forky, sid
size: 932 kB
sloc: asm: 6,093; makefile: 32
file content (1056 lines) | stat: -rw-r--r-- 20,154 bytes
// Code generated by command: go run count_pair_asm.go -pkg mem -out ../mem/count_pair_amd64.s -stubs ../mem/count_pair_amd64.go. DO NOT EDIT.

//go:build !purego

#include "textflag.h"

// func countPair1(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT
TEXT ·countPair1(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	XORQ DX, DX
	SUBQ $0x01, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JLE  done

generic:
	MOVQ    DX, BX
	INCQ    BX
	MOVB    (AX), SI
	CMPB    SI, 1(AX)
	CMOVQEQ BX, DX
	ADDQ    $0x01, AX
	SUBQ    $0x01, CX
	CMPQ    CX, $0x00
	JG      generic

done:
	MOVQ DX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000101
	JL   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   128(AX), Y8
	VMOVDQU   160(AX), Y10
	VMOVDQU   192(AX), Y12
	VMOVDQU   224(AX), Y14
	VMOVDQU   1(AX), Y1
	VMOVDQU   33(AX), Y3
	VMOVDQU   65(AX), Y5
	VMOVDQU   97(AX), Y7
	VMOVDQU   129(AX), Y9
	VMOVDQU   161(AX), Y11
	VMOVDQU   193(AX), Y13
	VMOVDQU   225(AX), Y15
	VPCMPEQB  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQB  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	VPCMPEQB  Y4, Y5, Y5
	VPMOVMSKB Y5, DI
	VPCMPEQB  Y6, Y7, Y7
	VPMOVMSKB Y7, R8
	VPCMPEQB  Y8, Y9, Y9
	VPMOVMSKB Y9, R9
	VPCMPEQB  Y10, Y11, Y11
	VPMOVMSKB Y11, R10
	VPCMPEQB  Y12, Y13, Y13
	VPMOVMSKB Y13, R11
	VPCMPEQB  Y14, Y15, Y15
	VPMOVMSKB Y15, R12
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	POPCNTQ   DI, DI
	POPCNTQ   R8, R8
	POPCNTQ   R9, R9
	POPCNTQ   R10, R10
	POPCNTQ   R11, R11
	POPCNTQ   R12, R12
	ADDQ      SI, BX
	ADDQ      R8, DI
	ADDQ      DI, BX
	ADDQ      R10, R9
	ADDQ      R12, R11
	ADDQ      R11, R9
	ADDQ      R9, BX
	ADDQ      BX, DX
	ADDQ      $0x00000100, AX
	SUBQ      $0x00000100, CX
	CMPQ      CX, $0x00000101
	JGE       avx2_loop256

avx2_tail128:
	CMPQ      CX, $0x81
	JL        avx2_tail64
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   1(AX), Y1
	VMOVDQU   33(AX), Y3
	VMOVDQU   65(AX), Y5
	VMOVDQU   97(AX), Y7
	VPCMPEQB  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQB  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	VPCMPEQB  Y4, Y5, Y5
	VPMOVMSKB Y5, DI
	VPCMPEQB  Y6, Y7, Y7
	VPMOVMSKB Y7, R8
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	POPCNTQ   DI, DI
	POPCNTQ   R8, R8
	ADDQ      SI, BX
	ADDQ      R8, DI
	ADDQ      DI, BX
	ADDQ      BX, DX
	ADDQ      $0x00000080, AX
	SUBQ      $0x00000080, CX

avx2_tail64:
	CMPQ      CX, $0x41
	JL        avx2_tail32
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   1(AX), Y1
	VMOVDQU   33(AX), Y3
	VPCMPEQB  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQB  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	ADDQ      SI, BX
	ADDQ      BX, DX
	ADDQ      $0x00000040, AX
	SUBQ      $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x21
	JL        avx2_tail16
	VMOVDQU   (AX), Y0
	VMOVDQU   1(AX), Y1
	VPCMPEQB  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	POPCNTQ   BX, BX
	ADDQ      BX, DX
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	CMPQ      CX, $0x11
	JL        avx2_tail
	VMOVDQU   (AX), X0
	VMOVDQU   1(AX), X1
	VPCMPEQB  X0, X1, X1
	VPMOVMSKB X1, BX
	POPCNTQ   BX, BX
	ADDQ      BX, DX
	ADDQ      $0x10, AX
	SUBQ      $0x10, CX

avx2_tail:
	VZEROUPPER
	JMP tail

// func countPair2(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT
TEXT ·countPair2(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	XORQ DX, DX
	SUBQ $0x02, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JLE  done

generic:
	MOVQ    DX, BX
	INCQ    BX
	MOVW    (AX), SI
	CMPW    SI, 2(AX)
	CMOVQEQ BX, DX
	ADDQ    $0x02, AX
	SUBQ    $0x02, CX
	CMPQ    CX, $0x00
	JG      generic

done:
	MOVQ DX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000102
	JL   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   128(AX), Y8
	VMOVDQU   160(AX), Y10
	VMOVDQU   192(AX), Y12
	VMOVDQU   224(AX), Y14
	VMOVDQU   2(AX), Y1
	VMOVDQU   34(AX), Y3
	VMOVDQU   66(AX), Y5
	VMOVDQU   98(AX), Y7
	VMOVDQU   130(AX), Y9
	VMOVDQU   162(AX), Y11
	VMOVDQU   194(AX), Y13
	VMOVDQU   226(AX), Y15
	VPCMPEQW  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQW  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	VPCMPEQW  Y4, Y5, Y5
	VPMOVMSKB Y5, DI
	VPCMPEQW  Y6, Y7, Y7
	VPMOVMSKB Y7, R8
	VPCMPEQW  Y8, Y9, Y9
	VPMOVMSKB Y9, R9
	VPCMPEQW  Y10, Y11, Y11
	VPMOVMSKB Y11, R10
	VPCMPEQW  Y12, Y13, Y13
	VPMOVMSKB Y13, R11
	VPCMPEQW  Y14, Y15, Y15
	VPMOVMSKB Y15, R12
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	POPCNTQ   DI, DI
	POPCNTQ   R8, R8
	POPCNTQ   R9, R9
	POPCNTQ   R10, R10
	POPCNTQ   R11, R11
	POPCNTQ   R12, R12
	ADDQ      SI, BX
	ADDQ      R8, DI
	ADDQ      DI, BX
	ADDQ      R10, R9
	ADDQ      R12, R11
	ADDQ      R11, R9
	ADDQ      R9, BX
	ADDQ      BX, DX
	ADDQ      $0x00000100, AX
	SUBQ      $0x00000100, CX
	CMPQ      CX, $0x00000102
	JGE       avx2_loop256

avx2_tail128:
	CMPQ      CX, $0x82
	JL        avx2_tail64
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   2(AX), Y1
	VMOVDQU   34(AX), Y3
	VMOVDQU   66(AX), Y5
	VMOVDQU   98(AX), Y7
	VPCMPEQW  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQW  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	VPCMPEQW  Y4, Y5, Y5
	VPMOVMSKB Y5, DI
	VPCMPEQW  Y6, Y7, Y7
	VPMOVMSKB Y7, R8
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	POPCNTQ   DI, DI
	POPCNTQ   R8, R8
	ADDQ      SI, BX
	ADDQ      R8, DI
	ADDQ      DI, BX
	ADDQ      BX, DX
	ADDQ      $0x00000080, AX
	SUBQ      $0x00000080, CX

avx2_tail64:
	CMPQ      CX, $0x42
	JL        avx2_tail32
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   2(AX), Y1
	VMOVDQU   34(AX), Y3
	VPCMPEQW  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQW  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	ADDQ      SI, BX
	ADDQ      BX, DX
	ADDQ      $0x00000040, AX
	SUBQ      $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x22
	JL        avx2_tail16
	VMOVDQU   (AX), Y0
	VMOVDQU   2(AX), Y1
	VPCMPEQW  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	POPCNTQ   BX, BX
	ADDQ      BX, DX
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	CMPQ      CX, $0x12
	JL        avx2_tail
	VMOVDQU   (AX), X0
	VMOVDQU   2(AX), X1
	VPCMPEQW  X0, X1, X1
	VPMOVMSKB X1, BX
	POPCNTQ   BX, BX
	ADDQ      BX, DX
	ADDQ      $0x10, AX
	SUBQ      $0x10, CX

avx2_tail:
	VZEROUPPER
	SHRQ $0x01, DX
	JMP  tail

// func countPair4(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT
TEXT ·countPair4(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	XORQ DX, DX
	SUBQ $0x04, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JLE  done

generic:
	MOVQ    DX, BX
	INCQ    BX
	MOVL    (AX), SI
	CMPL    SI, 4(AX)
	CMOVQEQ BX, DX
	ADDQ    $0x04, AX
	SUBQ    $0x04, CX
	CMPQ    CX, $0x00
	JG      generic

done:
	MOVQ DX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000104
	JL   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   128(AX), Y8
	VMOVDQU   160(AX), Y10
	VMOVDQU   192(AX), Y12
	VMOVDQU   224(AX), Y14
	VMOVDQU   4(AX), Y1
	VMOVDQU   36(AX), Y3
	VMOVDQU   68(AX), Y5
	VMOVDQU   100(AX), Y7
	VMOVDQU   132(AX), Y9
	VMOVDQU   164(AX), Y11
	VMOVDQU   196(AX), Y13
	VMOVDQU   228(AX), Y15
	VPCMPEQD  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQD  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	VPCMPEQD  Y4, Y5, Y5
	VPMOVMSKB Y5, DI
	VPCMPEQD  Y6, Y7, Y7
	VPMOVMSKB Y7, R8
	VPCMPEQD  Y8, Y9, Y9
	VPMOVMSKB Y9, R9
	VPCMPEQD  Y10, Y11, Y11
	VPMOVMSKB Y11, R10
	VPCMPEQD  Y12, Y13, Y13
	VPMOVMSKB Y13, R11
	VPCMPEQD  Y14, Y15, Y15
	VPMOVMSKB Y15, R12
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	POPCNTQ   DI, DI
	POPCNTQ   R8, R8
	POPCNTQ   R9, R9
	POPCNTQ   R10, R10
	POPCNTQ   R11, R11
	POPCNTQ   R12, R12
	ADDQ      SI, BX
	ADDQ      R8, DI
	ADDQ      DI, BX
	ADDQ      R10, R9
	ADDQ      R12, R11
	ADDQ      R11, R9
	ADDQ      R9, BX
	ADDQ      BX, DX
	ADDQ      $0x00000100, AX
	SUBQ      $0x00000100, CX
	CMPQ      CX, $0x00000104
	JGE       avx2_loop256

avx2_tail128:
	CMPQ      CX, $0x84
	JL        avx2_tail64
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   4(AX), Y1
	VMOVDQU   36(AX), Y3
	VMOVDQU   68(AX), Y5
	VMOVDQU   100(AX), Y7
	VPCMPEQD  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQD  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	VPCMPEQD  Y4, Y5, Y5
	VPMOVMSKB Y5, DI
	VPCMPEQD  Y6, Y7, Y7
	VPMOVMSKB Y7, R8
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	POPCNTQ   DI, DI
	POPCNTQ   R8, R8
	ADDQ      SI, BX
	ADDQ      R8, DI
	ADDQ      DI, BX
	ADDQ      BX, DX
	ADDQ      $0x00000080, AX
	SUBQ      $0x00000080, CX

avx2_tail64:
	CMPQ      CX, $0x44
	JL        avx2_tail32
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   4(AX), Y1
	VMOVDQU   36(AX), Y3
	VPCMPEQD  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQD  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	ADDQ      SI, BX
	ADDQ      BX, DX
	ADDQ      $0x00000040, AX
	SUBQ      $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x24
	JL        avx2_tail16
	VMOVDQU   (AX), Y0
	VMOVDQU   4(AX), Y1
	VPCMPEQD  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	POPCNTQ   BX, BX
	ADDQ      BX, DX
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	CMPQ      CX, $0x14
	JL        avx2_tail
	VMOVDQU   (AX), X0
	VMOVDQU   4(AX), X1
	VPCMPEQD  X0, X1, X1
	VPMOVMSKB X1, BX
	POPCNTQ   BX, BX
	ADDQ      BX, DX
	ADDQ      $0x10, AX
	SUBQ      $0x10, CX

avx2_tail:
	VZEROUPPER
	SHRQ $0x02, DX
	JMP  tail

// func countPair8(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT
TEXT ·countPair8(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	XORQ DX, DX
	SUBQ $0x08, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JLE  done

generic:
	MOVQ    DX, BX
	INCQ    BX
	MOVQ    (AX), SI
	CMPQ    SI, 8(AX)
	CMOVQEQ BX, DX
	ADDQ    $0x08, AX
	SUBQ    $0x08, CX
	CMPQ    CX, $0x00
	JG      generic

done:
	MOVQ DX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000108
	JL   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   128(AX), Y8
	VMOVDQU   160(AX), Y10
	VMOVDQU   192(AX), Y12
	VMOVDQU   224(AX), Y14
	VMOVDQU   8(AX), Y1
	VMOVDQU   40(AX), Y3
	VMOVDQU   72(AX), Y5
	VMOVDQU   104(AX), Y7
	VMOVDQU   136(AX), Y9
	VMOVDQU   168(AX), Y11
	VMOVDQU   200(AX), Y13
	VMOVDQU   232(AX), Y15
	VPCMPEQQ  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQQ  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	VPCMPEQQ  Y4, Y5, Y5
	VPMOVMSKB Y5, DI
	VPCMPEQQ  Y6, Y7, Y7
	VPMOVMSKB Y7, R8
	VPCMPEQQ  Y8, Y9, Y9
	VPMOVMSKB Y9, R9
	VPCMPEQQ  Y10, Y11, Y11
	VPMOVMSKB Y11, R10
	VPCMPEQQ  Y12, Y13, Y13
	VPMOVMSKB Y13, R11
	VPCMPEQQ  Y14, Y15, Y15
	VPMOVMSKB Y15, R12
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	POPCNTQ   DI, DI
	POPCNTQ   R8, R8
	POPCNTQ   R9, R9
	POPCNTQ   R10, R10
	POPCNTQ   R11, R11
	POPCNTQ   R12, R12
	ADDQ      SI, BX
	ADDQ      R8, DI
	ADDQ      DI, BX
	ADDQ      R10, R9
	ADDQ      R12, R11
	ADDQ      R11, R9
	ADDQ      R9, BX
	ADDQ      BX, DX
	ADDQ      $0x00000100, AX
	SUBQ      $0x00000100, CX
	CMPQ      CX, $0x00000108
	JGE       avx2_loop256

avx2_tail128:
	CMPQ      CX, $0x88
	JL        avx2_tail64
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y6
	VMOVDQU   8(AX), Y1
	VMOVDQU   40(AX), Y3
	VMOVDQU   72(AX), Y5
	VMOVDQU   104(AX), Y7
	VPCMPEQQ  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQQ  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	VPCMPEQQ  Y4, Y5, Y5
	VPMOVMSKB Y5, DI
	VPCMPEQQ  Y6, Y7, Y7
	VPMOVMSKB Y7, R8
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	POPCNTQ   DI, DI
	POPCNTQ   R8, R8
	ADDQ      SI, BX
	ADDQ      R8, DI
	ADDQ      DI, BX
	ADDQ      BX, DX
	ADDQ      $0x00000080, AX
	SUBQ      $0x00000080, CX

avx2_tail64:
	CMPQ      CX, $0x48
	JL        avx2_tail32
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   8(AX), Y1
	VMOVDQU   40(AX), Y3
	VPCMPEQQ  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQQ  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	POPCNTQ   BX, BX
	POPCNTQ   SI, SI
	ADDQ      SI, BX
	ADDQ      BX, DX
	ADDQ      $0x00000040, AX
	SUBQ      $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x28
	JL        avx2_tail16
	VMOVDQU   (AX), Y0
	VMOVDQU   8(AX), Y1
	VPCMPEQQ  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	POPCNTQ   BX, BX
	ADDQ      BX, DX
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	CMPQ      CX, $0x18
	JL        avx2_tail
	VMOVDQU   (AX), X0
	VMOVDQU   8(AX), X1
	VPCMPEQQ  X0, X1, X1
	VPMOVMSKB X1, BX
	POPCNTQ   BX, BX
	ADDQ      BX, DX
	ADDQ      $0x10, AX
	SUBQ      $0x10, CX

avx2_tail:
	VZEROUPPER
	SHRQ $0x03, DX
	JMP  tail

// func countPair16(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT, SSE2, SSE4.1
TEXT ·countPair16(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	XORQ DX, DX
	SUBQ $0x10, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JLE  done

generic:
	MOVQ     DX, BX
	INCQ     BX
	MOVOU    (AX), X0
	MOVOU    16(AX), X1
	PCMPEQQ  X0, X1
	PMOVMSKB X1, SI
	CMPL     SI, $0x0000ffff
	CMOVQEQ  BX, DX
	ADDQ     $0x10, AX
	SUBQ     $0x10, CX
	CMPQ     CX, $0x00
	JG       generic

done:
	MOVQ DX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000110
	JL   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	VMOVDQU    (AX), Y0
	VMOVDQU    32(AX), Y2
	VMOVDQU    64(AX), Y4
	VMOVDQU    96(AX), Y6
	VMOVDQU    128(AX), Y8
	VMOVDQU    160(AX), Y10
	VMOVDQU    192(AX), Y12
	VMOVDQU    224(AX), Y14
	VPERM2I128 $0x21, Y2, Y0, Y1
	VPERM2I128 $0x21, Y4, Y2, Y3
	VPERM2I128 $0x21, Y6, Y4, Y5
	VPERM2I128 $0x21, Y8, Y6, Y7
	VPERM2I128 $0x21, Y10, Y8, Y9
	VPERM2I128 $0x21, Y12, Y10, Y11
	VPERM2I128 $0x21, Y14, Y12, Y13
	VMOVDQU    240(AX), Y15
	VPCMPEQQ   Y0, Y1, Y1
	VPERMQ     $0xb1, Y1, Y0
	VPAND      Y1, Y0, Y0
	VPMOVMSKB  Y0, BX
	VPCMPEQQ   Y2, Y3, Y3
	VPERMQ     $0xb1, Y3, Y2
	VPAND      Y3, Y2, Y2
	VPMOVMSKB  Y2, SI
	VPCMPEQQ   Y4, Y5, Y5
	VPERMQ     $0xb1, Y5, Y4
	VPAND      Y5, Y4, Y4
	VPMOVMSKB  Y4, DI
	VPCMPEQQ   Y6, Y7, Y7
	VPERMQ     $0xb1, Y7, Y6
	VPAND      Y7, Y6, Y6
	VPMOVMSKB  Y6, R8
	VPCMPEQQ   Y8, Y9, Y9
	VPERMQ     $0xb1, Y9, Y8
	VPAND      Y9, Y8, Y8
	VPMOVMSKB  Y8, R9
	VPCMPEQQ   Y10, Y11, Y11
	VPERMQ     $0xb1, Y11, Y10
	VPAND      Y11, Y10, Y10
	VPMOVMSKB  Y10, R10
	VPCMPEQQ   Y12, Y13, Y13
	VPERMQ     $0xb1, Y13, Y12
	VPAND      Y13, Y12, Y12
	VPMOVMSKB  Y12, R11
	VPCMPEQQ   Y14, Y15, Y15
	VPERMQ     $0xb1, Y15, Y14
	VPAND      Y15, Y14, Y14
	VPMOVMSKB  Y14, R12
	POPCNTQ    BX, BX
	POPCNTQ    SI, SI
	POPCNTQ    DI, DI
	POPCNTQ    R8, R8
	POPCNTQ    R9, R9
	POPCNTQ    R10, R10
	POPCNTQ    R11, R11
	POPCNTQ    R12, R12
	ADDQ       SI, BX
	ADDQ       R8, DI
	ADDQ       DI, BX
	ADDQ       R10, R9
	ADDQ       R12, R11
	ADDQ       R11, R9
	ADDQ       R9, BX
	ADDQ       BX, DX
	ADDQ       $0x00000100, AX
	SUBQ       $0x00000100, CX
	CMPQ       CX, $0x00000110
	JGE        avx2_loop256

avx2_tail128:
	CMPQ       CX, $0x90
	JL         avx2_tail64
	VMOVDQU    (AX), Y0
	VMOVDQU    32(AX), Y2
	VMOVDQU    64(AX), Y4
	VMOVDQU    96(AX), Y6
	VPERM2I128 $0x21, Y2, Y0, Y1
	VPERM2I128 $0x21, Y4, Y2, Y3
	VPERM2I128 $0x21, Y6, Y4, Y5
	VMOVDQU    112(AX), Y7
	VPCMPEQQ   Y0, Y1, Y1
	VPERMQ     $0xb1, Y1, Y0
	VPAND      Y1, Y0, Y0
	VPMOVMSKB  Y0, BX
	VPCMPEQQ   Y2, Y3, Y3
	VPERMQ     $0xb1, Y3, Y2
	VPAND      Y3, Y2, Y2
	VPMOVMSKB  Y2, SI
	VPCMPEQQ   Y4, Y5, Y5
	VPERMQ     $0xb1, Y5, Y4
	VPAND      Y5, Y4, Y4
	VPMOVMSKB  Y4, DI
	VPCMPEQQ   Y6, Y7, Y7
	VPERMQ     $0xb1, Y7, Y6
	VPAND      Y7, Y6, Y6
	VPMOVMSKB  Y6, R8
	POPCNTQ    BX, BX
	POPCNTQ    SI, SI
	POPCNTQ    DI, DI
	POPCNTQ    R8, R8
	ADDQ       SI, BX
	ADDQ       R8, DI
	ADDQ       DI, BX
	ADDQ       BX, DX
	ADDQ       $0x00000080, AX
	SUBQ       $0x00000080, CX

avx2_tail64:
	CMPQ       CX, $0x50
	JL         avx2_tail32
	VMOVDQU    (AX), Y0
	VMOVDQU    32(AX), Y2
	VPERM2I128 $0x21, Y2, Y0, Y1
	VMOVDQU    48(AX), Y3
	VPCMPEQQ   Y0, Y1, Y1
	VPERMQ     $0xb1, Y1, Y0
	VPAND      Y1, Y0, Y0
	VPMOVMSKB  Y0, BX
	VPCMPEQQ   Y2, Y3, Y3
	VPERMQ     $0xb1, Y3, Y2
	VPAND      Y3, Y2, Y2
	VPMOVMSKB  Y2, SI
	POPCNTQ    BX, BX
	POPCNTQ    SI, SI
	ADDQ       SI, BX
	ADDQ       BX, DX
	ADDQ       $0x00000040, AX
	SUBQ       $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x30
	JL        avx2_tail16
	VMOVDQU   (AX), Y0
	VMOVDQU   16(AX), Y1
	VPCMPEQQ  Y0, Y1, Y1
	VPERMQ    $0xb1, Y1, Y0
	VPAND     Y1, Y0, Y0
	VPMOVMSKB Y0, BX
	POPCNTQ   BX, BX
	ADDQ      BX, DX
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	VZEROUPPER
	SHRQ $0x04, DX
	JMP  tail

// func countPair32(b []byte) int
// Requires: AVX, AVX2, CMOV, POPCNT, SSE2, SSE4.1
TEXT ·countPair32(SB), NOSPLIT, $0-32
	MOVQ b_base+0(FP), AX
	MOVQ b_len+8(FP), CX
	XORQ DX, DX
	SUBQ $0x20, CX
	BTL  $0x08, github·com∕segmentio∕asm∕cpu·X86+0(SB)
	JCS  avx2

tail:
	CMPQ CX, $0x00
	JLE  done

generic:
	MOVQ     DX, BX
	INCQ     BX
	MOVOU    (AX), X0
	MOVOU    16(AX), X1
	MOVOU    32(AX), X2
	MOVOU    48(AX), X3
	PCMPEQQ  X0, X2
	PCMPEQQ  X1, X3
	PMOVMSKB X2, SI
	PMOVMSKB X3, DI
	ANDL     DI, SI
	CMPL     SI, $0x0000ffff
	CMOVQEQ  BX, DX
	ADDQ     $0x20, AX
	SUBQ     $0x20, CX
	CMPQ     CX, $0x00
	JG       generic

done:
	MOVQ DX, ret+24(FP)
	RET

avx2:
	CMPQ CX, $0x00000120
	JL   avx2_tail128
	XORQ BX, BX
	XORQ SI, SI
	XORQ DI, DI
	XORQ R8, R8
	XORQ R9, R9
	XORQ R10, R10
	XORQ R11, R11
	XORQ R12, R12

avx2_loop256:
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y5
	VMOVDQU   128(AX), Y7
	VMOVDQU   160(AX), Y8
	VMOVDQU   192(AX), Y9
	VMOVDQU   224(AX), Y10
	VMOVDQU   256(AX), Y11
	VPCMPEQQ  Y0, Y2, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQQ  Y2, Y4, Y3
	VPMOVMSKB Y3, SI
	VPCMPEQQ  Y4, Y5, Y0
	VPMOVMSKB Y0, DI
	VPCMPEQQ  Y5, Y7, Y6
	VPMOVMSKB Y6, R8
	VPCMPEQQ  Y7, Y8, Y0
	VPMOVMSKB Y0, R9
	VPCMPEQQ  Y8, Y9, Y0
	VPMOVMSKB Y0, R10
	VPCMPEQQ  Y9, Y10, Y0
	VPMOVMSKB Y0, R11
	VPCMPEQQ  Y10, Y11, Y11
	VPMOVMSKB Y11, R12
	POPCNTQ   BX, BX
	SHRQ      $0x05, BX
	POPCNTQ   SI, SI
	SHRQ      $0x05, SI
	POPCNTQ   DI, DI
	SHRQ      $0x05, DI
	POPCNTQ   R8, R8
	SHRQ      $0x05, R8
	POPCNTQ   R9, R9
	SHRQ      $0x05, R9
	POPCNTQ   R10, R10
	SHRQ      $0x05, R10
	POPCNTQ   R11, R11
	SHRQ      $0x05, R11
	POPCNTQ   R12, R12
	SHRQ      $0x05, R12
	ADDQ      SI, BX
	ADDQ      R8, DI
	ADDQ      DI, BX
	ADDQ      R10, R9
	ADDQ      R12, R11
	ADDQ      R11, R9
	ADDQ      R9, BX
	ADDQ      BX, DX
	ADDQ      $0x00000100, AX
	SUBQ      $0x00000100, CX
	CMPQ      CX, $0x00000120
	JGE       avx2_loop256

avx2_tail128:
	CMPQ      CX, $0xa0
	JL        avx2_tail64
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y4
	VMOVDQU   96(AX), Y5
	VMOVDQU   128(AX), Y6
	VPCMPEQQ  Y0, Y2, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQQ  Y2, Y4, Y3
	VPMOVMSKB Y3, SI
	VPCMPEQQ  Y4, Y5, Y0
	VPMOVMSKB Y0, DI
	VPCMPEQQ  Y5, Y6, Y6
	VPMOVMSKB Y6, R8
	POPCNTQ   BX, BX
	SHRQ      $0x05, BX
	POPCNTQ   SI, SI
	SHRQ      $0x05, SI
	POPCNTQ   DI, DI
	SHRQ      $0x05, DI
	POPCNTQ   R8, R8
	SHRQ      $0x05, R8
	ADDQ      SI, BX
	ADDQ      R8, DI
	ADDQ      DI, BX
	ADDQ      BX, DX
	ADDQ      $0x00000080, AX
	SUBQ      $0x00000080, CX

avx2_tail64:
	CMPQ      CX, $0x60
	JL        avx2_tail32
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y2
	VMOVDQU   64(AX), Y3
	VPCMPEQQ  Y0, Y2, Y1
	VPMOVMSKB Y1, BX
	VPCMPEQQ  Y2, Y3, Y3
	VPMOVMSKB Y3, SI
	POPCNTQ   BX, BX
	SHRQ      $0x05, BX
	POPCNTQ   SI, SI
	SHRQ      $0x05, SI
	ADDQ      SI, BX
	ADDQ      BX, DX
	ADDQ      $0x00000040, AX
	SUBQ      $0x00000040, CX

avx2_tail32:
	CMPQ      CX, $0x40
	JL        avx2_tail16
	VMOVDQU   (AX), Y0
	VMOVDQU   32(AX), Y1
	VPCMPEQQ  Y0, Y1, Y1
	VPMOVMSKB Y1, BX
	POPCNTQ   BX, BX
	SHRQ      $0x05, BX
	ADDQ      BX, DX
	ADDQ      $0x00000020, AX
	SUBQ      $0x00000020, CX

avx2_tail16:
	VZEROUPPER
	JMP tail