; This file is generated from a similarly-named Perl script in the BoringSSL
; source tree. Do not edit by hand.

%ifidn __OUTPUT_FORMAT__, win64
default	rel
%define XMMWORD
%define YMMWORD
%define ZMMWORD
%define _CET_ENDBR

%include "openssl/boringssl_prefix_symbols_nasm.inc"
%ifndef MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX
section	.text code align=64


global	rsaz_amm52x40_x1_ifma256

ALIGN	32
rsaz_amm52x40_x1_ifma256:
	mov	QWORD[8+rsp],rdi	;WIN64 prologue
	mov	QWORD[16+rsp],rsi
	mov	rax,rsp
$L$SEH_begin_rsaz_amm52x40_x1_ifma256:
	mov	rdi,rcx
	mov	rsi,rdx
	mov	rdx,r8
	mov	rcx,r9
	mov	r8,QWORD[40+rsp]



DB	243,15,30,250
	push	rbx

	push	rbp

	push	r12

	push	r13

	push	r14

	push	r15

	lea	rsp,[((-168))+rsp]
	vmovdqa64	XMMWORD[rsp],xmm6
	vmovdqa64	XMMWORD[16+rsp],xmm7
	vmovdqa64	XMMWORD[32+rsp],xmm8
	vmovdqa64	XMMWORD[48+rsp],xmm9
	vmovdqa64	XMMWORD[64+rsp],xmm10
	vmovdqa64	XMMWORD[80+rsp],xmm11
	vmovdqa64	XMMWORD[96+rsp],xmm12
	vmovdqa64	XMMWORD[112+rsp],xmm13
	vmovdqa64	XMMWORD[128+rsp],xmm14
	vmovdqa64	XMMWORD[144+rsp],xmm15
$L$rsaz_amm52x40_x1_ifma256_body:

	vpxord	ymm0,ymm0,ymm0
	vmovdqa64	ymm3,ymm0
	vmovdqa64	ymm4,ymm0
	vmovdqa64	ymm5,ymm0
	vmovdqa64	ymm6,ymm0
	vmovdqa64	ymm7,ymm0
	vmovdqa64	ymm8,ymm0
	vmovdqa64	ymm9,ymm0
	vmovdqa64	ymm10,ymm0
	vmovdqa64	ymm11,ymm0
	vmovdqa64	ymm12,ymm0

	xor	r9d,r9d

	mov	r11,rdx
	mov	rax,0xfffffffffffff


	mov	ebx,10

ALIGN	32
$L$loop10:
	mov	r13,QWORD[r11]

	vpbroadcastq	ymm1,r13
	mov	rdx,QWORD[rsi]
	mulx	r12,r13,r13
	add	r9,r13
	mov	r10,r12
	adc	r10,0

	mov	r13,r8
	imul	r13,r9
	and	r13,rax

	vpbroadcastq	ymm2,r13
	mov	rdx,QWORD[rcx]
	mulx	r12,r13,r13
	add	r9,r13
	adc	r10,r12

	shr	r9,52
	sal	r10,12
	or	r9,r10

	vpmadd52luq	ymm3,ymm1,YMMWORD[rsi]
	vpmadd52luq	ymm4,ymm1,YMMWORD[32+rsi]
	vpmadd52luq	ymm5,ymm1,YMMWORD[64+rsi]
	vpmadd52luq	ymm6,ymm1,YMMWORD[96+rsi]
	vpmadd52luq	ymm7,ymm1,YMMWORD[128+rsi]
	vpmadd52luq	ymm8,ymm1,YMMWORD[160+rsi]
	vpmadd52luq	ymm9,ymm1,YMMWORD[192+rsi]
	vpmadd52luq	ymm10,ymm1,YMMWORD[224+rsi]
	vpmadd52luq	ymm11,ymm1,YMMWORD[256+rsi]
	vpmadd52luq	ymm12,ymm1,YMMWORD[288+rsi]

	vpmadd52luq	ymm3,ymm2,YMMWORD[rcx]
	vpmadd52luq	ymm4,ymm2,YMMWORD[32+rcx]
	vpmadd52luq	ymm5,ymm2,YMMWORD[64+rcx]
	vpmadd52luq	ymm6,ymm2,YMMWORD[96+rcx]
	vpmadd52luq	ymm7,ymm2,YMMWORD[128+rcx]
	vpmadd52luq	ymm8,ymm2,YMMWORD[160+rcx]
	vpmadd52luq	ymm9,ymm2,YMMWORD[192+rcx]
	vpmadd52luq	ymm10,ymm2,YMMWORD[224+rcx]
	vpmadd52luq	ymm11,ymm2,YMMWORD[256+rcx]
	vpmadd52luq	ymm12,ymm2,YMMWORD[288+rcx]


	valignq	ymm3,ymm4,ymm3,1
	valignq	ymm4,ymm5,ymm4,1
	valignq	ymm5,ymm6,ymm5,1
	valignq	ymm6,ymm7,ymm6,1
	valignq	ymm7,ymm8,ymm7,1
	valignq	ymm8,ymm9,ymm8,1
	valignq	ymm9,ymm10,ymm9,1
	valignq	ymm10,ymm11,ymm10,1
	valignq	ymm11,ymm12,ymm11,1
	valignq	ymm12,ymm0,ymm12,1

	vmovq	r13,xmm3
	add	r9,r13

	vpmadd52huq	ymm3,ymm1,YMMWORD[rsi]
	vpmadd52huq	ymm4,ymm1,YMMWORD[32+rsi]
	vpmadd52huq	ymm5,ymm1,YMMWORD[64+rsi]
	vpmadd52huq	ymm6,ymm1,YMMWORD[96+rsi]
	vpmadd52huq	ymm7,ymm1,YMMWORD[128+rsi]
	vpmadd52huq	ymm8,ymm1,YMMWORD[160+rsi]
	vpmadd52huq	ymm9,ymm1,YMMWORD[192+rsi]
	vpmadd52huq	ymm10,ymm1,YMMWORD[224+rsi]
	vpmadd52huq	ymm11,ymm1,YMMWORD[256+rsi]
	vpmadd52huq	ymm12,ymm1,YMMWORD[288+rsi]

	vpmadd52huq	ymm3,ymm2,YMMWORD[rcx]
	vpmadd52huq	ymm4,ymm2,YMMWORD[32+rcx]
	vpmadd52huq	ymm5,ymm2,YMMWORD[64+rcx]
	vpmadd52huq	ymm6,ymm2,YMMWORD[96+rcx]
	vpmadd52huq	ymm7,ymm2,YMMWORD[128+rcx]
	vpmadd52huq	ymm8,ymm2,YMMWORD[160+rcx]
	vpmadd52huq	ymm9,ymm2,YMMWORD[192+rcx]
	vpmadd52huq	ymm10,ymm2,YMMWORD[224+rcx]
	vpmadd52huq	ymm11,ymm2,YMMWORD[256+rcx]
	vpmadd52huq	ymm12,ymm2,YMMWORD[288+rcx]
	mov	r13,QWORD[8+r11]

	vpbroadcastq	ymm1,r13
	mov	rdx,QWORD[rsi]
	mulx	r12,r13,r13
	add	r9,r13
	mov	r10,r12
	adc	r10,0

	mov	r13,r8
	imul	r13,r9
	and	r13,rax

	vpbroadcastq	ymm2,r13
	mov	rdx,QWORD[rcx]
	mulx	r12,r13,r13
	add	r9,r13
	adc	r10,r12

	shr	r9,52
	sal	r10,12
	or	r9,r10

	vpmadd52luq	ymm3,ymm1,YMMWORD[rsi]
	vpmadd52luq	ymm4,ymm1,YMMWORD[32+rsi]
	vpmadd52luq	ymm5,ymm1,YMMWORD[64+rsi]
	vpmadd52luq	ymm6,ymm1,YMMWORD[96+rsi]
	vpmadd52luq	ymm7,ymm1,YMMWORD[128+rsi]
	vpmadd52luq	ymm8,ymm1,YMMWORD[160+rsi]
	vpmadd52luq	ymm9,ymm1,YMMWORD[192+rsi]
	vpmadd52luq	ymm10,ymm1,YMMWORD[224+rsi]
	vpmadd52luq	ymm11,ymm1,YMMWORD[256+rsi]
	vpmadd52luq	ymm12,ymm1,YMMWORD[288+rsi]

	vpmadd52luq	ymm3,ymm2,YMMWORD[rcx]
	vpmadd52luq	ymm4,ymm2,YMMWORD[32+rcx]
	vpmadd52luq	ymm5,ymm2,YMMWORD[64+rcx]
	vpmadd52luq	ymm6,ymm2,YMMWORD[96+rcx]
	vpmadd52luq	ymm7,ymm2,YMMWORD[128+rcx]
	vpmadd52luq	ymm8,ymm2,YMMWORD[160+rcx]
	vpmadd52luq	ymm9,ymm2,YMMWORD[192+rcx]
	vpmadd52luq	ymm10,ymm2,YMMWORD[224+rcx]
	vpmadd52luq	ymm11,ymm2,YMMWORD[256+rcx]
	vpmadd52luq	ymm12,ymm2,YMMWORD[288+rcx]


	valignq	ymm3,ymm4,ymm3,1
	valignq	ymm4,ymm5,ymm4,1
	valignq	ymm5,ymm6,ymm5,1
	valignq	ymm6,ymm7,ymm6,1
	valignq	ymm7,ymm8,ymm7,1
	valignq	ymm8,ymm9,ymm8,1
	valignq	ymm9,ymm10,ymm9,1
	valignq	ymm10,ymm11,ymm10,1
	valignq	ymm11,ymm12,ymm11,1
	valignq	ymm12,ymm0,ymm12,1

	vmovq	r13,xmm3
	add	r9,r13

	vpmadd52huq	ymm3,ymm1,YMMWORD[rsi]
	vpmadd52huq	ymm4,ymm1,YMMWORD[32+rsi]
	vpmadd52huq	ymm5,ymm1,YMMWORD[64+rsi]
	vpmadd52huq	ymm6,ymm1,YMMWORD[96+rsi]
	vpmadd52huq	ymm7,ymm1,YMMWORD[128+rsi]
	vpmadd52huq	ymm8,ymm1,YMMWORD[160+rsi]
	vpmadd52huq	ymm9,ymm1,YMMWORD[192+rsi]
	vpmadd52huq	ymm10,ymm1,YMMWORD[224+rsi]
	vpmadd52huq	ymm11,ymm1,YMMWORD[256+rsi]
	vpmadd52huq	ymm12,ymm1,YMMWORD[288+rsi]

	vpmadd52huq	ymm3,ymm2,YMMWORD[rcx]
	vpmadd52huq	ymm4,ymm2,YMMWORD[32+rcx]
	vpmadd52huq	ymm5,ymm2,YMMWORD[64+rcx]
	vpmadd52huq	ymm6,ymm2,YMMWORD[96+rcx]
	vpmadd52huq	ymm7,ymm2,YMMWORD[128+rcx]
	vpmadd52huq	ymm8,ymm2,YMMWORD[160+rcx]
	vpmadd52huq	ymm9,ymm2,YMMWORD[192+rcx]
	vpmadd52huq	ymm10,ymm2,YMMWORD[224+rcx]
	vpmadd52huq	ymm11,ymm2,YMMWORD[256+rcx]
	vpmadd52huq	ymm12,ymm2,YMMWORD[288+rcx]
	mov	r13,QWORD[16+r11]

	vpbroadcastq	ymm1,r13
	mov	rdx,QWORD[rsi]
	mulx	r12,r13,r13
	add	r9,r13
	mov	r10,r12
	adc	r10,0

	mov	r13,r8
	imul	r13,r9
	and	r13,rax

	vpbroadcastq	ymm2,r13
	mov	rdx,QWORD[rcx]
	mulx	r12,r13,r13
	add	r9,r13
	adc	r10,r12

	shr	r9,52
	sal	r10,12
	or	r9,r10

	vpmadd52luq	ymm3,ymm1,YMMWORD[rsi]
	vpmadd52luq	ymm4,ymm1,YMMWORD[32+rsi]
	vpmadd52luq	ymm5,ymm1,YMMWORD[64+rsi]
	vpmadd52luq	ymm6,ymm1,YMMWORD[96+rsi]
	vpmadd52luq	ymm7,ymm1,YMMWORD[128+rsi]
	vpmadd52luq	ymm8,ymm1,YMMWORD[160+rsi]
	vpmadd52luq	ymm9,ymm1,YMMWORD[192+rsi]
	vpmadd52luq	ymm10,ymm1,YMMWORD[224+rsi]
	vpmadd52luq	ymm11,ymm1,YMMWORD[256+rsi]
	vpmadd52luq	ymm12,ymm1,YMMWORD[288+rsi]

	vpmadd52luq	ymm3,ymm2,YMMWORD[rcx]
	vpmadd52luq	ymm4,ymm2,YMMWORD[32+rcx]
	vpmadd52luq	ymm5,ymm2,YMMWORD[64+rcx]
	vpmadd52luq	ymm6,ymm2,YMMWORD[96+rcx]
	vpmadd52luq	ymm7,ymm2,YMMWORD[128+rcx]
	vpmadd52luq	ymm8,ymm2,YMMWORD[160+rcx]
	vpmadd52luq	ymm9,ymm2,YMMWORD[192+rcx]
	vpmadd52luq	ymm10,ymm2,YMMWORD[224+rcx]
	vpmadd52luq	ymm11,ymm2,YMMWORD[256+rcx]
	vpmadd52luq	ymm12,ymm2,YMMWORD[288+rcx]


	valignq	ymm3,ymm4,ymm3,1
	valignq	ymm4,ymm5,ymm4,1
	valignq	ymm5,ymm6,ymm5,1
	valignq	ymm6,ymm7,ymm6,1
	valignq	ymm7,ymm8,ymm7,1
	valignq	ymm8,ymm9,ymm8,1
	valignq	ymm9,ymm10,ymm9,1
	valignq	ymm10,ymm11,ymm10,1
	valignq	ymm11,ymm12,ymm11,1
	valignq	ymm12,ymm0,ymm12,1

	vmovq	r13,xmm3
	add	r9,r13

	vpmadd52huq	ymm3,ymm1,YMMWORD[rsi]
	vpmadd52huq	ymm4,ymm1,YMMWORD[32+rsi]
	vpmadd52huq	ymm5,ymm1,YMMWORD[64+rsi]
	vpmadd52huq	ymm6,ymm1,YMMWORD[96+rsi]
	vpmadd52huq	ymm7,ymm1,YMMWORD[128+rsi]
	vpmadd52huq	ymm8,ymm1,YMMWORD[160+rsi]
	vpmadd52huq	ymm9,ymm1,YMMWORD[192+rsi]
	vpmadd52huq	ymm10,ymm1,YMMWORD[224+rsi]
	vpmadd52huq	ymm11,ymm1,YMMWORD[256+rsi]
	vpmadd52huq	ymm12,ymm1,YMMWORD[288+rsi]

	vpmadd52huq	ymm3,ymm2,YMMWORD[rcx]
	vpmadd52huq	ymm4,ymm2,YMMWORD[32+rcx]
	vpmadd52huq	ymm5,ymm2,YMMWORD[64+rcx]
	vpmadd52huq	ymm6,ymm2,YMMWORD[96+rcx]
	vpmadd52huq	ymm7,ymm2,YMMWORD[128+rcx]
	vpmadd52huq	ymm8,ymm2,YMMWORD[160+rcx]
	vpmadd52huq	ymm9,ymm2,YMMWORD[192+rcx]
	vpmadd52huq	ymm10,ymm2,YMMWORD[224+rcx]
	vpmadd52huq	ymm11,ymm2,YMMWORD[256+rcx]
	vpmadd52huq	ymm12,ymm2,YMMWORD[288+rcx]
	mov	r13,QWORD[24+r11]

	vpbroadcastq	ymm1,r13
	mov	rdx,QWORD[rsi]
	mulx	r12,r13,r13
	add	r9,r13
	mov	r10,r12
	adc	r10,0

	mov	r13,r8
	imul	r13,r9
	and	r13,rax

	vpbroadcastq	ymm2,r13
	mov	rdx,QWORD[rcx]
	mulx	r12,r13,r13
	add	r9,r13
	adc	r10,r12

	shr	r9,52
	sal	r10,12
	or	r9,r10

	vpmadd52luq	ymm3,ymm1,YMMWORD[rsi]
	vpmadd52luq	ymm4,ymm1,YMMWORD[32+rsi]
	vpmadd52luq	ymm5,ymm1,YMMWORD[64+rsi]
	vpmadd52luq	ymm6,ymm1,YMMWORD[96+rsi]
	vpmadd52luq	ymm7,ymm1,YMMWORD[128+rsi]
	vpmadd52luq	ymm8,ymm1,YMMWORD[160+rsi]
	vpmadd52luq	ymm9,ymm1,YMMWORD[192+rsi]
	vpmadd52luq	ymm10,ymm1,YMMWORD[224+rsi]
	vpmadd52luq	ymm11,ymm1,YMMWORD[256+rsi]
	vpmadd52luq	ymm12,ymm1,YMMWORD[288+rsi]

	vpmadd52luq	ymm3,ymm2,YMMWORD[rcx]
	vpmadd52luq	ymm4,ymm2,YMMWORD[32+rcx]
	vpmadd52luq	ymm5,ymm2,YMMWORD[64+rcx]
	vpmadd52luq	ymm6,ymm2,YMMWORD[96+rcx]
	vpmadd52luq	ymm7,ymm2,YMMWORD[128+rcx]
	vpmadd52luq	ymm8,ymm2,YMMWORD[160+rcx]
	vpmadd52luq	ymm9,ymm2,YMMWORD[192+rcx]
	vpmadd52luq	ymm10,ymm2,YMMWORD[224+rcx]
	vpmadd52luq	ymm11,ymm2,YMMWORD[256+rcx]
	vpmadd52luq	ymm12,ymm2,YMMWORD[288+rcx]


	valignq	ymm3,ymm4,ymm3,1
	valignq	ymm4,ymm5,ymm4,1
	valignq	ymm5,ymm6,ymm5,1
	valignq	ymm6,ymm7,ymm6,1
	valignq	ymm7,ymm8,ymm7,1
	valignq	ymm8,ymm9,ymm8,1
	valignq	ymm9,ymm10,ymm9,1
	valignq	ymm10,ymm11,ymm10,1
	valignq	ymm11,ymm12,ymm11,1
	valignq	ymm12,ymm0,ymm12,1

	vmovq	r13,xmm3
	add	r9,r13

	vpmadd52huq	ymm3,ymm1,YMMWORD[rsi]
	vpmadd52huq	ymm4,ymm1,YMMWORD[32+rsi]
	vpmadd52huq	ymm5,ymm1,YMMWORD[64+rsi]
	vpmadd52huq	ymm6,ymm1,YMMWORD[96+rsi]
	vpmadd52huq	ymm7,ymm1,YMMWORD[128+rsi]
	vpmadd52huq	ymm8,ymm1,YMMWORD[160+rsi]
	vpmadd52huq	ymm9,ymm1,YMMWORD[192+rsi]
	vpmadd52huq	ymm10,ymm1,YMMWORD[224+rsi]
	vpmadd52huq	ymm11,ymm1,YMMWORD[256+rsi]
	vpmadd52huq	ymm12,ymm1,YMMWORD[288+rsi]

	vpmadd52huq	ymm3,ymm2,YMMWORD[rcx]
	vpmadd52huq	ymm4,ymm2,YMMWORD[32+rcx]
	vpmadd52huq	ymm5,ymm2,YMMWORD[64+rcx]
	vpmadd52huq	ymm6,ymm2,YMMWORD[96+rcx]
	vpmadd52huq	ymm7,ymm2,YMMWORD[128+rcx]
	vpmadd52huq	ymm8,ymm2,YMMWORD[160+rcx]
	vpmadd52huq	ymm9,ymm2,YMMWORD[192+rcx]
	vpmadd52huq	ymm10,ymm2,YMMWORD[224+rcx]
	vpmadd52huq	ymm11,ymm2,YMMWORD[256+rcx]
	vpmadd52huq	ymm12,ymm2,YMMWORD[288+rcx]
	lea	r11,[32+r11]
	dec	ebx
	jne	NEAR $L$loop10

	vpbroadcastq	ymm0,r9
	vpblendd	ymm3,ymm3,ymm0,3



	vpsrlq	ymm0,ymm3,52
	vpsrlq	ymm1,ymm4,52
	vpsrlq	ymm2,ymm5,52
	vpsrlq	ymm23,ymm6,52
	vpsrlq	ymm24,ymm7,52
	vpsrlq	ymm25,ymm8,52
	vpsrlq	ymm26,ymm9,52
	vpsrlq	ymm27,ymm10,52
	vpsrlq	ymm28,ymm11,52
	vpsrlq	ymm29,ymm12,52


	valignq	ymm29,ymm29,ymm28,3
	valignq	ymm28,ymm28,ymm27,3
	valignq	ymm27,ymm27,ymm26,3
	valignq	ymm26,ymm26,ymm25,3
	valignq	ymm25,ymm25,ymm24,3
	valignq	ymm24,ymm24,ymm23,3
	valignq	ymm23,ymm23,ymm2,3
	valignq	ymm2,ymm2,ymm1,3
	valignq	ymm1,ymm1,ymm0,3
	valignq	ymm0,ymm0,YMMWORD[$L$zeros],3


	vpandq	ymm3,ymm3,YMMWORD[$L$mask52x4]
	vpandq	ymm4,ymm4,YMMWORD[$L$mask52x4]
	vpandq	ymm5,ymm5,YMMWORD[$L$mask52x4]
	vpandq	ymm6,ymm6,YMMWORD[$L$mask52x4]
	vpandq	ymm7,ymm7,YMMWORD[$L$mask52x4]
	vpandq	ymm8,ymm8,YMMWORD[$L$mask52x4]
	vpandq	ymm9,ymm9,YMMWORD[$L$mask52x4]
	vpandq	ymm10,ymm10,YMMWORD[$L$mask52x4]
	vpandq	ymm11,ymm11,YMMWORD[$L$mask52x4]
	vpandq	ymm12,ymm12,YMMWORD[$L$mask52x4]


	vpaddq	ymm3,ymm3,ymm0
	vpaddq	ymm4,ymm4,ymm1
	vpaddq	ymm5,ymm5,ymm2
	vpaddq	ymm6,ymm6,ymm23
	vpaddq	ymm7,ymm7,ymm24
	vpaddq	ymm8,ymm8,ymm25
	vpaddq	ymm9,ymm9,ymm26
	vpaddq	ymm10,ymm10,ymm27
	vpaddq	ymm11,ymm11,ymm28
	vpaddq	ymm12,ymm12,ymm29



	vpcmpuq	k1,ymm3,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm4,YMMWORD[$L$mask52x4],6
	kmovb	r14d,k1
	kmovb	r13d,k2
	shl	r13b,4
	or	r14b,r13b

	vpcmpuq	k1,ymm5,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm6,YMMWORD[$L$mask52x4],6
	kmovb	r13d,k1
	kmovb	r12d,k2
	shl	r12b,4
	or	r13b,r12b

	vpcmpuq	k1,ymm7,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm8,YMMWORD[$L$mask52x4],6
	kmovb	r12d,k1
	kmovb	r11d,k2
	shl	r11b,4
	or	r12b,r11b

	vpcmpuq	k1,ymm9,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm10,YMMWORD[$L$mask52x4],6
	kmovb	r11d,k1
	kmovb	r10d,k2
	shl	r10b,4
	or	r11b,r10b

	vpcmpuq	k1,ymm11,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm12,YMMWORD[$L$mask52x4],6
	kmovb	r10d,k1
	kmovb	r9d,k2
	shl	r9b,4
	or	r10b,r9b

	add	r14b,r14b
	adc	r13b,r13b
	adc	r12b,r12b
	adc	r11b,r11b
	adc	r10b,r10b


	vpcmpuq	k1,ymm3,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm4,YMMWORD[$L$mask52x4],0
	kmovb	r9d,k1
	kmovb	r8d,k2
	shl	r8b,4
	or	r9b,r8b

	vpcmpuq	k1,ymm5,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm6,YMMWORD[$L$mask52x4],0
	kmovb	r8d,k1
	kmovb	edx,k2
	shl	dl,4
	or	r8b,dl

	vpcmpuq	k1,ymm7,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm8,YMMWORD[$L$mask52x4],0
	kmovb	edx,k1
	kmovb	ecx,k2
	shl	cl,4
	or	dl,cl

	vpcmpuq	k1,ymm9,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm10,YMMWORD[$L$mask52x4],0
	kmovb	ecx,k1
	kmovb	ebx,k2
	shl	bl,4
	or	cl,bl

	vpcmpuq	k1,ymm11,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm12,YMMWORD[$L$mask52x4],0
	kmovb	ebx,k1
	kmovb	eax,k2
	shl	al,4
	or	bl,al

	add	r14b,r9b
	adc	r13b,r8b
	adc	r12b,dl
	adc	r11b,cl
	adc	r10b,bl

	xor	r14b,r9b
	xor	r13b,r8b
	xor	r12b,dl
	xor	r11b,cl
	xor	r10b,bl

	kmovb	k1,r14d
	shr	r14b,4
	kmovb	k2,r14d
	kmovb	k3,r13d
	shr	r13b,4
	kmovb	k4,r13d
	kmovb	k5,r12d
	shr	r12b,4
	kmovb	k6,r12d
	kmovb	k7,r11d

	vpsubq	ymm3{k1},ymm3,YMMWORD[$L$mask52x4]
	vpsubq	ymm4{k2},ymm4,YMMWORD[$L$mask52x4]
	vpsubq	ymm5{k3},ymm5,YMMWORD[$L$mask52x4]
	vpsubq	ymm6{k4},ymm6,YMMWORD[$L$mask52x4]
	vpsubq	ymm7{k5},ymm7,YMMWORD[$L$mask52x4]
	vpsubq	ymm8{k6},ymm8,YMMWORD[$L$mask52x4]
	vpsubq	ymm9{k7},ymm9,YMMWORD[$L$mask52x4]

	vpandq	ymm3,ymm3,YMMWORD[$L$mask52x4]
	vpandq	ymm4,ymm4,YMMWORD[$L$mask52x4]
	vpandq	ymm5,ymm5,YMMWORD[$L$mask52x4]
	vpandq	ymm6,ymm6,YMMWORD[$L$mask52x4]
	vpandq	ymm7,ymm7,YMMWORD[$L$mask52x4]
	vpandq	ymm8,ymm8,YMMWORD[$L$mask52x4]
	vpandq	ymm9,ymm9,YMMWORD[$L$mask52x4]

	shr	r11b,4
	kmovb	k1,r11d
	kmovb	k2,r10d
	shr	r10b,4
	kmovb	k3,r10d

	vpsubq	ymm10{k1},ymm10,YMMWORD[$L$mask52x4]
	vpsubq	ymm11{k2},ymm11,YMMWORD[$L$mask52x4]
	vpsubq	ymm12{k3},ymm12,YMMWORD[$L$mask52x4]

	vpandq	ymm10,ymm10,YMMWORD[$L$mask52x4]
	vpandq	ymm11,ymm11,YMMWORD[$L$mask52x4]
	vpandq	ymm12,ymm12,YMMWORD[$L$mask52x4]

	vmovdqu64	YMMWORD[rdi],ymm3
	vmovdqu64	YMMWORD[32+rdi],ymm4
	vmovdqu64	YMMWORD[64+rdi],ymm5
	vmovdqu64	YMMWORD[96+rdi],ymm6
	vmovdqu64	YMMWORD[128+rdi],ymm7
	vmovdqu64	YMMWORD[160+rdi],ymm8
	vmovdqu64	YMMWORD[192+rdi],ymm9
	vmovdqu64	YMMWORD[224+rdi],ymm10
	vmovdqu64	YMMWORD[256+rdi],ymm11
	vmovdqu64	YMMWORD[288+rdi],ymm12

	vzeroupper
	lea	rax,[rsp]

	vmovdqa64	xmm6,XMMWORD[rax]
	vmovdqa64	xmm7,XMMWORD[16+rax]
	vmovdqa64	xmm8,XMMWORD[32+rax]
	vmovdqa64	xmm9,XMMWORD[48+rax]
	vmovdqa64	xmm10,XMMWORD[64+rax]
	vmovdqa64	xmm11,XMMWORD[80+rax]
	vmovdqa64	xmm12,XMMWORD[96+rax]
	vmovdqa64	xmm13,XMMWORD[112+rax]
	vmovdqa64	xmm14,XMMWORD[128+rax]
	vmovdqa64	xmm15,XMMWORD[144+rax]
	lea	rax,[168+rsp]
	mov	r15,QWORD[rax]

	mov	r14,QWORD[8+rax]

	mov	r13,QWORD[16+rax]

	mov	r12,QWORD[24+rax]

	mov	rbp,QWORD[32+rax]

	mov	rbx,QWORD[40+rax]

	lea	rsp,[48+rax]

$L$rsaz_amm52x40_x1_ifma256_epilogue:

	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
	mov	rsi,QWORD[16+rsp]
	DB	0F3h,0C3h		;repret

$L$SEH_end_rsaz_amm52x40_x1_ifma256:
section	.rdata rdata align=8
ALIGN	32
$L$mask52x4:
	DQ	0xfffffffffffff
	DQ	0xfffffffffffff
	DQ	0xfffffffffffff
	DQ	0xfffffffffffff
section	.text

section	.text


global	rsaz_amm52x40_x2_ifma256

ALIGN	32
rsaz_amm52x40_x2_ifma256:
	mov	QWORD[8+rsp],rdi	;WIN64 prologue
	mov	QWORD[16+rsp],rsi
	mov	rax,rsp
$L$SEH_begin_rsaz_amm52x40_x2_ifma256:
	mov	rdi,rcx
	mov	rsi,rdx
	mov	rdx,r8
	mov	rcx,r9
	mov	r8,QWORD[40+rsp]



DB	243,15,30,250
	push	rbx

	push	rbp

	push	r12

	push	r13

	push	r14

	push	r15

	lea	rsp,[((-168))+rsp]
	vmovdqa64	XMMWORD[rsp],xmm6
	vmovdqa64	XMMWORD[16+rsp],xmm7
	vmovdqa64	XMMWORD[32+rsp],xmm8
	vmovdqa64	XMMWORD[48+rsp],xmm9
	vmovdqa64	XMMWORD[64+rsp],xmm10
	vmovdqa64	XMMWORD[80+rsp],xmm11
	vmovdqa64	XMMWORD[96+rsp],xmm12
	vmovdqa64	XMMWORD[112+rsp],xmm13
	vmovdqa64	XMMWORD[128+rsp],xmm14
	vmovdqa64	XMMWORD[144+rsp],xmm15
$L$rsaz_amm52x40_x2_ifma256_body:

	vpxord	ymm0,ymm0,ymm0
	vmovdqa64	ymm3,ymm0
	vmovdqa64	ymm4,ymm0
	vmovdqa64	ymm5,ymm0
	vmovdqa64	ymm6,ymm0
	vmovdqa64	ymm7,ymm0
	vmovdqa64	ymm8,ymm0
	vmovdqa64	ymm9,ymm0
	vmovdqa64	ymm10,ymm0
	vmovdqa64	ymm11,ymm0
	vmovdqa64	ymm12,ymm0

	vmovdqa64	ymm13,ymm0
	vmovdqa64	ymm14,ymm0
	vmovdqa64	ymm15,ymm0
	vmovdqa64	ymm16,ymm0
	vmovdqa64	ymm17,ymm0
	vmovdqa64	ymm18,ymm0
	vmovdqa64	ymm19,ymm0
	vmovdqa64	ymm20,ymm0
	vmovdqa64	ymm21,ymm0
	vmovdqa64	ymm22,ymm0


	xor	r9d,r9d
	xor	r15d,r15d

	mov	r11,rdx
	mov	rax,0xfffffffffffff

	mov	ebx,40

ALIGN	32
$L$loop40:
	mov	r13,QWORD[r11]

	vpbroadcastq	ymm1,r13
	mov	rdx,QWORD[rsi]
	mulx	r12,r13,r13
	add	r9,r13
	mov	r10,r12
	adc	r10,0

	mov	r13,QWORD[r8]
	imul	r13,r9
	and	r13,rax

	vpbroadcastq	ymm2,r13
	mov	rdx,QWORD[rcx]
	mulx	r12,r13,r13
	add	r9,r13
	adc	r10,r12

	shr	r9,52
	sal	r10,12
	or	r9,r10

	vpmadd52luq	ymm3,ymm1,YMMWORD[rsi]
	vpmadd52luq	ymm4,ymm1,YMMWORD[32+rsi]
	vpmadd52luq	ymm5,ymm1,YMMWORD[64+rsi]
	vpmadd52luq	ymm6,ymm1,YMMWORD[96+rsi]
	vpmadd52luq	ymm7,ymm1,YMMWORD[128+rsi]
	vpmadd52luq	ymm8,ymm1,YMMWORD[160+rsi]
	vpmadd52luq	ymm9,ymm1,YMMWORD[192+rsi]
	vpmadd52luq	ymm10,ymm1,YMMWORD[224+rsi]
	vpmadd52luq	ymm11,ymm1,YMMWORD[256+rsi]
	vpmadd52luq	ymm12,ymm1,YMMWORD[288+rsi]

	vpmadd52luq	ymm3,ymm2,YMMWORD[rcx]
	vpmadd52luq	ymm4,ymm2,YMMWORD[32+rcx]
	vpmadd52luq	ymm5,ymm2,YMMWORD[64+rcx]
	vpmadd52luq	ymm6,ymm2,YMMWORD[96+rcx]
	vpmadd52luq	ymm7,ymm2,YMMWORD[128+rcx]
	vpmadd52luq	ymm8,ymm2,YMMWORD[160+rcx]
	vpmadd52luq	ymm9,ymm2,YMMWORD[192+rcx]
	vpmadd52luq	ymm10,ymm2,YMMWORD[224+rcx]
	vpmadd52luq	ymm11,ymm2,YMMWORD[256+rcx]
	vpmadd52luq	ymm12,ymm2,YMMWORD[288+rcx]


	valignq	ymm3,ymm4,ymm3,1
	valignq	ymm4,ymm5,ymm4,1
	valignq	ymm5,ymm6,ymm5,1
	valignq	ymm6,ymm7,ymm6,1
	valignq	ymm7,ymm8,ymm7,1
	valignq	ymm8,ymm9,ymm8,1
	valignq	ymm9,ymm10,ymm9,1
	valignq	ymm10,ymm11,ymm10,1
	valignq	ymm11,ymm12,ymm11,1
	valignq	ymm12,ymm0,ymm12,1

	vmovq	r13,xmm3
	add	r9,r13

	vpmadd52huq	ymm3,ymm1,YMMWORD[rsi]
	vpmadd52huq	ymm4,ymm1,YMMWORD[32+rsi]
	vpmadd52huq	ymm5,ymm1,YMMWORD[64+rsi]
	vpmadd52huq	ymm6,ymm1,YMMWORD[96+rsi]
	vpmadd52huq	ymm7,ymm1,YMMWORD[128+rsi]
	vpmadd52huq	ymm8,ymm1,YMMWORD[160+rsi]
	vpmadd52huq	ymm9,ymm1,YMMWORD[192+rsi]
	vpmadd52huq	ymm10,ymm1,YMMWORD[224+rsi]
	vpmadd52huq	ymm11,ymm1,YMMWORD[256+rsi]
	vpmadd52huq	ymm12,ymm1,YMMWORD[288+rsi]

	vpmadd52huq	ymm3,ymm2,YMMWORD[rcx]
	vpmadd52huq	ymm4,ymm2,YMMWORD[32+rcx]
	vpmadd52huq	ymm5,ymm2,YMMWORD[64+rcx]
	vpmadd52huq	ymm6,ymm2,YMMWORD[96+rcx]
	vpmadd52huq	ymm7,ymm2,YMMWORD[128+rcx]
	vpmadd52huq	ymm8,ymm2,YMMWORD[160+rcx]
	vpmadd52huq	ymm9,ymm2,YMMWORD[192+rcx]
	vpmadd52huq	ymm10,ymm2,YMMWORD[224+rcx]
	vpmadd52huq	ymm11,ymm2,YMMWORD[256+rcx]
	vpmadd52huq	ymm12,ymm2,YMMWORD[288+rcx]
	mov	r13,QWORD[320+r11]

	vpbroadcastq	ymm1,r13
	mov	rdx,QWORD[320+rsi]
	mulx	r12,r13,r13
	add	r15,r13
	mov	r10,r12
	adc	r10,0

	mov	r13,QWORD[8+r8]
	imul	r13,r15
	and	r13,rax

	vpbroadcastq	ymm2,r13
	mov	rdx,QWORD[320+rcx]
	mulx	r12,r13,r13
	add	r15,r13
	adc	r10,r12

	shr	r15,52
	sal	r10,12
	or	r15,r10

	vpmadd52luq	ymm13,ymm1,YMMWORD[320+rsi]
	vpmadd52luq	ymm14,ymm1,YMMWORD[352+rsi]
	vpmadd52luq	ymm15,ymm1,YMMWORD[384+rsi]
	vpmadd52luq	ymm16,ymm1,YMMWORD[416+rsi]
	vpmadd52luq	ymm17,ymm1,YMMWORD[448+rsi]
	vpmadd52luq	ymm18,ymm1,YMMWORD[480+rsi]
	vpmadd52luq	ymm19,ymm1,YMMWORD[512+rsi]
	vpmadd52luq	ymm20,ymm1,YMMWORD[544+rsi]
	vpmadd52luq	ymm21,ymm1,YMMWORD[576+rsi]
	vpmadd52luq	ymm22,ymm1,YMMWORD[608+rsi]

	vpmadd52luq	ymm13,ymm2,YMMWORD[320+rcx]
	vpmadd52luq	ymm14,ymm2,YMMWORD[352+rcx]
	vpmadd52luq	ymm15,ymm2,YMMWORD[384+rcx]
	vpmadd52luq	ymm16,ymm2,YMMWORD[416+rcx]
	vpmadd52luq	ymm17,ymm2,YMMWORD[448+rcx]
	vpmadd52luq	ymm18,ymm2,YMMWORD[480+rcx]
	vpmadd52luq	ymm19,ymm2,YMMWORD[512+rcx]
	vpmadd52luq	ymm20,ymm2,YMMWORD[544+rcx]
	vpmadd52luq	ymm21,ymm2,YMMWORD[576+rcx]
	vpmadd52luq	ymm22,ymm2,YMMWORD[608+rcx]


	valignq	ymm13,ymm14,ymm13,1
	valignq	ymm14,ymm15,ymm14,1
	valignq	ymm15,ymm16,ymm15,1
	valignq	ymm16,ymm17,ymm16,1
	valignq	ymm17,ymm18,ymm17,1
	valignq	ymm18,ymm19,ymm18,1
	valignq	ymm19,ymm20,ymm19,1
	valignq	ymm20,ymm21,ymm20,1
	valignq	ymm21,ymm22,ymm21,1
	valignq	ymm22,ymm0,ymm22,1

	vmovq	r13,xmm13
	add	r15,r13

	vpmadd52huq	ymm13,ymm1,YMMWORD[320+rsi]
	vpmadd52huq	ymm14,ymm1,YMMWORD[352+rsi]
	vpmadd52huq	ymm15,ymm1,YMMWORD[384+rsi]
	vpmadd52huq	ymm16,ymm1,YMMWORD[416+rsi]
	vpmadd52huq	ymm17,ymm1,YMMWORD[448+rsi]
	vpmadd52huq	ymm18,ymm1,YMMWORD[480+rsi]
	vpmadd52huq	ymm19,ymm1,YMMWORD[512+rsi]
	vpmadd52huq	ymm20,ymm1,YMMWORD[544+rsi]
	vpmadd52huq	ymm21,ymm1,YMMWORD[576+rsi]
	vpmadd52huq	ymm22,ymm1,YMMWORD[608+rsi]

	vpmadd52huq	ymm13,ymm2,YMMWORD[320+rcx]
	vpmadd52huq	ymm14,ymm2,YMMWORD[352+rcx]
	vpmadd52huq	ymm15,ymm2,YMMWORD[384+rcx]
	vpmadd52huq	ymm16,ymm2,YMMWORD[416+rcx]
	vpmadd52huq	ymm17,ymm2,YMMWORD[448+rcx]
	vpmadd52huq	ymm18,ymm2,YMMWORD[480+rcx]
	vpmadd52huq	ymm19,ymm2,YMMWORD[512+rcx]
	vpmadd52huq	ymm20,ymm2,YMMWORD[544+rcx]
	vpmadd52huq	ymm21,ymm2,YMMWORD[576+rcx]
	vpmadd52huq	ymm22,ymm2,YMMWORD[608+rcx]
	lea	r11,[8+r11]
	dec	ebx
	jne	NEAR $L$loop40

	vpbroadcastq	ymm0,r9
	vpblendd	ymm3,ymm3,ymm0,3



	vpsrlq	ymm0,ymm3,52
	vpsrlq	ymm1,ymm4,52
	vpsrlq	ymm2,ymm5,52
	vpsrlq	ymm23,ymm6,52
	vpsrlq	ymm24,ymm7,52
	vpsrlq	ymm25,ymm8,52
	vpsrlq	ymm26,ymm9,52
	vpsrlq	ymm27,ymm10,52
	vpsrlq	ymm28,ymm11,52
	vpsrlq	ymm29,ymm12,52


	valignq	ymm29,ymm29,ymm28,3
	valignq	ymm28,ymm28,ymm27,3
	valignq	ymm27,ymm27,ymm26,3
	valignq	ymm26,ymm26,ymm25,3
	valignq	ymm25,ymm25,ymm24,3
	valignq	ymm24,ymm24,ymm23,3
	valignq	ymm23,ymm23,ymm2,3
	valignq	ymm2,ymm2,ymm1,3
	valignq	ymm1,ymm1,ymm0,3
	valignq	ymm0,ymm0,YMMWORD[$L$zeros],3


	vpandq	ymm3,ymm3,YMMWORD[$L$mask52x4]
	vpandq	ymm4,ymm4,YMMWORD[$L$mask52x4]
	vpandq	ymm5,ymm5,YMMWORD[$L$mask52x4]
	vpandq	ymm6,ymm6,YMMWORD[$L$mask52x4]
	vpandq	ymm7,ymm7,YMMWORD[$L$mask52x4]
	vpandq	ymm8,ymm8,YMMWORD[$L$mask52x4]
	vpandq	ymm9,ymm9,YMMWORD[$L$mask52x4]
	vpandq	ymm10,ymm10,YMMWORD[$L$mask52x4]
	vpandq	ymm11,ymm11,YMMWORD[$L$mask52x4]
	vpandq	ymm12,ymm12,YMMWORD[$L$mask52x4]


	vpaddq	ymm3,ymm3,ymm0
	vpaddq	ymm4,ymm4,ymm1
	vpaddq	ymm5,ymm5,ymm2
	vpaddq	ymm6,ymm6,ymm23
	vpaddq	ymm7,ymm7,ymm24
	vpaddq	ymm8,ymm8,ymm25
	vpaddq	ymm9,ymm9,ymm26
	vpaddq	ymm10,ymm10,ymm27
	vpaddq	ymm11,ymm11,ymm28
	vpaddq	ymm12,ymm12,ymm29



	vpcmpuq	k1,ymm3,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm4,YMMWORD[$L$mask52x4],6
	kmovb	r14d,k1
	kmovb	r13d,k2
	shl	r13b,4
	or	r14b,r13b

	vpcmpuq	k1,ymm5,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm6,YMMWORD[$L$mask52x4],6
	kmovb	r13d,k1
	kmovb	r12d,k2
	shl	r12b,4
	or	r13b,r12b

	vpcmpuq	k1,ymm7,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm8,YMMWORD[$L$mask52x4],6
	kmovb	r12d,k1
	kmovb	r11d,k2
	shl	r11b,4
	or	r12b,r11b

	vpcmpuq	k1,ymm9,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm10,YMMWORD[$L$mask52x4],6
	kmovb	r11d,k1
	kmovb	r10d,k2
	shl	r10b,4
	or	r11b,r10b

	vpcmpuq	k1,ymm11,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm12,YMMWORD[$L$mask52x4],6
	kmovb	r10d,k1
	kmovb	r9d,k2
	shl	r9b,4
	or	r10b,r9b

	add	r14b,r14b
	adc	r13b,r13b
	adc	r12b,r12b
	adc	r11b,r11b
	adc	r10b,r10b


	vpcmpuq	k1,ymm3,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm4,YMMWORD[$L$mask52x4],0
	kmovb	r9d,k1
	kmovb	r8d,k2
	shl	r8b,4
	or	r9b,r8b

	vpcmpuq	k1,ymm5,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm6,YMMWORD[$L$mask52x4],0
	kmovb	r8d,k1
	kmovb	edx,k2
	shl	dl,4
	or	r8b,dl

	vpcmpuq	k1,ymm7,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm8,YMMWORD[$L$mask52x4],0
	kmovb	edx,k1
	kmovb	ecx,k2
	shl	cl,4
	or	dl,cl

	vpcmpuq	k1,ymm9,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm10,YMMWORD[$L$mask52x4],0
	kmovb	ecx,k1
	kmovb	ebx,k2
	shl	bl,4
	or	cl,bl

	vpcmpuq	k1,ymm11,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm12,YMMWORD[$L$mask52x4],0
	kmovb	ebx,k1
	kmovb	eax,k2
	shl	al,4
	or	bl,al

	add	r14b,r9b
	adc	r13b,r8b
	adc	r12b,dl
	adc	r11b,cl
	adc	r10b,bl

	xor	r14b,r9b
	xor	r13b,r8b
	xor	r12b,dl
	xor	r11b,cl
	xor	r10b,bl

	kmovb	k1,r14d
	shr	r14b,4
	kmovb	k2,r14d
	kmovb	k3,r13d
	shr	r13b,4
	kmovb	k4,r13d
	kmovb	k5,r12d
	shr	r12b,4
	kmovb	k6,r12d
	kmovb	k7,r11d

	vpsubq	ymm3{k1},ymm3,YMMWORD[$L$mask52x4]
	vpsubq	ymm4{k2},ymm4,YMMWORD[$L$mask52x4]
	vpsubq	ymm5{k3},ymm5,YMMWORD[$L$mask52x4]
	vpsubq	ymm6{k4},ymm6,YMMWORD[$L$mask52x4]
	vpsubq	ymm7{k5},ymm7,YMMWORD[$L$mask52x4]
	vpsubq	ymm8{k6},ymm8,YMMWORD[$L$mask52x4]
	vpsubq	ymm9{k7},ymm9,YMMWORD[$L$mask52x4]

	vpandq	ymm3,ymm3,YMMWORD[$L$mask52x4]
	vpandq	ymm4,ymm4,YMMWORD[$L$mask52x4]
	vpandq	ymm5,ymm5,YMMWORD[$L$mask52x4]
	vpandq	ymm6,ymm6,YMMWORD[$L$mask52x4]
	vpandq	ymm7,ymm7,YMMWORD[$L$mask52x4]
	vpandq	ymm8,ymm8,YMMWORD[$L$mask52x4]
	vpandq	ymm9,ymm9,YMMWORD[$L$mask52x4]

	shr	r11b,4
	kmovb	k1,r11d
	kmovb	k2,r10d
	shr	r10b,4
	kmovb	k3,r10d

	vpsubq	ymm10{k1},ymm10,YMMWORD[$L$mask52x4]
	vpsubq	ymm11{k2},ymm11,YMMWORD[$L$mask52x4]
	vpsubq	ymm12{k3},ymm12,YMMWORD[$L$mask52x4]

	vpandq	ymm10,ymm10,YMMWORD[$L$mask52x4]
	vpandq	ymm11,ymm11,YMMWORD[$L$mask52x4]
	vpandq	ymm12,ymm12,YMMWORD[$L$mask52x4]

	vpbroadcastq	ymm0,r15
	vpblendd	ymm13,ymm13,ymm0,3



	vpsrlq	ymm0,ymm13,52
	vpsrlq	ymm1,ymm14,52
	vpsrlq	ymm2,ymm15,52
	vpsrlq	ymm23,ymm16,52
	vpsrlq	ymm24,ymm17,52
	vpsrlq	ymm25,ymm18,52
	vpsrlq	ymm26,ymm19,52
	vpsrlq	ymm27,ymm20,52
	vpsrlq	ymm28,ymm21,52
	vpsrlq	ymm29,ymm22,52


	valignq	ymm29,ymm29,ymm28,3
	valignq	ymm28,ymm28,ymm27,3
	valignq	ymm27,ymm27,ymm26,3
	valignq	ymm26,ymm26,ymm25,3
	valignq	ymm25,ymm25,ymm24,3
	valignq	ymm24,ymm24,ymm23,3
	valignq	ymm23,ymm23,ymm2,3
	valignq	ymm2,ymm2,ymm1,3
	valignq	ymm1,ymm1,ymm0,3
	valignq	ymm0,ymm0,YMMWORD[$L$zeros],3


	vpandq	ymm13,ymm13,YMMWORD[$L$mask52x4]
	vpandq	ymm14,ymm14,YMMWORD[$L$mask52x4]
	vpandq	ymm15,ymm15,YMMWORD[$L$mask52x4]
	vpandq	ymm16,ymm16,YMMWORD[$L$mask52x4]
	vpandq	ymm17,ymm17,YMMWORD[$L$mask52x4]
	vpandq	ymm18,ymm18,YMMWORD[$L$mask52x4]
	vpandq	ymm19,ymm19,YMMWORD[$L$mask52x4]
	vpandq	ymm20,ymm20,YMMWORD[$L$mask52x4]
	vpandq	ymm21,ymm21,YMMWORD[$L$mask52x4]
	vpandq	ymm22,ymm22,YMMWORD[$L$mask52x4]


	vpaddq	ymm13,ymm13,ymm0
	vpaddq	ymm14,ymm14,ymm1
	vpaddq	ymm15,ymm15,ymm2
	vpaddq	ymm16,ymm16,ymm23
	vpaddq	ymm17,ymm17,ymm24
	vpaddq	ymm18,ymm18,ymm25
	vpaddq	ymm19,ymm19,ymm26
	vpaddq	ymm20,ymm20,ymm27
	vpaddq	ymm21,ymm21,ymm28
	vpaddq	ymm22,ymm22,ymm29



	vpcmpuq	k1,ymm13,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm14,YMMWORD[$L$mask52x4],6
	kmovb	r14d,k1
	kmovb	r13d,k2
	shl	r13b,4
	or	r14b,r13b

	vpcmpuq	k1,ymm15,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm16,YMMWORD[$L$mask52x4],6
	kmovb	r13d,k1
	kmovb	r12d,k2
	shl	r12b,4
	or	r13b,r12b

	vpcmpuq	k1,ymm17,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm18,YMMWORD[$L$mask52x4],6
	kmovb	r12d,k1
	kmovb	r11d,k2
	shl	r11b,4
	or	r12b,r11b

	vpcmpuq	k1,ymm19,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm20,YMMWORD[$L$mask52x4],6
	kmovb	r11d,k1
	kmovb	r10d,k2
	shl	r10b,4
	or	r11b,r10b

	vpcmpuq	k1,ymm21,YMMWORD[$L$mask52x4],6
	vpcmpuq	k2,ymm22,YMMWORD[$L$mask52x4],6
	kmovb	r10d,k1
	kmovb	r9d,k2
	shl	r9b,4
	or	r10b,r9b

	add	r14b,r14b
	adc	r13b,r13b
	adc	r12b,r12b
	adc	r11b,r11b
	adc	r10b,r10b


	vpcmpuq	k1,ymm13,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm14,YMMWORD[$L$mask52x4],0
	kmovb	r9d,k1
	kmovb	r8d,k2
	shl	r8b,4
	or	r9b,r8b

	vpcmpuq	k1,ymm15,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm16,YMMWORD[$L$mask52x4],0
	kmovb	r8d,k1
	kmovb	edx,k2
	shl	dl,4
	or	r8b,dl

	vpcmpuq	k1,ymm17,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm18,YMMWORD[$L$mask52x4],0
	kmovb	edx,k1
	kmovb	ecx,k2
	shl	cl,4
	or	dl,cl

	vpcmpuq	k1,ymm19,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm20,YMMWORD[$L$mask52x4],0
	kmovb	ecx,k1
	kmovb	ebx,k2
	shl	bl,4
	or	cl,bl

	vpcmpuq	k1,ymm21,YMMWORD[$L$mask52x4],0
	vpcmpuq	k2,ymm22,YMMWORD[$L$mask52x4],0
	kmovb	ebx,k1
	kmovb	eax,k2
	shl	al,4
	or	bl,al

	add	r14b,r9b
	adc	r13b,r8b
	adc	r12b,dl
	adc	r11b,cl
	adc	r10b,bl

	xor	r14b,r9b
	xor	r13b,r8b
	xor	r12b,dl
	xor	r11b,cl
	xor	r10b,bl

	kmovb	k1,r14d
	shr	r14b,4
	kmovb	k2,r14d
	kmovb	k3,r13d
	shr	r13b,4
	kmovb	k4,r13d
	kmovb	k5,r12d
	shr	r12b,4
	kmovb	k6,r12d
	kmovb	k7,r11d

	vpsubq	ymm13{k1},ymm13,YMMWORD[$L$mask52x4]
	vpsubq	ymm14{k2},ymm14,YMMWORD[$L$mask52x4]
	vpsubq	ymm15{k3},ymm15,YMMWORD[$L$mask52x4]
	vpsubq	ymm16{k4},ymm16,YMMWORD[$L$mask52x4]
	vpsubq	ymm17{k5},ymm17,YMMWORD[$L$mask52x4]
	vpsubq	ymm18{k6},ymm18,YMMWORD[$L$mask52x4]
	vpsubq	ymm19{k7},ymm19,YMMWORD[$L$mask52x4]

	vpandq	ymm13,ymm13,YMMWORD[$L$mask52x4]
	vpandq	ymm14,ymm14,YMMWORD[$L$mask52x4]
	vpandq	ymm15,ymm15,YMMWORD[$L$mask52x4]
	vpandq	ymm16,ymm16,YMMWORD[$L$mask52x4]
	vpandq	ymm17,ymm17,YMMWORD[$L$mask52x4]
	vpandq	ymm18,ymm18,YMMWORD[$L$mask52x4]
	vpandq	ymm19,ymm19,YMMWORD[$L$mask52x4]

	shr	r11b,4
	kmovb	k1,r11d
	kmovb	k2,r10d
	shr	r10b,4
	kmovb	k3,r10d

	vpsubq	ymm20{k1},ymm20,YMMWORD[$L$mask52x4]
	vpsubq	ymm21{k2},ymm21,YMMWORD[$L$mask52x4]
	vpsubq	ymm22{k3},ymm22,YMMWORD[$L$mask52x4]

	vpandq	ymm20,ymm20,YMMWORD[$L$mask52x4]
	vpandq	ymm21,ymm21,YMMWORD[$L$mask52x4]
	vpandq	ymm22,ymm22,YMMWORD[$L$mask52x4]

	vmovdqu64	YMMWORD[rdi],ymm3
	vmovdqu64	YMMWORD[32+rdi],ymm4
	vmovdqu64	YMMWORD[64+rdi],ymm5
	vmovdqu64	YMMWORD[96+rdi],ymm6
	vmovdqu64	YMMWORD[128+rdi],ymm7
	vmovdqu64	YMMWORD[160+rdi],ymm8
	vmovdqu64	YMMWORD[192+rdi],ymm9
	vmovdqu64	YMMWORD[224+rdi],ymm10
	vmovdqu64	YMMWORD[256+rdi],ymm11
	vmovdqu64	YMMWORD[288+rdi],ymm12

	vmovdqu64	YMMWORD[320+rdi],ymm13
	vmovdqu64	YMMWORD[352+rdi],ymm14
	vmovdqu64	YMMWORD[384+rdi],ymm15
	vmovdqu64	YMMWORD[416+rdi],ymm16
	vmovdqu64	YMMWORD[448+rdi],ymm17
	vmovdqu64	YMMWORD[480+rdi],ymm18
	vmovdqu64	YMMWORD[512+rdi],ymm19
	vmovdqu64	YMMWORD[544+rdi],ymm20
	vmovdqu64	YMMWORD[576+rdi],ymm21
	vmovdqu64	YMMWORD[608+rdi],ymm22

	vzeroupper
	lea	rax,[rsp]

	vmovdqa64	xmm6,XMMWORD[rax]
	vmovdqa64	xmm7,XMMWORD[16+rax]
	vmovdqa64	xmm8,XMMWORD[32+rax]
	vmovdqa64	xmm9,XMMWORD[48+rax]
	vmovdqa64	xmm10,XMMWORD[64+rax]
	vmovdqa64	xmm11,XMMWORD[80+rax]
	vmovdqa64	xmm12,XMMWORD[96+rax]
	vmovdqa64	xmm13,XMMWORD[112+rax]
	vmovdqa64	xmm14,XMMWORD[128+rax]
	vmovdqa64	xmm15,XMMWORD[144+rax]
	lea	rax,[168+rsp]
	mov	r15,QWORD[rax]

	mov	r14,QWORD[8+rax]

	mov	r13,QWORD[16+rax]

	mov	r12,QWORD[24+rax]

	mov	rbp,QWORD[32+rax]

	mov	rbx,QWORD[40+rax]

	lea	rsp,[48+rax]

$L$rsaz_amm52x40_x2_ifma256_epilogue:
	mov	rdi,QWORD[8+rsp]	;WIN64 epilogue
	mov	rsi,QWORD[16+rsp]
	DB	0F3h,0C3h		;repret

$L$SEH_end_rsaz_amm52x40_x2_ifma256:
section	.text


ALIGN	32
global	extract_multiplier_2x40_win5

extract_multiplier_2x40_win5:

DB	243,15,30,250
	vmovdqa64	ymm24,YMMWORD[$L$ones]
	vpbroadcastq	ymm22,r8
	vpbroadcastq	ymm23,r9
	lea	rax,[20480+rdx]


	mov	r10,rdx


	vpxor	xmm0,xmm0,xmm0
	vmovdqa64	ymm1,ymm0
	vmovdqa64	ymm2,ymm0
	vmovdqa64	ymm3,ymm0
	vmovdqa64	ymm4,ymm0
	vmovdqa64	ymm5,ymm0
	vmovdqa64	ymm16,ymm0
	vmovdqa64	ymm17,ymm0
	vmovdqa64	ymm18,ymm0
	vmovdqa64	ymm19,ymm0
	vpxorq	ymm21,ymm21,ymm21
ALIGN	32
$L$loop_0:
	vpcmpq	k1,ymm22,ymm21,0
	vmovdqu64	ymm20,YMMWORD[rdx]
	vpblendmq	ymm0{k1},ymm0,ymm20
	vmovdqu64	ymm20,YMMWORD[32+rdx]
	vpblendmq	ymm1{k1},ymm1,ymm20
	vmovdqu64	ymm20,YMMWORD[64+rdx]
	vpblendmq	ymm2{k1},ymm2,ymm20
	vmovdqu64	ymm20,YMMWORD[96+rdx]
	vpblendmq	ymm3{k1},ymm3,ymm20
	vmovdqu64	ymm20,YMMWORD[128+rdx]
	vpblendmq	ymm4{k1},ymm4,ymm20
	vmovdqu64	ymm20,YMMWORD[160+rdx]
	vpblendmq	ymm5{k1},ymm5,ymm20
	vmovdqu64	ymm20,YMMWORD[192+rdx]
	vpblendmq	ymm16{k1},ymm16,ymm20
	vmovdqu64	ymm20,YMMWORD[224+rdx]
	vpblendmq	ymm17{k1},ymm17,ymm20
	vmovdqu64	ymm20,YMMWORD[256+rdx]
	vpblendmq	ymm18{k1},ymm18,ymm20
	vmovdqu64	ymm20,YMMWORD[288+rdx]
	vpblendmq	ymm19{k1},ymm19,ymm20
	vpaddq	ymm21,ymm21,ymm24
	add	rdx,640
	cmp	rax,rdx
	jne	NEAR $L$loop_0
	vmovdqu64	YMMWORD[rcx],ymm0
	vmovdqu64	YMMWORD[32+rcx],ymm1
	vmovdqu64	YMMWORD[64+rcx],ymm2
	vmovdqu64	YMMWORD[96+rcx],ymm3
	vmovdqu64	YMMWORD[128+rcx],ymm4
	vmovdqu64	YMMWORD[160+rcx],ymm5
	vmovdqu64	YMMWORD[192+rcx],ymm16
	vmovdqu64	YMMWORD[224+rcx],ymm17
	vmovdqu64	YMMWORD[256+rcx],ymm18
	vmovdqu64	YMMWORD[288+rcx],ymm19
	mov	rdx,r10
	vpxorq	ymm21,ymm21,ymm21
ALIGN	32
$L$loop_320:
	vpcmpq	k1,ymm23,ymm21,0
	vmovdqu64	ymm20,YMMWORD[320+rdx]
	vpblendmq	ymm0{k1},ymm0,ymm20
	vmovdqu64	ymm20,YMMWORD[352+rdx]
	vpblendmq	ymm1{k1},ymm1,ymm20
	vmovdqu64	ymm20,YMMWORD[384+rdx]
	vpblendmq	ymm2{k1},ymm2,ymm20
	vmovdqu64	ymm20,YMMWORD[416+rdx]
	vpblendmq	ymm3{k1},ymm3,ymm20
	vmovdqu64	ymm20,YMMWORD[448+rdx]
	vpblendmq	ymm4{k1},ymm4,ymm20
	vmovdqu64	ymm20,YMMWORD[480+rdx]
	vpblendmq	ymm5{k1},ymm5,ymm20
	vmovdqu64	ymm20,YMMWORD[512+rdx]
	vpblendmq	ymm16{k1},ymm16,ymm20
	vmovdqu64	ymm20,YMMWORD[544+rdx]
	vpblendmq	ymm17{k1},ymm17,ymm20
	vmovdqu64	ymm20,YMMWORD[576+rdx]
	vpblendmq	ymm18{k1},ymm18,ymm20
	vmovdqu64	ymm20,YMMWORD[608+rdx]
	vpblendmq	ymm19{k1},ymm19,ymm20
	vpaddq	ymm21,ymm21,ymm24
	add	rdx,640
	cmp	rax,rdx
	jne	NEAR $L$loop_320
	vmovdqu64	YMMWORD[320+rcx],ymm0
	vmovdqu64	YMMWORD[352+rcx],ymm1
	vmovdqu64	YMMWORD[384+rcx],ymm2
	vmovdqu64	YMMWORD[416+rcx],ymm3
	vmovdqu64	YMMWORD[448+rcx],ymm4
	vmovdqu64	YMMWORD[480+rcx],ymm5
	vmovdqu64	YMMWORD[512+rcx],ymm16
	vmovdqu64	YMMWORD[544+rcx],ymm17
	vmovdqu64	YMMWORD[576+rcx],ymm18
	vmovdqu64	YMMWORD[608+rcx],ymm19

	DB	0F3h,0C3h		;repret


section	.rdata
ALIGN	32
$L$ones:
	DQ	1,1,1,1
$L$zeros:
	DQ	0,0,0,0
section	.text

EXTERN	__imp_RtlVirtualUnwind

ALIGN	16
rsaz_avx_handler:
	push	rsi
	push	rdi
	push	rbx
	push	rbp
	push	r12
	push	r13
	push	r14
	push	r15
	pushfq
	sub	rsp,64

	mov	rax,QWORD[120+r8]
	mov	rbx,QWORD[248+r8]

	mov	rsi,QWORD[8+r9]
	mov	r11,QWORD[56+r9]

	mov	r10d,DWORD[r11]
	lea	r10,[r10*1+rsi]
	cmp	rbx,r10
	jb	NEAR $L$common_seh_tail

	mov	r10d,DWORD[4+r11]
	lea	r10,[r10*1+rsi]
	cmp	rbx,r10
	jae	NEAR $L$common_seh_tail

	mov	rax,QWORD[152+r8]

	lea	rsi,[rax]
	lea	rdi,[512+r8]
	mov	ecx,20
	DD	0xa548f3fc

	lea	rax,[216+rax]

	mov	rbx,QWORD[((-8))+rax]
	mov	rbp,QWORD[((-16))+rax]
	mov	r12,QWORD[((-24))+rax]
	mov	r13,QWORD[((-32))+rax]
	mov	r14,QWORD[((-40))+rax]
	mov	r15,QWORD[((-48))+rax]
	mov	QWORD[144+r8],rbx
	mov	QWORD[160+r8],rbp
	mov	QWORD[216+r8],r12
	mov	QWORD[224+r8],r13
	mov	QWORD[232+r8],r14
	mov	QWORD[240+r8],r15

$L$common_seh_tail:
	mov	rdi,QWORD[8+rax]
	mov	rsi,QWORD[16+rax]
	mov	QWORD[152+r8],rax
	mov	QWORD[168+r8],rsi
	mov	QWORD[176+r8],rdi

	mov	rdi,QWORD[40+r9]
	mov	rsi,r8
	mov	ecx,154
	DD	0xa548f3fc

	mov	rsi,r9
	xor	rcx,rcx
	mov	rdx,QWORD[8+rsi]
	mov	r8,QWORD[rsi]
	mov	r9,QWORD[16+rsi]
	mov	r10,QWORD[40+rsi]
	lea	r11,[56+rsi]
	lea	r12,[24+rsi]
	mov	QWORD[32+rsp],r10
	mov	QWORD[40+rsp],r11
	mov	QWORD[48+rsp],r12
	mov	QWORD[56+rsp],rcx
	call	QWORD[__imp_RtlVirtualUnwind]

	mov	eax,1
	add	rsp,64
	popfq
	pop	r15
	pop	r14
	pop	r13
	pop	r12
	pop	rbp
	pop	rbx
	pop	rdi
	pop	rsi
	DB	0F3h,0C3h		;repret


section	.pdata rdata align=4
ALIGN	4
	DD	$L$SEH_begin_rsaz_amm52x40_x1_ifma256 wrt ..imagebase
	DD	$L$SEH_end_rsaz_amm52x40_x1_ifma256 wrt ..imagebase
	DD	$L$SEH_info_rsaz_amm52x40_x1_ifma256 wrt ..imagebase

	DD	$L$SEH_begin_rsaz_amm52x40_x2_ifma256 wrt ..imagebase
	DD	$L$SEH_end_rsaz_amm52x40_x2_ifma256 wrt ..imagebase
	DD	$L$SEH_info_rsaz_amm52x40_x2_ifma256 wrt ..imagebase

section	.xdata rdata align=4
ALIGN	4
$L$SEH_info_rsaz_amm52x40_x1_ifma256:
	DB	9,0,0,0
	DD	rsaz_avx_handler wrt ..imagebase
	DD	$L$rsaz_amm52x40_x1_ifma256_body wrt ..imagebase,$L$rsaz_amm52x40_x1_ifma256_epilogue wrt ..imagebase
ALIGN	4
$L$SEH_info_rsaz_amm52x40_x2_ifma256:
	DB	9,0,0,0
	DD	rsaz_avx_handler wrt ..imagebase
	DD	$L$rsaz_amm52x40_x2_ifma256_body wrt ..imagebase,$L$rsaz_amm52x40_x2_ifma256_epilogue wrt ..imagebase

%endif
%else
; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738
ret
%endif
