vendor: OpenCV 5.0.0 snapshot at 40738fb16ceddb5fb3fea747585f7ce6abb0605b
This commit is contained in:
Gitea Mirror Bot
+529
@@ -0,0 +1,529 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
FgemmKernelAvx512FCommon.h
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the floating point matrix/matrix
|
||||
multiply operation (SGEMM and DGEMM).
|
||||
|
||||
This implementation uses AVX512F instructions.
|
||||
|
||||
--*/
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for 2 ZMMWORDs by N rows of the output
|
||||
matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
BroadcastOffset - Supplies the byte offset from matrix A to fetch elements.
|
||||
|
||||
PrefetchOffset - Optionally supplies the byte offset from matrix B to
|
||||
prefetch elements.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address into the matrix A data.
|
||||
|
||||
rbx - Supplies the address into the matrix A data plus 3 rows.
|
||||
|
||||
r13 - Supplies the address into the matrix A data plus 6 rows.
|
||||
|
||||
r14 - Supplies the address into the matrix A data plus 9 rows.
|
||||
|
||||
rsi - Supplies the address into the matrix B data.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
zmm4-zmm27 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvx512FBy2 RowCount, VectorOffset, BroadcastOffset, PrefetchOffset
|
||||
|
||||
.ifnb \PrefetchOffset\()
|
||||
prefetcht0 [rsi+\VectorOffset\()+\PrefetchOffset\()]
|
||||
prefetcht0 [rsi+r12+\VectorOffset\()+\PrefetchOffset\()]
|
||||
.endif
|
||||
.if \RowCount\() == 1
|
||||
vbroadcastsf zmm3,[rdi+\BroadcastOffset\()]
|
||||
vfmadd231pf zmm4,zmm3,ZMMWORD PTR [rsi+\VectorOffset\()]
|
||||
vfmadd231pf zmm5,zmm3,ZMMWORD PTR [rsi+r12+\VectorOffset\()]
|
||||
.else
|
||||
vmovapf zmm0,ZMMWORD PTR [rsi+\VectorOffset\()]
|
||||
vmovapf zmm1,ZMMWORD PTR [rsi+r12+\VectorOffset\()]
|
||||
EmitIfCountGE \RowCount\(), 1, "vbroadcastsf zmm3,[rdi+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd231pf zmm4,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd231pf zmm5,zmm3,zmm1"
|
||||
EmitIfCountGE \RowCount\(), 2, "vbroadcastsf zmm3,[rdi+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd231pf zmm6,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd231pf zmm7,zmm3,zmm1"
|
||||
EmitIfCountGE \RowCount\(), 3, "vbroadcastsf zmm3,[rdi+r10*2+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd231pf zmm8,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd231pf zmm9,zmm3,zmm1"
|
||||
EmitIfCountGE \RowCount\(), 4, "vbroadcastsf zmm3,[rbx+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd231pf zmm10,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd231pf zmm11,zmm3,zmm1"
|
||||
EmitIfCountGE \RowCount\(), 5, "vbroadcastsf zmm3,[rbx+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd231pf zmm12,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd231pf zmm13,zmm3,zmm1"
|
||||
EmitIfCountGE \RowCount\(), 6, "vbroadcastsf zmm3,[rbx+r10*2+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd231pf zmm14,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd231pf zmm15,zmm3,zmm1"
|
||||
EmitIfCountGE \RowCount\(), 12, "vbroadcastsf zmm3,[r13+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm16,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm17,zmm3,zmm1"
|
||||
EmitIfCountGE \RowCount\(), 12, "vbroadcastsf zmm3,[r13+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm18,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm19,zmm3,zmm1"
|
||||
EmitIfCountGE \RowCount\(), 12, "vbroadcastsf zmm3,[r13+r10*2+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm20,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm21,zmm3,zmm1"
|
||||
EmitIfCountGE \RowCount\(), 12, "vbroadcastsf zmm3,[r14+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm22,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm23,zmm3,zmm1"
|
||||
EmitIfCountGE \RowCount\(), 12, "vbroadcastsf zmm3,[r14+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm24,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm25,zmm3,zmm1"
|
||||
EmitIfCountGE \RowCount\(), 12, "vbroadcastsf zmm3,[r14+r10*2+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm26,zmm3,zmm0"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf zmm27,zmm3,zmm1"
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for 1 ZMMWORD by N rows of the output
|
||||
matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
BroadcastOffset - Supplies the byte offset from matrix A to fetch elements.
|
||||
|
||||
PrefetchOffset - Optionally supplies the byte offset from matrix B to
|
||||
prefetch elements.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address into the matrix A data.
|
||||
|
||||
rbx - Supplies the address into the matrix A data plus 3 rows.
|
||||
|
||||
r13 - Supplies the address into the matrix A data plus 6 rows.
|
||||
|
||||
r14 - Supplies the address into the matrix A data plus 9 rows.
|
||||
|
||||
rsi - Supplies the address into the matrix B data.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
zmm4-zmm27 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvx512FBy1 RowCount, VectorOffset, BroadcastOffset, PrefetchOffset
|
||||
|
||||
.ifnb \PrefetchOffset\()
|
||||
prefetcht0 [rsi+\VectorOffset\()+\PrefetchOffset\()]
|
||||
.endif
|
||||
vmovapf zmm0,ZMMWORD PTR [rsi+\VectorOffset\()]
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd231pf_bcst zmm5,zmm0,[rdi+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd231pf_bcst zmm7,zmm0,[rdi+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd231pf_bcst zmm9,zmm0,[rdi+r10*2+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd231pf_bcst zmm11,zmm0,[rbx+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd231pf_bcst zmm13,zmm0,[rbx+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd231pf_bcst zmm15,zmm0,[rbx+r10*2+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf_bcst zmm17,zmm0,[r13+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf_bcst zmm19,zmm0,[r13+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf_bcst zmm21,zmm0,[r13+r10*2+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf_bcst zmm23,zmm0,[r14+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf_bcst zmm25,zmm0,[r14+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd231pf_bcst zmm27,zmm0,[r14+r10*2+\BroadcastOffset\()]"
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to execute the block compute macro multiple
|
||||
times and advancing the matrix A and matrix B data pointers.
|
||||
|
||||
Arguments:
|
||||
|
||||
ComputeBlock - Supplies the macro to compute a single block.
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address into the matrix A data.
|
||||
|
||||
rsi - Supplies the address into the matrix B data.
|
||||
|
||||
rcx - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
zmm4-zmm27 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvx512FLoop ComputeBlock, RowCount
|
||||
|
||||
.if \RowCount\() > 3
|
||||
lea rbx,[r10*2+r10]
|
||||
.if \RowCount\() == 12
|
||||
lea r13,[rdi+rbx*2] # compute matrix A plus 6 rows
|
||||
lea r14,[r13+rbx] # compute matrix A plus 9 rows
|
||||
.endif
|
||||
add rbx,rdi # compute matrix A plus 3 rows
|
||||
.endif
|
||||
ComputeBlockLoop \ComputeBlock\(), \RowCount\(), \RowCount\() > 3
|
||||
.if \RowCount\() > 3
|
||||
lea rbx,[rax*2+rax]
|
||||
.if \RowCount\() == 12
|
||||
lea r13,[rdx+rbx*2] # compute matrix C plus 6 rows
|
||||
lea r14,[r13+rbx] # compute matrix C plus 9 rows
|
||||
.endif
|
||||
add rbx,rdx # compute matrix C plus 3 rows
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to compute matrix multiplication for a fixed set
|
||||
of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address of matrix A.
|
||||
|
||||
rsi - Supplies the address of matrix B.
|
||||
|
||||
r11 - Supplies the address of matrix A.
|
||||
|
||||
r9 - Supplies the number of columns from matrix B and matrix C to iterate
|
||||
over.
|
||||
|
||||
rdx - Supplies the address of matrix C.
|
||||
|
||||
rcx - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
rax - Supplies the length in bytes of a row from matrix C.
|
||||
|
||||
r15 - Stores the ZeroMode argument from the stack frame.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ProcessCountM RowCount
|
||||
|
||||
cmp r9,.LFgemmZmmElementCount
|
||||
jbe .LProcessRemainingCountN\@
|
||||
|
||||
.LProcessNextColumnLoop2xN\@:
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm16,zmm4"
|
||||
# clear upper block accumulators
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm17,zmm5"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm18,zmm4"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm19,zmm5"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm20,zmm4"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm21,zmm5"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm22,zmm4"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm23,zmm5"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm24,zmm4"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm25,zmm5"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm26,zmm4"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm27,zmm5"
|
||||
ComputeBlockAvx512FLoop ComputeBlockAvx512FBy2, \RowCount\()
|
||||
add rsi,r12 # advance matrix B by 64*CountK bytes
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LMultiplyAlpha2xNBlock\@
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd213pf zmm4,zmm31,ZMMWORD PTR [rdx]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd213pf zmm6,zmm31,ZMMWORD PTR [rdx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd213pf zmm8,zmm31,ZMMWORD PTR [rdx+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd213pf zmm10,zmm31,ZMMWORD PTR [rbx]"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd213pf zmm12,zmm31,ZMMWORD PTR [rbx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd213pf zmm14,zmm31,ZMMWORD PTR [rbx+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm16,zmm31,ZMMWORD PTR [r13]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm18,zmm31,ZMMWORD PTR [r13+rax]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm20,zmm31,ZMMWORD PTR [r13+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm22,zmm31,ZMMWORD PTR [r14]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm24,zmm31,ZMMWORD PTR [r14+rax]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm26,zmm31,ZMMWORD PTR [r14+rax*2]"
|
||||
jmp .LStore2xNBlock\@
|
||||
|
||||
.LMultiplyAlpha2xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf zmm4,zmm4,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf zmm6,zmm6,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf zmm8,zmm8,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf zmm10,zmm10,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmulpf zmm12,zmm12,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmulpf zmm14,zmm14,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm16,zmm16,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm18,zmm18,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm20,zmm20,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm22,zmm22,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm24,zmm24,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm26,zmm26,zmm31"
|
||||
|
||||
.LStore2xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmovupf ZMMWORD PTR [rdx],zmm4"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmovupf ZMMWORD PTR [rdx+rax],zmm6"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmovupf ZMMWORD PTR [rdx+rax*2],zmm8"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmovupf ZMMWORD PTR [rbx],zmm10"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmovupf ZMMWORD PTR [rbx+rax],zmm12"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmovupf ZMMWORD PTR [rbx+rax*2],zmm14"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r13],zmm16"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r13+rax],zmm18"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r13+rax*2],zmm20"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r14],zmm22"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r14+rax],zmm24"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r14+rax*2],zmm26"
|
||||
add rdx,64 # advance matrix C by ZMMWORD
|
||||
.if \RowCount\() > 3
|
||||
add rbx,64 # advance matrix C plus 3 rows by ZMMWORD
|
||||
.if \RowCount\() == 12
|
||||
add r13,64 # advance matrix C plus 6 rows by ZMMWORD
|
||||
add r14,64 # advance matrix C plus 9 rows by ZMMWORD
|
||||
.endif
|
||||
.endif
|
||||
sub r9,.LFgemmZmmElementCount
|
||||
|
||||
.LOutput1xNBlock\@:
|
||||
sub r9,.LFgemmZmmElementCount
|
||||
jae .LOutput1xNBlockWithMask\@
|
||||
lea rcx,[r9+.LFgemmZmmElementCount]
|
||||
# correct for over-subtract above
|
||||
mov ebp,1
|
||||
shl ebp,cl
|
||||
dec ebp
|
||||
kmovw k1,ebp # update mask for remaining columns
|
||||
xor r9,r9 # no more columns remaining
|
||||
|
||||
.LOutput1xNBlockWithMask\@:
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LMultiplyAlpha1xNBlockWithMask\@
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd213pf zmm5{k1},zmm31,ZMMWORD PTR [rdx]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd213pf zmm7{k1},zmm31,ZMMWORD PTR [rdx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd213pf zmm9{k1},zmm31,ZMMWORD PTR [rdx+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd213pf zmm11{k1},zmm31,ZMMWORD PTR [rbx]"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd213pf zmm13{k1},zmm31,ZMMWORD PTR [rbx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd213pf zmm15{k1},zmm31,ZMMWORD PTR [rbx+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm17{k1},zmm31,ZMMWORD PTR [r13]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm19{k1},zmm31,ZMMWORD PTR [r13+rax]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm21{k1},zmm31,ZMMWORD PTR [r13+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm23{k1},zmm31,ZMMWORD PTR [r14]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm25{k1},zmm31,ZMMWORD PTR [r14+rax]"
|
||||
EmitIfCountGE \RowCount\(), 12, "vfmadd213pf zmm27{k1},zmm31,ZMMWORD PTR [r14+rax*2]"
|
||||
jmp .LStore1xNBlockWithMask\@
|
||||
|
||||
.LMultiplyAlpha1xNBlockWithMask\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf zmm5,zmm5,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf zmm7,zmm7,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf zmm9,zmm9,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf zmm11,zmm11,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmulpf zmm13,zmm13,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmulpf zmm15,zmm15,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm17,zmm17,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm19,zmm19,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm21,zmm21,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm23,zmm23,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm25,zmm25,zmm31"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmulpf zmm27,zmm27,zmm31"
|
||||
|
||||
.LStore1xNBlockWithMask\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmovupf ZMMWORD PTR [rdx]{k1},zmm5"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmovupf ZMMWORD PTR [rdx+rax]{k1},zmm7"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmovupf ZMMWORD PTR [rdx+rax*2]{k1},zmm9"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmovupf ZMMWORD PTR [rbx]{k1},zmm11"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmovupf ZMMWORD PTR [rbx+rax]{k1},zmm13"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmovupf ZMMWORD PTR [rbx+rax*2]{k1},zmm15"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r13]{k1},zmm17"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r13+rax]{k1},zmm19"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r13+rax*2]{k1},zmm21"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r14]{k1},zmm23"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r14+rax]{k1},zmm25"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovupf ZMMWORD PTR [r14+rax*2]{k1},zmm27"
|
||||
add rdx,64 # advance matrix C by ZMMWORD
|
||||
mov rdi,r11 # reload matrix A
|
||||
vzeroall
|
||||
cmp r9,.LFgemmZmmElementCount
|
||||
ja .LProcessNextColumnLoop2xN\@
|
||||
test r9,r9
|
||||
jz .LExitKernel
|
||||
|
||||
.LProcessRemainingCountN\@:
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm17,zmm5"
|
||||
# clear upper block accumulators
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm19,zmm5"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm21,zmm5"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm23,zmm5"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm25,zmm5"
|
||||
EmitIfCountGE \RowCount\(), 12, "vmovapf zmm27,zmm5"
|
||||
ComputeBlockAvx512FLoop ComputeBlockAvx512FBy1, \RowCount\()
|
||||
jmp .LOutput1xNBlock\@
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates the inner kernel to compute matrix multiplication.
|
||||
|
||||
Arguments:
|
||||
|
||||
FunctionName - Supplies the name for the generated function.
|
||||
|
||||
--*/
|
||||
|
||||
.macro FgemmKernelAvx512FFunction FunctionName
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A (rdi) - Supplies the address of matrix A.
|
||||
|
||||
B (rsi) - Supplies the address of matrix B. The matrix data has been packed
|
||||
using MlasSgemmCopyPackB or MlasSgemmTransposePackB.
|
||||
|
||||
C (rdx) - Supplies the address of matrix C.
|
||||
|
||||
CountK (rcx) - Supplies the number of columns from matrix A and the number
|
||||
of rows from matrix B to iterate over.
|
||||
|
||||
CountM (r8) - Supplies the maximum number of rows that can be processed for
|
||||
matrix A and matrix C. The actual number of rows handled for this
|
||||
invocation depends on the kernel implementation.
|
||||
|
||||
CountN (r9) - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
Alpha (xmm0) - Supplies the scalar alpha multiplier (see GEMM definition).
|
||||
|
||||
ZeroMode - Supplies true if the output matrix must be zero initialized,
|
||||
else false if the output matrix is accumulated into.
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY \FunctionName\()
|
||||
|
||||
push rbp
|
||||
push rbx
|
||||
push r15
|
||||
mov .LFgemmKernelFrame_SavedR12[rsp],r12
|
||||
mov .LFgemmKernelFrame_SavedR13[rsp],r13
|
||||
mov .LFgemmKernelFrame_SavedR14[rsp],r14
|
||||
mov r11,rdi
|
||||
mov r10,.LFgemmKernelFrame_lda[rsp]
|
||||
shl r10,.LFgemmElementShift # convert lda to bytes
|
||||
mov rax,.LFgemmKernelFrame_ldc[rsp]
|
||||
shl rax,.LFgemmElementShift # convert ldc to bytes
|
||||
mov r12,rcx
|
||||
shl r12,6 # compute 64*CountK bytes
|
||||
mov ebp,-1
|
||||
kmovw k1,ebp # update mask to write all columns
|
||||
movzx r15,BYTE PTR .LFgemmKernelFrame_ZeroMode[rsp]
|
||||
vbroadcastsf zmm31,xmm0
|
||||
vzeroall
|
||||
|
||||
//
|
||||
// Process CountM rows of the matrices.
|
||||
//
|
||||
|
||||
cmp r8,12
|
||||
jb .LProcessCountMLessThan12
|
||||
mov r8d,12 # return 12 rows handled
|
||||
ProcessCountM 12
|
||||
|
||||
.LProcessCountMLessThan12:
|
||||
cmp r8,5
|
||||
ja .LProcessCountM6
|
||||
je .LProcessCountM5
|
||||
cmp r8,3
|
||||
ja .LProcessCountM4
|
||||
je .LProcessCountM3
|
||||
cmp r8,1
|
||||
je .LProcessCountM1
|
||||
|
||||
.LProcessCountM2:
|
||||
ProcessCountM 2
|
||||
|
||||
.LProcessCountM4:
|
||||
ProcessCountM 4
|
||||
|
||||
.LProcessCountM6:
|
||||
mov r8d,6 # return 6 rows handled
|
||||
ProcessCountM 6
|
||||
|
||||
//
|
||||
// Restore non-volatile registers and return.
|
||||
//
|
||||
|
||||
.LExitKernel:
|
||||
mov eax,r8d
|
||||
mov r12,.LFgemmKernelFrame_SavedR12[rsp]
|
||||
mov r13,.LFgemmKernelFrame_SavedR13[rsp]
|
||||
mov r14,.LFgemmKernelFrame_SavedR14[rsp]
|
||||
pop r15
|
||||
pop rbx
|
||||
pop rbp
|
||||
ret
|
||||
|
||||
.LProcessCountM1:
|
||||
ProcessCountM 1
|
||||
|
||||
.LProcessCountM3:
|
||||
ProcessCountM 3
|
||||
|
||||
.LProcessCountM5:
|
||||
ProcessCountM 5
|
||||
|
||||
.endm
|
||||
+451
@@ -0,0 +1,451 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
FgemmKernelAvxCommon.h
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the floating point matrix/matrix
|
||||
multiply operation (SGEMM and DGEMM).
|
||||
|
||||
This implementation uses AVX instructions.
|
||||
|
||||
--*/
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for 2 YMMWORDs by N rows of the output
|
||||
matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
BroadcastOffset - Supplies the byte offset from matrix A to fetch elements.
|
||||
|
||||
PrefetchOffset - Optionally supplies the byte offset from matrix B to
|
||||
prefetch elements.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address into the matrix A data.
|
||||
|
||||
rbx - Supplies the address into the matrix A data plus 2 rows.
|
||||
|
||||
rsi - Supplies the address into the matrix B data.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ymm8-ymm15 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvxBy16 RowCount, VectorOffset, BroadcastOffset, PrefetchOffset
|
||||
|
||||
.if \RowCount\() == 1
|
||||
vbroadcastsf ymm3,[rdi+\BroadcastOffset\()]
|
||||
vmulpf ymm4,ymm3,YMMWORD PTR [rsi+\VectorOffset\()]
|
||||
vaddpf ymm8,ymm8,ymm4
|
||||
vmulpf ymm5,ymm3,YMMWORD PTR [rsi+\VectorOffset\()+32]
|
||||
vaddpf ymm9,ymm9,ymm5
|
||||
.else
|
||||
vmovapf ymm0,YMMWORD PTR [rsi+\VectorOffset\()]
|
||||
vmovapf ymm1,YMMWORD PTR [rsi+\VectorOffset\()+32]
|
||||
EmitIfCountGE \RowCount\(), 1, "vbroadcastsf ymm3,[rdi+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf ymm4,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 1, "vaddpf ymm8,ymm8,ymm4"
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf ymm5,ymm3,ymm1"
|
||||
EmitIfCountGE \RowCount\(), 1, "vaddpf ymm9,ymm9,ymm5"
|
||||
EmitIfCountGE \RowCount\(), 2, "vbroadcastsf ymm3,[rdi+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf ymm6,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 2, "vaddpf ymm10,ymm10,ymm6"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf ymm7,ymm3,ymm1"
|
||||
EmitIfCountGE \RowCount\(), 2, "vaddpf ymm11,ymm11,ymm7"
|
||||
EmitIfCountGE \RowCount\(), 3, "vbroadcastsf ymm3,[rbx+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf ymm4,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 3, "vaddpf ymm12,ymm12,ymm4"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf ymm5,ymm3,ymm1"
|
||||
EmitIfCountGE \RowCount\(), 3, "vaddpf ymm13,ymm13,ymm5"
|
||||
EmitIfCountGE \RowCount\(), 4, "vbroadcastsf ymm3,[rbx+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf ymm6,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 4, "vaddpf ymm14,ymm14,ymm6"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf ymm7,ymm3,ymm1"
|
||||
EmitIfCountGE \RowCount\(), 4, "vaddpf ymm15,ymm15,ymm7"
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for 1 YMMWORD by N rows of the output
|
||||
matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
BroadcastOffset - Supplies the byte offset from matrix A to fetch elements.
|
||||
|
||||
PrefetchOffset - Optionally supplies the byte offset from matrix B to
|
||||
prefetch elements.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address into the matrix A data.
|
||||
|
||||
rbx - Supplies the address into the matrix A data plus 2 rows.
|
||||
|
||||
rsi - Supplies the address into the matrix B data.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ymm8-ymm15 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvxBy8 RowCount, VectorOffset, BroadcastOffset, PrefetchOffset
|
||||
|
||||
.if \RowCount\() == 1
|
||||
vbroadcastsf ymm3,[rdi+\BroadcastOffset\()]
|
||||
vmulpf ymm5,ymm3,YMMWORD PTR [rsi+\VectorOffset\()]
|
||||
vaddpf ymm9,ymm9,ymm5
|
||||
.else
|
||||
vmovapf ymm0,YMMWORD PTR [rsi+\VectorOffset\()]
|
||||
EmitIfCountGE \RowCount\(), 1, "vbroadcastsf ymm3,[rdi+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf ymm5,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 1, "vaddpf ymm9,ymm9,ymm5"
|
||||
EmitIfCountGE \RowCount\(), 2, "vbroadcastsf ymm3,[rdi+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf ymm7,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 2, "vaddpf ymm11,ymm11,ymm7"
|
||||
EmitIfCountGE \RowCount\(), 3, "vbroadcastsf ymm3,[rbx+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf ymm5,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 3, "vaddpf ymm13,ymm13,ymm5"
|
||||
EmitIfCountGE \RowCount\(), 4, "vbroadcastsf ymm3,[rbx+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf ymm7,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 4, "vaddpf ymm15,ymm15,ymm7"
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to execute the block compute macro multiple
|
||||
times and advancing the matrix A and matrix B data pointers.
|
||||
|
||||
Arguments:
|
||||
|
||||
ComputeBlock - Supplies the macro to compute a single block.
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address into the matrix A data.
|
||||
|
||||
rsi - Supplies the address into the matrix B data.
|
||||
|
||||
rcx - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ymm4-ymm15 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvxLoop ComputeBlock, RowCount
|
||||
|
||||
.if \RowCount\() > 2
|
||||
lea rbx,[rdi+r10*2] # compute matrix A plus 2 rows
|
||||
.endif
|
||||
ComputeBlockLoop \ComputeBlock\(), \RowCount\(), \RowCount\() > 2
|
||||
.if \RowCount\() > 2
|
||||
lea rbx,[rdx+rax*2] # compute matrix C plus 2 rows
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to compute matrix multiplication for a fixed set
|
||||
of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
Fallthrough - Supplies a non-blank value if the macro may fall through to
|
||||
the ExitKernel label.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address of matrix A.
|
||||
|
||||
rsi - Supplies the address of matrix B.
|
||||
|
||||
r11 - Supplies the address of matrix A.
|
||||
|
||||
r9 - Supplies the number of columns from matrix B and matrix C to iterate
|
||||
over.
|
||||
|
||||
rdx - Supplies the address of matrix C.
|
||||
|
||||
rcx - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
rax - Supplies the length in bytes of a row from matrix C.
|
||||
|
||||
r15 - Stores the ZeroMode argument from the stack frame.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ProcessCountM RowCount, Fallthrough
|
||||
|
||||
cmp r9,.LFgemmYmmElementCount
|
||||
jbe .LProcessRemainingCountN\@
|
||||
|
||||
.LProcessNextColumnLoop2xN\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vxorpf xmm8,xmm8,xmm8"
|
||||
EmitIfCountGE \RowCount\(), 1, "vxorpf xmm9,xmm9,xmm9"
|
||||
EmitIfCountGE \RowCount\(), 2, "vxorpf xmm10,xmm10,xmm10"
|
||||
EmitIfCountGE \RowCount\(), 2, "vxorpf xmm11,xmm11,xmm11"
|
||||
EmitIfCountGE \RowCount\(), 3, "vxorpf xmm12,xmm12,xmm12"
|
||||
EmitIfCountGE \RowCount\(), 3, "vxorpf xmm13,xmm13,xmm13"
|
||||
EmitIfCountGE \RowCount\(), 4, "vxorpf xmm14,xmm14,xmm14"
|
||||
EmitIfCountGE \RowCount\(), 4, "vxorpf xmm15,xmm15,xmm15"
|
||||
ComputeBlockAvxLoop ComputeBlockAvxBy16, \RowCount\()
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf ymm8,ymm8,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf ymm9,ymm9,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf ymm10,ymm10,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf ymm11,ymm11,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf ymm12,ymm12,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf ymm13,ymm13,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf ymm14,ymm14,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf ymm15,ymm15,ymm2"
|
||||
sub r9,2*.LFgemmYmmElementCount
|
||||
jb .LOutputMasked2xNBlock\@
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LStore2xNBlock\@
|
||||
EmitIfCountGE \RowCount\(), 1, "vaddpf ymm8,ymm8,YMMWORD PTR [rdx]"
|
||||
EmitIfCountGE \RowCount\(), 1, "vaddpf ymm9,ymm9,YMMWORD PTR [rdx+32]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vaddpf ymm10,ymm10,YMMWORD PTR [rdx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vaddpf ymm11,ymm11,YMMWORD PTR [rdx+rax+32]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vaddpf ymm12,ymm12,YMMWORD PTR [rbx]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vaddpf ymm13,ymm13,YMMWORD PTR [rbx+32]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vaddpf ymm14,ymm14,YMMWORD PTR [rbx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vaddpf ymm15,ymm15,YMMWORD PTR [rbx+rax+32]"
|
||||
|
||||
.LStore2xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmovupf YMMWORD PTR [rdx],ymm8"
|
||||
EmitIfCountGE \RowCount\(), 1, "vmovupf YMMWORD PTR [rdx+32],ymm9"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmovupf YMMWORD PTR [rdx+rax],ymm10"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmovupf YMMWORD PTR [rdx+rax+32],ymm11"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmovupf YMMWORD PTR [rbx],ymm12"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmovupf YMMWORD PTR [rbx+32],ymm13"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmovupf YMMWORD PTR [rbx+rax],ymm14"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmovupf YMMWORD PTR [rbx+rax+32],ymm15"
|
||||
add rdx,2*32 # advance matrix C by 2 YMMWORDs
|
||||
mov rdi,r11 # reload matrix A
|
||||
cmp r9,.LFgemmYmmElementCount
|
||||
ja .LProcessNextColumnLoop2xN\@
|
||||
test r9,r9
|
||||
jz .LExitKernel
|
||||
|
||||
.LProcessRemainingCountN\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vxorpf xmm9,xmm9,xmm9"
|
||||
EmitIfCountGE \RowCount\(), 2, "vxorpf xmm11,xmm11,xmm11"
|
||||
EmitIfCountGE \RowCount\(), 3, "vxorpf xmm13,xmm13,xmm13"
|
||||
EmitIfCountGE \RowCount\(), 4, "vxorpf xmm15,xmm15,xmm15"
|
||||
ComputeBlockAvxLoop ComputeBlockAvxBy8, \RowCount\()
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf ymm9,ymm9,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf ymm11,ymm11,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf ymm13,ymm13,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf ymm15,ymm15,ymm2"
|
||||
cmp r9,.LFgemmYmmElementCount
|
||||
jb .LOutputMasked1xNBlock\@
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LStore1xNBlock\@
|
||||
EmitIfCountGE \RowCount\(), 1, "vaddpf ymm9,ymm9,YMMWORD PTR [rdx]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vaddpf ymm11,ymm11,YMMWORD PTR [rdx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vaddpf ymm13,ymm13,YMMWORD PTR [rbx]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vaddpf ymm15,ymm15,YMMWORD PTR [rbx+rax]"
|
||||
|
||||
.LStore1xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmovupf YMMWORD PTR [rdx],ymm9"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmovupf YMMWORD PTR [rdx+rax],ymm11"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmovupf YMMWORD PTR [rbx],ymm13"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmovupf YMMWORD PTR [rbx+rax],ymm15"
|
||||
jmp .LExitKernel
|
||||
|
||||
.LOutputMasked2xNBlock\@:
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LStoreMasked2xNBlock\@
|
||||
EmitIfCountGE \RowCount\(), 1, "vaddpf ymm8,ymm8,YMMWORD PTR [rdx]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vaddpf ymm10,ymm10,YMMWORD PTR [rdx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vaddpf ymm12,ymm12,YMMWORD PTR [rbx]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vaddpf ymm14,ymm14,YMMWORD PTR [rbx+rax]"
|
||||
|
||||
.LStoreMasked2xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmovupf YMMWORD PTR [rdx],ymm8"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmovupf YMMWORD PTR [rdx+rax],ymm10"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmovupf YMMWORD PTR [rbx],ymm12"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmovupf YMMWORD PTR [rbx+rax],ymm14"
|
||||
add rdx,32 # advance matrix C by YMMWORD
|
||||
.if \RowCount\() > 2
|
||||
add rbx,32 # advance matrix C plus 2 rows by YMMWORD
|
||||
.endif
|
||||
add r9,.LFgemmYmmElementCount # correct for over-subtract above
|
||||
|
||||
.LOutputMasked1xNBlock\@:
|
||||
neg r9
|
||||
lea rdi,C_UNDERSCORE(MlasMaskMoveTableAvx)[rip+8*4]
|
||||
vmovdqu ymm0,YMMWORD PTR [rdi+r9*.LFgemmElementSize]
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LStoreMasked1xNBlock\@
|
||||
EmitIfCountGE \RowCount\(), 1, "vmaskmovpf ymm8,ymm0,YMMWORD PTR [rdx]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmaskmovpf ymm10,ymm0,YMMWORD PTR [rdx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmaskmovpf ymm12,ymm0,YMMWORD PTR [rbx]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmaskmovpf ymm14,ymm0,YMMWORD PTR [rbx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 1, "vaddpf ymm9,ymm9,ymm8"
|
||||
EmitIfCountGE \RowCount\(), 2, "vaddpf ymm11,ymm11,ymm10"
|
||||
EmitIfCountGE \RowCount\(), 3, "vaddpf ymm13,ymm13,ymm12"
|
||||
EmitIfCountGE \RowCount\(), 4, "vaddpf ymm15,ymm15,ymm14"
|
||||
|
||||
.LStoreMasked1xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmaskmovpf YMMWORD PTR [rdx],ymm0,ymm9"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmaskmovpf YMMWORD PTR [rdx+rax],ymm0,ymm11"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmaskmovpf YMMWORD PTR [rbx],ymm0,ymm13"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmaskmovpf YMMWORD PTR [rbx+rax],ymm0,ymm15"
|
||||
.ifb \Fallthrough\()
|
||||
jmp .LExitKernel
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates the inner kernel to compute matrix multiplication.
|
||||
|
||||
Arguments:
|
||||
|
||||
FunctionName - Supplies the name for the generated function.
|
||||
|
||||
--*/
|
||||
|
||||
.macro FgemmKernelAvxFunction FunctionName
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A (rdi) - Supplies the address of matrix A.
|
||||
|
||||
B (rsi) - Supplies the address of matrix B. The matrix data has been packed
|
||||
using MlasSgemmCopyPackB or MlasSgemmTransposePackB.
|
||||
|
||||
C (rdx) - Supplies the address of matrix C.
|
||||
|
||||
CountK (rcx) - Supplies the number of columns from matrix A and the number
|
||||
of rows from matrix B to iterate over.
|
||||
|
||||
CountM (r8) - Supplies the maximum number of rows that can be processed for
|
||||
matrix A and matrix C. The actual number of rows handled for this
|
||||
invocation depends on the kernel implementation.
|
||||
|
||||
CountN (r9) - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
Alpha (xmm0) - Supplies the scalar alpha multiplier (see GEMM definition).
|
||||
|
||||
ZeroMode - Supplies true if the output matrix must be zero initialized,
|
||||
else false if the output matrix is accumulated into.
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY \FunctionName\()
|
||||
|
||||
push rbp
|
||||
push rbx
|
||||
push r15
|
||||
mov r11,rdi
|
||||
mov r10,.LFgemmKernelFrame_lda[rsp]
|
||||
shl r10,.LFgemmElementShift # convert lda to bytes
|
||||
mov rax,.LFgemmKernelFrame_ldc[rsp]
|
||||
shl rax,.LFgemmElementShift # convert ldc to bytes
|
||||
movzx r15,BYTE PTR .LFgemmKernelFrame_ZeroMode[rsp]
|
||||
vmovsf .LFgemmKernelFrame_alpha[rsp],xmm0
|
||||
vbroadcastsf ymm2,.LFgemmKernelFrame_alpha[rsp]
|
||||
|
||||
//
|
||||
// Process 4 rows of the matrices.
|
||||
//
|
||||
|
||||
cmp r8,4
|
||||
jb .LProcessCountMLessThan4
|
||||
mov r8d,4 # return 4 rows handled
|
||||
ProcessCountM 4, Fallthrough
|
||||
|
||||
//
|
||||
// Restore non-volatile registers and return.
|
||||
//
|
||||
|
||||
.LExitKernel:
|
||||
vzeroupper
|
||||
mov eax,r8d
|
||||
pop r15
|
||||
pop rbx
|
||||
pop rbp
|
||||
ret
|
||||
|
||||
//
|
||||
// Process 2 rows of the matrices.
|
||||
//
|
||||
|
||||
.LProcessCountMLessThan4:
|
||||
cmp r8,2
|
||||
jb .LProcessCountMLessThan2
|
||||
mov r8d,2 # return 2 rows handled
|
||||
ProcessCountM 2
|
||||
|
||||
//
|
||||
// Process 1 row of the matrices.
|
||||
//
|
||||
|
||||
.LProcessCountMLessThan2:
|
||||
ProcessCountM 1
|
||||
|
||||
.endm
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
FgemmKernelCommon.h
|
||||
|
||||
Abstract:
|
||||
|
||||
This module contains common kernel macros and structures for the floating
|
||||
point matrix/matrix multiply operation (SGEMM and DGEMM).
|
||||
|
||||
--*/
|
||||
|
||||
//
|
||||
// Stack frame layout for the floating point kernels.
|
||||
//
|
||||
|
||||
.equ .LFgemmKernelFrame_SavedR12, -32
|
||||
.equ .LFgemmKernelFrame_SavedR13, -24
|
||||
.equ .LFgemmKernelFrame_SavedR14, -16
|
||||
.equ .LFgemmKernelFrame_alpha, -8
|
||||
.equ .LFgemmKernelFrame_SavedR15, 0
|
||||
.equ .LFgemmKernelFrame_SavedRbx, 8
|
||||
.equ .LFgemmKernelFrame_SavedRbp, 16
|
||||
.equ .LFgemmKernelFrame_ReturnAddress, 24
|
||||
.equ .LFgemmKernelFrame_lda, 32
|
||||
.equ .LFgemmKernelFrame_ldc, 40
|
||||
.equ .LFgemmKernelFrame_ZeroMode, 48
|
||||
|
||||
//
|
||||
// Define the number of elements per vector register.
|
||||
//
|
||||
|
||||
.equ .LFgemmXmmElementCount, 16 / .LFgemmElementSize
|
||||
.equ .LFgemmYmmElementCount, 32 / .LFgemmElementSize
|
||||
.equ .LFgemmZmmElementCount, 64 / .LFgemmElementSize
|
||||
|
||||
//
|
||||
// Define the typed instruction template.
|
||||
//
|
||||
|
||||
#define FGEMM_TYPED_INSTRUCTION(Untyped, Typed) \
|
||||
.macro Untyped Operand:vararg; Typed \Operand\(); .endm;
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to execute the block compute macro multiple
|
||||
times and advancing the matrix A and matrix B data pointers.
|
||||
|
||||
Arguments:
|
||||
|
||||
ComputeBlock - Supplies the macro to compute a single block.
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
AdvanceMatrixAPlusRows - Supplies a non-zero value if the data pointer
|
||||
in rbx should also be advanced as part of the loop.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address into the matrix A data.
|
||||
|
||||
rbx - Supplies the address into the matrix A data plus 3 rows.
|
||||
|
||||
rsi - Supplies the address into the matrix B data.
|
||||
|
||||
rcx - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
ymm4-ymm15 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockLoop ComputeBlock, RowCount, AdvanceMatrixAPlusRows
|
||||
|
||||
mov rbp,rcx # reload CountK
|
||||
sub rbp,4
|
||||
jb .LProcessRemainingBlocks\@
|
||||
|
||||
.LComputeBlockBy4Loop\@:
|
||||
\ComputeBlock\() \RowCount\(), 0, .LFgemmElementSize*0, 64*4
|
||||
\ComputeBlock\() \RowCount\(), 2*32, .LFgemmElementSize*1, 64*4
|
||||
add_immed rsi,2*2*32 # advance matrix B by 128 bytes
|
||||
\ComputeBlock\() \RowCount\(), 0, .LFgemmElementSize*2, 64*4
|
||||
\ComputeBlock\() \RowCount\(), 2*32, .LFgemmElementSize*3, 64*4
|
||||
add_immed rsi,2*2*32 # advance matrix B by 128 bytes
|
||||
add rdi,4*.LFgemmElementSize # advance matrix A by 4 elements
|
||||
.if \RowCount\() > 3
|
||||
add rbx,4*.LFgemmElementSize # advance matrix A plus rows by 4 elements
|
||||
.if \RowCount\() == 12
|
||||
add r13,4*.LFgemmElementSize
|
||||
add r14,4*.LFgemmElementSize
|
||||
.endif
|
||||
.endif
|
||||
sub rbp,4
|
||||
jae .LComputeBlockBy4Loop\@
|
||||
|
||||
.LProcessRemainingBlocks\@:
|
||||
add rbp,4 # correct for over-subtract above
|
||||
jz .LOutputBlock\@
|
||||
|
||||
.LComputeBlockBy1Loop\@:
|
||||
\ComputeBlock\() \RowCount\(), 0, 0
|
||||
add rsi,2*32 # advance matrix B by 64 bytes
|
||||
add rdi,.LFgemmElementSize # advance matrix A by 1 element
|
||||
.if \RowCount\() > 3
|
||||
add rbx,.LFgemmElementSize # advance matrix A plus rows by 1 element
|
||||
.if \RowCount\() == 12
|
||||
add r13,.LFgemmElementSize
|
||||
add r14,.LFgemmElementSize
|
||||
.endif
|
||||
.endif
|
||||
dec rbp
|
||||
jne .LComputeBlockBy1Loop\@
|
||||
|
||||
.LOutputBlock\@:
|
||||
|
||||
.endm
|
||||
+512
@@ -0,0 +1,512 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
FgemmKernelFma3Common.h
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the floating point matrix/matrix
|
||||
multiply operation (SGEMM and DGEMM).
|
||||
|
||||
This implementation uses AVX fused multiply/add instructions.
|
||||
|
||||
--*/
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for 2 YMMWORDs by N rows of the output
|
||||
matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
BroadcastOffset - Supplies the byte offset from matrix A to fetch elements.
|
||||
|
||||
PrefetchOffset - Optionally supplies the byte offset from matrix B to
|
||||
prefetch elements.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address into the matrix A data.
|
||||
|
||||
rbx - Supplies the address into the matrix A data plus 3 rows.
|
||||
|
||||
rsi - Supplies the address into the matrix B data.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ymm4-ymm15 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockFma3By2 RowCount, VectorOffset, BroadcastOffset, PrefetchOffset
|
||||
|
||||
.ifnb \PrefetchOffset\()
|
||||
prefetcht0 [rsi+\VectorOffset\()+\PrefetchOffset\()]
|
||||
.endif
|
||||
.if \RowCount\() == 1
|
||||
vbroadcastsf ymm3,[rdi+\BroadcastOffset\()]
|
||||
vfmadd231pf ymm4,ymm3,YMMWORD PTR [rsi+\VectorOffset\()]
|
||||
vfmadd231pf ymm5,ymm3,YMMWORD PTR [rsi+\VectorOffset\()+32]
|
||||
.else
|
||||
vmovapf ymm0,YMMWORD PTR [rsi+\VectorOffset\()]
|
||||
vmovapf ymm1,YMMWORD PTR [rsi+\VectorOffset\()+32]
|
||||
EmitIfCountGE \RowCount\(), 1, "vbroadcastsf ymm3,[rdi+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd231pf ymm4,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd231pf ymm5,ymm3,ymm1"
|
||||
EmitIfCountGE \RowCount\(), 2, "vbroadcastsf ymm3,[rdi+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd231pf ymm6,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd231pf ymm7,ymm3,ymm1"
|
||||
EmitIfCountGE \RowCount\(), 3, "vbroadcastsf ymm3,[rdi+r10*2+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd231pf ymm8,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd231pf ymm9,ymm3,ymm1"
|
||||
EmitIfCountGE \RowCount\(), 4, "vbroadcastsf ymm3,[rbx+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd231pf ymm10,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd231pf ymm11,ymm3,ymm1"
|
||||
EmitIfCountGE \RowCount\(), 5, "vbroadcastsf ymm3,[rbx+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd231pf ymm12,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd231pf ymm13,ymm3,ymm1"
|
||||
EmitIfCountGE \RowCount\(), 6, "vbroadcastsf ymm3,[rbx+r10*2+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd231pf ymm14,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd231pf ymm15,ymm3,ymm1"
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for 1 YMMWORD by N rows of the output
|
||||
matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
BroadcastOffset - Supplies the byte offset from matrix A to fetch elements.
|
||||
|
||||
PrefetchOffset - Optionally supplies the byte offset from matrix B to
|
||||
prefetch elements.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address into the matrix A data.
|
||||
|
||||
rbx - Supplies the address into the matrix A data plus 3 rows.
|
||||
|
||||
rsi - Supplies the address into the matrix B data.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ymm4-ymm15 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockFma3By1 RowCount, VectorOffset, BroadcastOffset, PrefetchOffset
|
||||
|
||||
.ifnb \PrefetchOffset\()
|
||||
prefetcht0 [rsi+\VectorOffset\()+\PrefetchOffset\()]
|
||||
.endif
|
||||
.if \RowCount\() == 1
|
||||
vbroadcastsf ymm3,[rdi+\BroadcastOffset\()]
|
||||
vfmadd231pf ymm5,ymm3,YMMWORD PTR [rsi+\VectorOffset\()]
|
||||
.else
|
||||
vmovapf ymm0,YMMWORD PTR [rsi+\VectorOffset\()]
|
||||
EmitIfCountGE \RowCount\(), 1, "vbroadcastsf ymm3,[rdi+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd231pf ymm5,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 2, "vbroadcastsf ymm3,[rdi+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd231pf ymm7,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 3, "vbroadcastsf ymm3,[rdi+r10*2+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd231pf ymm9,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 4, "vbroadcastsf ymm3,[rbx+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd231pf ymm11,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 5, "vbroadcastsf ymm3,[rbx+r10+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd231pf ymm13,ymm3,ymm0"
|
||||
EmitIfCountGE \RowCount\(), 6, "vbroadcastsf ymm3,[rbx+r10*2+\BroadcastOffset\()]"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd231pf ymm15,ymm3,ymm0"
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to execute the block compute macro multiple
|
||||
times and advancing the matrix A and matrix B data pointers.
|
||||
|
||||
Arguments:
|
||||
|
||||
ComputeBlock - Supplies the macro to compute a single block.
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address into the matrix A data.
|
||||
|
||||
rsi - Supplies the address into the matrix B data.
|
||||
|
||||
rcx - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ymm4-ymm15 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockFma3Loop ComputeBlock, RowCount
|
||||
|
||||
.if \RowCount\() > 3
|
||||
lea rbx,[r10*2+r10]
|
||||
add rbx,rdi # compute matrix A plus 3 rows
|
||||
.endif
|
||||
ComputeBlockLoop \ComputeBlock\(), \RowCount\(), \RowCount\() > 3
|
||||
vbroadcastsf ymm2,[rsp+.LFgemmKernelFrame_alpha]
|
||||
.if \RowCount\() > 3
|
||||
lea rbx,[rax*2+rax]
|
||||
add rbx,rdx # compute matrix C plus 3 rows
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to compute matrix multiplication for a fixed set
|
||||
of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
Fallthrough - Supplies a non-blank value if the macro may fall through to
|
||||
the ExitKernelAndZeroUpper label.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address of matrix A.
|
||||
|
||||
rsi - Supplies the address of matrix B.
|
||||
|
||||
r11 - Supplies the address of matrix A.
|
||||
|
||||
r9 - Supplies the number of columns from matrix B and matrix C to iterate
|
||||
over.
|
||||
|
||||
rdx - Supplies the address of matrix C.
|
||||
|
||||
rcx - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
rax - Supplies the length in bytes of a row from matrix C.
|
||||
|
||||
r15 - Stores the ZeroMode argument from the stack frame.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ProcessCountM RowCount, Fallthrough
|
||||
|
||||
cmp r9,.LFgemmYmmElementCount
|
||||
jbe .LProcessRemainingCountN\@
|
||||
|
||||
.LProcessNextColumnLoop2xN\@:
|
||||
ComputeBlockFma3Loop ComputeBlockFma3By2, \RowCount\()
|
||||
EmitIfCountGE \RowCount\(), 1, "prefetcht0 [rdx+64]"
|
||||
EmitIfCountGE \RowCount\(), 2, "prefetcht0 [rdx+rax+64]"
|
||||
EmitIfCountGE \RowCount\(), 3, "prefetcht0 [rdx+rax*2+64]"
|
||||
EmitIfCountGE \RowCount\(), 4, "prefetcht0 [rbx+64]"
|
||||
EmitIfCountGE \RowCount\(), 5, "prefetcht0 [rbx+rax+64]"
|
||||
EmitIfCountGE \RowCount\(), 6, "prefetcht0 [rbx+rax*2+64]"
|
||||
sub r9,2*.LFgemmYmmElementCount
|
||||
jb .LOutputMasked2xNBlock\@
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LMultiplyAlpha2xNBlock\@
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd213pf ymm4,ymm2,YMMWORD PTR [rdx]"
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd213pf ymm5,ymm2,YMMWORD PTR [rdx+32]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd213pf ymm6,ymm2,YMMWORD PTR [rdx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd213pf ymm7,ymm2,YMMWORD PTR [rdx+rax+32]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd213pf ymm8,ymm2,YMMWORD PTR [rdx+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd213pf ymm9,ymm2,YMMWORD PTR [rdx+rax*2+32]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd213pf ymm10,ymm2,YMMWORD PTR [rbx]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd213pf ymm11,ymm2,YMMWORD PTR [rbx+32]"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd213pf ymm12,ymm2,YMMWORD PTR [rbx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd213pf ymm13,ymm2,YMMWORD PTR [rbx+rax+32]"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd213pf ymm14,ymm2,YMMWORD PTR [rbx+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd213pf ymm15,ymm2,YMMWORD PTR [rbx+rax*2+32]"
|
||||
jmp .LStore2xNBlock\@
|
||||
|
||||
.LMultiplyAlpha2xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf ymm4,ymm4,ymm2"
|
||||
# multiply by alpha
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf ymm5,ymm5,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf ymm6,ymm6,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf ymm7,ymm7,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf ymm8,ymm8,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf ymm9,ymm9,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf ymm10,ymm10,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf ymm11,ymm11,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmulpf ymm12,ymm12,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmulpf ymm13,ymm13,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmulpf ymm14,ymm14,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmulpf ymm15,ymm15,ymm2"
|
||||
|
||||
.LStore2xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmovupf YMMWORD PTR [rdx],ymm4"
|
||||
EmitIfCountGE \RowCount\(), 1, "vmovupf YMMWORD PTR [rdx+32],ymm5"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmovupf YMMWORD PTR [rdx+rax],ymm6"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmovupf YMMWORD PTR [rdx+rax+32],ymm7"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmovupf YMMWORD PTR [rdx+rax*2],ymm8"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmovupf YMMWORD PTR [rdx+rax*2+32],ymm9"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmovupf YMMWORD PTR [rbx],ymm10"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmovupf YMMWORD PTR [rbx+32],ymm11"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmovupf YMMWORD PTR [rbx+rax],ymm12"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmovupf YMMWORD PTR [rbx+rax+32],ymm13"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmovupf YMMWORD PTR [rbx+rax*2],ymm14"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmovupf YMMWORD PTR [rbx+rax*2+32],ymm15"
|
||||
add rdx,2*32 # advance matrix C by 2 YMMWORDs
|
||||
mov rdi,r11 # reload matrix A
|
||||
vzeroall
|
||||
cmp r9,.LFgemmYmmElementCount
|
||||
ja .LProcessNextColumnLoop2xN\@
|
||||
test r9,r9
|
||||
jz .LExitKernel
|
||||
|
||||
.LProcessRemainingCountN\@:
|
||||
ComputeBlockFma3Loop ComputeBlockFma3By1, \RowCount\()
|
||||
cmp r9,.LFgemmYmmElementCount
|
||||
jb .LOutputMasked1xNBlock\@
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LMultiplyAlpha1xNBlock\@
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd213pf ymm5,ymm2,YMMWORD PTR [rdx]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd213pf ymm7,ymm2,YMMWORD PTR [rdx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd213pf ymm9,ymm2,YMMWORD PTR [rdx+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd213pf ymm11,ymm2,YMMWORD PTR [rbx]"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd213pf ymm13,ymm2,YMMWORD PTR [rbx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd213pf ymm15,ymm2,YMMWORD PTR [rbx+rax*2]"
|
||||
jmp .LStore1xNBlock\@
|
||||
|
||||
.LMultiplyAlpha1xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf ymm5,ymm5,ymm2"
|
||||
# multiply by alpha
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf ymm7,ymm7,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf ymm9,ymm9,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf ymm11,ymm11,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmulpf ymm13,ymm13,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmulpf ymm15,ymm15,ymm2"
|
||||
|
||||
.LStore1xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmovupf YMMWORD PTR [rdx],ymm5"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmovupf YMMWORD PTR [rdx+rax],ymm7"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmovupf YMMWORD PTR [rdx+rax*2],ymm9"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmovupf YMMWORD PTR [rbx],ymm11"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmovupf YMMWORD PTR [rbx+rax],ymm13"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmovupf YMMWORD PTR [rbx+rax*2],ymm15"
|
||||
jmp .LExitKernelAndZeroUpper
|
||||
|
||||
.LOutputMasked2xNBlock\@:
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LMultiplyAlphaMasked2xNBlock\@
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd213pf ymm4,ymm2,YMMWORD PTR [rdx]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd213pf ymm6,ymm2,YMMWORD PTR [rdx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd213pf ymm8,ymm2,YMMWORD PTR [rdx+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd213pf ymm10,ymm2,YMMWORD PTR [rbx]"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd213pf ymm12,ymm2,YMMWORD PTR [rbx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd213pf ymm14,ymm2,YMMWORD PTR [rbx+rax*2]"
|
||||
jmp .LStoreMasked2xNBlock\@
|
||||
|
||||
.LMultiplyAlphaMasked2xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf ymm4,ymm4,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf ymm6,ymm6,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf ymm8,ymm8,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf ymm10,ymm10,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmulpf ymm12,ymm12,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmulpf ymm14,ymm14,ymm2"
|
||||
|
||||
.LStoreMasked2xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmovupf YMMWORD PTR [rdx],ymm4"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmovupf YMMWORD PTR [rdx+rax],ymm6"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmovupf YMMWORD PTR [rdx+rax*2],ymm8"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmovupf YMMWORD PTR [rbx],ymm10"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmovupf YMMWORD PTR [rbx+rax],ymm12"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmovupf YMMWORD PTR [rbx+rax*2],ymm14"
|
||||
add rdx,32 # advance matrix C by YMMWORD
|
||||
.if \RowCount\() > 3
|
||||
add rbx,32 # advance matrix C plus 3 rows by YMMWORD
|
||||
.endif
|
||||
add r9,.LFgemmYmmElementCount # correct for over-subtract above
|
||||
|
||||
.LOutputMasked1xNBlock\@:
|
||||
neg r9
|
||||
lea rdi,C_UNDERSCORE(MlasMaskMoveTableAvx)[rip+8*4]
|
||||
vmovdqu ymm0,YMMWORD PTR [rdi+r9*.LFgemmElementSize]
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LMultiplyAlphaMasked1xNBlock\@
|
||||
EmitIfCountGE \RowCount\(), 1, "vmaskmovpf ymm4,ymm0,YMMWORD PTR [rdx]"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmaskmovpf ymm6,ymm0,YMMWORD PTR [rdx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmaskmovpf ymm8,ymm0,YMMWORD PTR [rdx+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmaskmovpf ymm10,ymm0,YMMWORD PTR [rbx]"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmaskmovpf ymm12,ymm0,YMMWORD PTR [rbx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmaskmovpf ymm14,ymm0,YMMWORD PTR [rbx+rax*2]"
|
||||
EmitIfCountGE \RowCount\(), 1, "vfmadd213pf ymm5,ymm2,ymm4"
|
||||
EmitIfCountGE \RowCount\(), 2, "vfmadd213pf ymm7,ymm2,ymm6"
|
||||
EmitIfCountGE \RowCount\(), 3, "vfmadd213pf ymm9,ymm2,ymm8"
|
||||
EmitIfCountGE \RowCount\(), 4, "vfmadd213pf ymm11,ymm2,ymm10"
|
||||
EmitIfCountGE \RowCount\(), 5, "vfmadd213pf ymm13,ymm2,ymm12"
|
||||
EmitIfCountGE \RowCount\(), 6, "vfmadd213pf ymm15,ymm2,ymm14"
|
||||
jmp .LStoreMasked1xNBlock\@
|
||||
|
||||
.LMultiplyAlphaMasked1xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmulpf ymm5,ymm5,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmulpf ymm7,ymm7,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmulpf ymm9,ymm9,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmulpf ymm11,ymm11,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmulpf ymm13,ymm13,ymm2"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmulpf ymm15,ymm15,ymm2"
|
||||
|
||||
.LStoreMasked1xNBlock\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "vmaskmovpf YMMWORD PTR [rdx],ymm0,ymm5"
|
||||
EmitIfCountGE \RowCount\(), 2, "vmaskmovpf YMMWORD PTR [rdx+rax],ymm0,ymm7"
|
||||
EmitIfCountGE \RowCount\(), 3, "vmaskmovpf YMMWORD PTR [rdx+rax*2],ymm0,ymm9"
|
||||
EmitIfCountGE \RowCount\(), 4, "vmaskmovpf YMMWORD PTR [rbx],ymm0,ymm11"
|
||||
EmitIfCountGE \RowCount\(), 5, "vmaskmovpf YMMWORD PTR [rbx+rax],ymm0,ymm13"
|
||||
EmitIfCountGE \RowCount\(), 6, "vmaskmovpf YMMWORD PTR [rbx+rax*2],ymm0,ymm15"
|
||||
.ifb \Fallthrough\()
|
||||
jmp .LExitKernelAndZeroUpper
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates the inner kernel to compute matrix multiplication.
|
||||
|
||||
Arguments:
|
||||
|
||||
FunctionName - Supplies the name for the generated function.
|
||||
|
||||
--*/
|
||||
|
||||
.macro FgemmKernelFma3Function FunctionName
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A (rdi) - Supplies the address of matrix A.
|
||||
|
||||
B (rsi) - Supplies the address of matrix B. The matrix data has been packed
|
||||
using MlasSgemmCopyPackB or MlasSgemmTransposePackB.
|
||||
|
||||
C (rdx) - Supplies the address of matrix C.
|
||||
|
||||
CountK (rcx) - Supplies the number of columns from matrix A and the number
|
||||
of rows from matrix B to iterate over.
|
||||
|
||||
CountM (r8) - Supplies the maximum number of rows that can be processed for
|
||||
matrix A and matrix C. The actual number of rows handled for this
|
||||
invocation depends on the kernel implementation.
|
||||
|
||||
CountN (r9) - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
Alpha (xmm0) - Supplies the scalar alpha multiplier (see GEMM definition).
|
||||
|
||||
ZeroMode - Supplies true if the output matrix must be zero initialized,
|
||||
else false if the output matrix is accumulated into.
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY \FunctionName\()
|
||||
|
||||
push rbp
|
||||
push rbx
|
||||
push r15
|
||||
mov r11,rdi
|
||||
mov r10,.LFgemmKernelFrame_lda[rsp]
|
||||
shl r10,.LFgemmElementShift # convert lda to bytes
|
||||
mov rax,.LFgemmKernelFrame_ldc[rsp]
|
||||
shl rax,.LFgemmElementShift # convert ldc to bytes
|
||||
movzx r15,BYTE PTR .LFgemmKernelFrame_ZeroMode[rsp]
|
||||
vmovsf .LFgemmKernelFrame_alpha[rsp],xmm0
|
||||
vzeroall
|
||||
|
||||
//
|
||||
// Process CountM rows of the matrices.
|
||||
//
|
||||
|
||||
cmp r8,5
|
||||
ja .LProcessCountM6
|
||||
je .LProcessCountM5
|
||||
cmp r8,3
|
||||
ja .LProcessCountM4
|
||||
je .LProcessCountM3
|
||||
cmp r8,1
|
||||
je .LProcessCountM1
|
||||
|
||||
.LProcessCountM2:
|
||||
ProcessCountM 2
|
||||
|
||||
.LProcessCountM4:
|
||||
ProcessCountM 4
|
||||
|
||||
.LProcessCountM6:
|
||||
mov r8d,6 # return 6 rows handled
|
||||
ProcessCountM 6, Fallthrough
|
||||
|
||||
//
|
||||
// Restore non-volatile registers and return.
|
||||
//
|
||||
|
||||
.LExitKernelAndZeroUpper:
|
||||
vzeroupper
|
||||
|
||||
.LExitKernel:
|
||||
mov eax,r8d
|
||||
pop r15
|
||||
pop rbx
|
||||
pop rbp
|
||||
ret
|
||||
|
||||
.LProcessCountM1:
|
||||
ProcessCountM 1
|
||||
|
||||
.LProcessCountM3:
|
||||
ProcessCountM 3
|
||||
|
||||
.LProcessCountM5:
|
||||
ProcessCountM 5
|
||||
|
||||
.endm
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
FgemmKernelSse2Common.h
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the floating point matrix/matrix
|
||||
multiply operation (SGEMM and DGEMM).
|
||||
|
||||
This implementation uses SSE2 instructions.
|
||||
|
||||
--*/
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This stores the block accumulators to the output matrix with an optional
|
||||
accumulation of the existing contents of the output matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorCount - Supplies the number of vector columns to process.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rax - Supplies the length in bytes of a row from matrix C.
|
||||
|
||||
rdx - Supplies the address of matrix C.
|
||||
|
||||
r15 - Stores the ZeroMode argument from the stack frame.
|
||||
|
||||
xmm8-xmm15 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro AccumulateAndStoreBlock RowCount, VectorCount
|
||||
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LSkipAccumulateOutput\@
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 1, "movupf xmm0,XMMWORD PTR [rdx]"
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 2, "movupf xmm1,XMMWORD PTR [rdx+16]"
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 3, "movupf xmm2,XMMWORD PTR [rdx+32]"
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 4, "movupf xmm3,XMMWORD PTR [rdx+48]"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 1, "movupf xmm4,XMMWORD PTR [rdx+rax]"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 2, "movupf xmm5,XMMWORD PTR [rdx+rax+16]"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 3, "movupf xmm6,XMMWORD PTR [rdx+rax+32]"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 4, "movupf xmm7,XMMWORD PTR [rdx+rax+48]"
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 1, "addpf xmm8,xmm0"
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 2, "addpf xmm9,xmm1"
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 3, "addpf xmm10,xmm2"
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 4, "addpf xmm11,xmm3"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 1, "addpf xmm12,xmm4"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 2, "addpf xmm13,xmm5"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 3, "addpf xmm14,xmm6"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 4, "addpf xmm15,xmm7"
|
||||
|
||||
.LSkipAccumulateOutput\@:
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 1, "movupf XMMWORD PTR [rdx],xmm8"
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 2, "movupf XMMWORD PTR [rdx+16],xmm9"
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 3, "movupf XMMWORD PTR [rdx+32],xmm10"
|
||||
EmitIfCount2GE \RowCount\(), 1, \VectorCount\(), 4, "movupf XMMWORD PTR [rdx+48],xmm11"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 1, "movupf XMMWORD PTR [rdx+rax],xmm12"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 2, "movupf XMMWORD PTR [rdx+rax+16],xmm13"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 3, "movupf XMMWORD PTR [rdx+rax+32],xmm14"
|
||||
EmitIfCount2GE \RowCount\(), 2, \VectorCount\(), 4, "movupf XMMWORD PTR [rdx+rax+48],xmm15"
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates the inner kernel to compute matrix multiplication.
|
||||
|
||||
Arguments:
|
||||
|
||||
FunctionName - Supplies the name for the generated function.
|
||||
|
||||
--*/
|
||||
|
||||
.macro FgemmKernelSse2Function FunctionName
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A (rdi) - Supplies the address of matrix A.
|
||||
|
||||
B (rsi) - Supplies the address of matrix B. The matrix data has been packed
|
||||
using MlasSgemmCopyPackB or MlasSgemmTransposePackB.
|
||||
|
||||
C (rdx) - Supplies the address of matrix C.
|
||||
|
||||
CountK (rcx) - Supplies the number of columns from matrix A and the number
|
||||
of rows from matrix B to iterate over.
|
||||
|
||||
CountM (r8) - Supplies the maximum number of rows that can be processed for
|
||||
matrix A and matrix C. The actual number of rows handled for this
|
||||
invocation depends on the kernel implementation.
|
||||
|
||||
CountN (r9) - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
Alpha (xmm0) - Supplies the scalar alpha multiplier (see GEMM definition).
|
||||
|
||||
ZeroMode - Supplies true if the output matrix must be zero initialized,
|
||||
else false if the output matrix is accumulated into.
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY \FunctionName\()
|
||||
|
||||
push rbp
|
||||
push rbx
|
||||
push r15
|
||||
mov r11,rdi
|
||||
mov r10,.LFgemmKernelFrame_lda[rsp]
|
||||
shl r10,.LFgemmElementShift # convert lda to bytes
|
||||
mov rax,.LFgemmKernelFrame_ldc[rsp]
|
||||
shl rax,.LFgemmElementShift # convert ldc to bytes
|
||||
movzx r15,BYTE PTR .LFgemmKernelFrame_ZeroMode[rsp]
|
||||
movsf .LFgemmKernelFrame_alpha[rsp],xmm0
|
||||
|
||||
//
|
||||
// Process CountM rows of the matrices.
|
||||
//
|
||||
|
||||
cmp r8,2
|
||||
jb .LProcessCountM1
|
||||
mov r8d,2 # return 2 rows handled
|
||||
ProcessCountM 2, Fallthrough
|
||||
|
||||
//
|
||||
// Restore non-volatile registers and return.
|
||||
//
|
||||
|
||||
.LExitKernel:
|
||||
mov eax,r8d
|
||||
pop r15
|
||||
pop rbx
|
||||
pop rbp
|
||||
ret
|
||||
|
||||
//
|
||||
// Process 1 row of the matrices.
|
||||
//
|
||||
|
||||
.LProcessCountM1:
|
||||
ProcessCountM 1
|
||||
|
||||
.endm
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelAvx.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM).
|
||||
|
||||
This implementation uses AVX instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
#include "SgemmKernelCommon.h"
|
||||
#include "FgemmKernelAvxCommon.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
.text
|
||||
|
||||
//
|
||||
// Generate the GEMM kernel.
|
||||
//
|
||||
|
||||
FgemmKernelAvxFunction MlasGemmFloatKernelAvx
|
||||
|
||||
.end
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelAvx512F.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM).
|
||||
|
||||
This implementation uses AVX512F instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
#include "SgemmKernelCommon.h"
|
||||
#include "FgemmKernelAvx512FCommon.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
.text
|
||||
|
||||
//
|
||||
// Generate the GEMM kernel.
|
||||
//
|
||||
|
||||
FgemmKernelAvx512FFunction MlasGemmFloatKernelAvx512F
|
||||
|
||||
.end
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelCommon.h
|
||||
|
||||
Abstract:
|
||||
|
||||
This module contains common kernel macros and structures for the single
|
||||
precision matrix/matrix multiply operation (SGEMM).
|
||||
|
||||
--*/
|
||||
|
||||
//
|
||||
// Define the single precision parameters.
|
||||
//
|
||||
|
||||
.equ .LFgemmElementShift, 2
|
||||
.equ .LFgemmElementSize, 1 << .LFgemmElementShift
|
||||
|
||||
#include "FgemmKernelCommon.h"
|
||||
|
||||
//
|
||||
// Define the typed instructions for single precision.
|
||||
//
|
||||
|
||||
FGEMM_TYPED_INSTRUCTION(addpf, addps)
|
||||
FGEMM_TYPED_INSTRUCTION(movsf, movss)
|
||||
FGEMM_TYPED_INSTRUCTION(movupf, movups)
|
||||
|
||||
FGEMM_TYPED_INSTRUCTION(vaddpf, vaddps)
|
||||
FGEMM_TYPED_INSTRUCTION(vbroadcastsf, vbroadcastss)
|
||||
FGEMM_TYPED_INSTRUCTION(vfmadd213pf, vfmadd213ps)
|
||||
FGEMM_TYPED_INSTRUCTION(vfmadd231pf, vfmadd231ps)
|
||||
FGEMM_TYPED_INSTRUCTION(vmaskmovpf, vmaskmovps)
|
||||
FGEMM_TYPED_INSTRUCTION(vmovapf, vmovaps)
|
||||
FGEMM_TYPED_INSTRUCTION(vmovsf, vmovss)
|
||||
FGEMM_TYPED_INSTRUCTION(vmovupf, vmovups)
|
||||
FGEMM_TYPED_INSTRUCTION(vmulpf, vmulps)
|
||||
FGEMM_TYPED_INSTRUCTION(vxorpf, vxorps)
|
||||
|
||||
.macro vfmadd231pf_bcst DestReg, SrcReg, Address
|
||||
|
||||
vfmadd231ps \DestReg\(), \SrcReg\(), \Address\(){1to16}
|
||||
|
||||
.endm
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelFma3.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM).
|
||||
|
||||
This implementation uses AVX fused multiply/add instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
#include "SgemmKernelCommon.h"
|
||||
#include "FgemmKernelFma3Common.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
.text
|
||||
|
||||
//
|
||||
// Generate the GEMM kernel.
|
||||
//
|
||||
|
||||
FgemmKernelFma3Function MlasGemmFloatKernelFma3
|
||||
|
||||
.end
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelM1Avx.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM). This handles the special case of M=1.
|
||||
|
||||
This implementation uses AVX instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
.text
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows. This handles the special case of M=1.
|
||||
|
||||
The elements in matrix B are not transposed.
|
||||
|
||||
Arguments:
|
||||
|
||||
A (rdi) - Supplies the address of matrix A.
|
||||
|
||||
B (rsi) - Supplies the address of matrix B.
|
||||
|
||||
C (rdx) - Supplies the address of matrix C.
|
||||
|
||||
CountK (rcx) - Supplies the number of columns from matrix A and the number
|
||||
of rows from matrix B to iterate over.
|
||||
|
||||
CountN (r8) - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
ldb (r9) - Supplies the first dimension of matrix B.
|
||||
|
||||
Beta (xmm0) - Supplies the scalar beta multiplier (see SGEMM definition).
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY MlasSgemmKernelM1Avx
|
||||
|
||||
push rbx
|
||||
shl r9,2 # convert ldb to bytes
|
||||
mov r10,rdx
|
||||
mov r11,rsi
|
||||
|
||||
//
|
||||
// Compute the initial results mask for zeroing or accumulate mode.
|
||||
//
|
||||
|
||||
vxorps xmm1,xmm1,xmm1
|
||||
vcmpeqss xmm0,xmm1,xmm0
|
||||
vshufps xmm0,xmm0,xmm0,0
|
||||
vinsertf128 ymm0,ymm0,xmm0,1
|
||||
|
||||
//
|
||||
// Compute the conditional load/store mask for an unaligned CountN.
|
||||
//
|
||||
|
||||
mov eax,r8d
|
||||
and eax,7
|
||||
vmovd xmm7,eax
|
||||
vshufps xmm7,xmm7,xmm7,0
|
||||
vpcmpgtd xmm6,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16]
|
||||
vpcmpgtd xmm7,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
|
||||
vinsertf128 ymm7,ymm7,xmm6,1
|
||||
|
||||
//
|
||||
// Process 4 rows of the matrices in a loop.
|
||||
//
|
||||
|
||||
sub rcx,4
|
||||
jb .LProcessRemainingCountK
|
||||
|
||||
.LProcessRowLoop4:
|
||||
vbroadcastss ymm2,DWORD PTR [rdi]
|
||||
mov rax,r8 # reload CountN
|
||||
vbroadcastss ymm3,DWORD PTR [rdi+4]
|
||||
mov rsi,r11 # reload matrix B
|
||||
vbroadcastss ymm4,DWORD PTR [rdi+8]
|
||||
mov rdx,r10 # reload matrix C
|
||||
vbroadcastss ymm5,DWORD PTR [rdi+12]
|
||||
add rdi,4*4 # advance matrix A by 4 columns
|
||||
lea r11,[rsi+r9*4] # advance matrix B by 4 rows
|
||||
sub rax,16
|
||||
jb .LProcessRemainingCountN4
|
||||
|
||||
.LProcessColumnLoop4:
|
||||
lea rbx,[rsi+r9*2] # compute matrix B plus 2 rows
|
||||
vmulps ymm1,ymm2,YMMWORD PTR [rsi]
|
||||
vmulps ymm6,ymm2,YMMWORD PTR [rsi+32]
|
||||
vmulps ymm8,ymm3,YMMWORD PTR [rsi+r9]
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vmulps ymm8,ymm3,YMMWORD PTR [rsi+r9+32]
|
||||
vaddps ymm6,ymm6,ymm8
|
||||
vmulps ymm8,ymm4,YMMWORD PTR [rbx]
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vmulps ymm8,ymm4,YMMWORD PTR [rbx+32]
|
||||
vaddps ymm6,ymm6,ymm8
|
||||
vmulps ymm8,ymm5,YMMWORD PTR [rbx+r9]
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vmulps ymm8,ymm5,YMMWORD PTR [rbx+r9+32]
|
||||
vaddps ymm6,ymm6,ymm8
|
||||
vandnps ymm8,ymm0,YMMWORD PTR [rdx]
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vandnps ymm8,ymm0,YMMWORD PTR [rdx+32]
|
||||
vaddps ymm6,ymm6,ymm8
|
||||
vmovups YMMWORD PTR [rdx],ymm1
|
||||
vmovups YMMWORD PTR [rdx+32],ymm6
|
||||
add rsi,16*4 # advance matrix B by 16 columns
|
||||
add rdx,16*4 # advance matrix C by 16 columns
|
||||
sub rax,16
|
||||
jae .LProcessColumnLoop4
|
||||
|
||||
.LProcessRemainingCountN4:
|
||||
test al,15 # test for unaligned columns
|
||||
jz .LProcessedRemainingCountN4
|
||||
test al,8 # CountN >= 8?
|
||||
jz .LProcessRemainingCountNSmall4
|
||||
lea rbx,[rsi+r9*2] # compute matrix B plus 2 rows
|
||||
vmulps ymm1,ymm2,YMMWORD PTR [rsi]
|
||||
vmulps ymm8,ymm3,YMMWORD PTR [rsi+r9]
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vmulps ymm8,ymm4,YMMWORD PTR [rbx]
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vmulps ymm8,ymm5,YMMWORD PTR [rbx+r9]
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vandnps ymm8,ymm0,YMMWORD PTR [rdx]
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vmovups YMMWORD PTR [rdx],ymm1
|
||||
add rsi,8*4 # advance matrix B by 8 columns
|
||||
add rdx,8*4 # advance matrix C by 8 columns
|
||||
test al,7
|
||||
jz .LProcessedRemainingCountN4
|
||||
|
||||
.LProcessRemainingCountNSmall4:
|
||||
lea rbx,[rsi+r9*2] # compute matrix B plus 2 rows
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi]
|
||||
vmulps ymm1,ymm2,ymm6
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi+r9]
|
||||
vmulps ymm8,ymm3,ymm6
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rbx]
|
||||
vmulps ymm8,ymm4,ymm6
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rbx+r9]
|
||||
vmulps ymm8,ymm5,ymm6
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx]
|
||||
vandnps ymm6,ymm0,ymm6
|
||||
vaddps ymm1,ymm1,ymm6
|
||||
vmaskmovps YMMWORD PTR [rdx],ymm7,ymm1
|
||||
|
||||
.LProcessedRemainingCountN4:
|
||||
vxorps xmm0,xmm0,xmm0 # switch to accumulate mode
|
||||
sub rcx,4
|
||||
jae .LProcessRowLoop4
|
||||
|
||||
.LProcessRemainingCountK:
|
||||
test cl,2
|
||||
jnz .LProcessRowLoop2
|
||||
test cl,1
|
||||
jnz .LProcessRowLoop1
|
||||
|
||||
.LExitKernel:
|
||||
vzeroupper
|
||||
pop rbx
|
||||
ret
|
||||
|
||||
//
|
||||
// Process 2 rows of the matrices.
|
||||
//
|
||||
|
||||
.LProcessRowLoop2:
|
||||
vbroadcastss ymm2,DWORD PTR [rdi]
|
||||
mov rax,r8 # reload CountN
|
||||
vbroadcastss ymm3,DWORD PTR [rdi+4]
|
||||
mov rsi,r11 # reload matrix B
|
||||
mov rdx,r10 # reload matrix C
|
||||
add rdi,2*4 # advance matrix A by 2 columns
|
||||
lea r11,[rsi+r9*2] # advance matrix B by 2 rows
|
||||
sub rax,8
|
||||
jb .LProcessRemainingCountN2
|
||||
|
||||
.LProcessColumnLoop2:
|
||||
vmulps ymm1,ymm2,YMMWORD PTR [rsi]
|
||||
vmulps ymm8,ymm3,YMMWORD PTR [rsi+r9]
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vandnps ymm6,ymm0,YMMWORD PTR [rdx]
|
||||
vaddps ymm1,ymm1,ymm6
|
||||
vmovups YMMWORD PTR [rdx],ymm1
|
||||
add rsi,8*4 # advance matrix B by 8 columns
|
||||
add rdx,8*4 # advance matrix C by 8 columns
|
||||
sub rax,8
|
||||
jae .LProcessColumnLoop2
|
||||
|
||||
.LProcessRemainingCountN2:
|
||||
test al,7 # test for unaligned columns
|
||||
jz .LProcessedRemainingCountN2
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi]
|
||||
vmulps ymm1,ymm2,ymm6
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi+r9]
|
||||
vmulps ymm8,ymm3,ymm6
|
||||
vaddps ymm1,ymm1,ymm8
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx]
|
||||
vandnps ymm6,ymm0,ymm6
|
||||
vaddps ymm1,ymm1,ymm6
|
||||
vmaskmovps YMMWORD PTR [rdx],ymm7,ymm1
|
||||
|
||||
.LProcessedRemainingCountN2:
|
||||
test cl,1
|
||||
jz .LExitKernel
|
||||
vxorps xmm0,xmm0,xmm0 # switch to accumulate mode
|
||||
|
||||
//
|
||||
// Process 1 row of the matrices.
|
||||
//
|
||||
|
||||
.LProcessRowLoop1:
|
||||
vbroadcastss ymm2,DWORD PTR [rdi]
|
||||
mov rax,r8 # reload CountN
|
||||
mov rsi,r11 # reload matrix B
|
||||
mov rdx,r10 # reload matrix C
|
||||
sub rax,8
|
||||
jb .LProcessRemainingCountN1
|
||||
|
||||
.LProcessColumnLoop1:
|
||||
vmulps ymm1,ymm2,YMMWORD PTR [rsi]
|
||||
vandnps ymm6,ymm0,YMMWORD PTR [rdx]
|
||||
vaddps ymm1,ymm1,ymm6
|
||||
vmovups YMMWORD PTR [rdx],ymm1
|
||||
add rsi,8*4 # advance matrix B by 8 columns
|
||||
add rdx,8*4 # advance matrix C by 8 columns
|
||||
sub rax,8
|
||||
jae .LProcessColumnLoop1
|
||||
|
||||
.LProcessRemainingCountN1:
|
||||
test al,7 # test for unaligned columns
|
||||
jz .LExitKernel
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi]
|
||||
vmulps ymm1,ymm2,ymm6
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rdx]
|
||||
vandnps ymm6,ymm0,ymm6
|
||||
vaddps ymm1,ymm1,ymm6
|
||||
vmaskmovps YMMWORD PTR [rdx],ymm7,ymm1
|
||||
jmp .LExitKernel
|
||||
|
||||
.end
|
||||
@@ -0,0 +1,275 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelM1TransposeBAvx.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM). This handles the special case of M=1.
|
||||
|
||||
This implementation uses AVX instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
.text
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows. This handles the special case of M=1.
|
||||
|
||||
The elements in matrix B are transposed.
|
||||
|
||||
Arguments:
|
||||
|
||||
A (rdi) - Supplies the address of matrix A.
|
||||
|
||||
B (rsi) - Supplies the address of matrix B. The elements are transposed.
|
||||
|
||||
C (rdx) - Supplies the address of matrix C.
|
||||
|
||||
CountK (rcx) - Supplies the number of columns from matrix A and the number
|
||||
of columns from matrix B to iterate over.
|
||||
|
||||
CountN (r8) - Supplies the number of rows from matrix B and the number of
|
||||
columns from matrix C to iterate over.
|
||||
|
||||
ldb (r9) - Supplies the first dimension of matrix B.
|
||||
|
||||
Beta (xmm0) - Supplies the scalar beta multiplier (see SGEMM definition).
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY MlasSgemmKernelM1TransposeBAvx
|
||||
|
||||
push rbx
|
||||
shl r9,2 # convert ldb to bytes
|
||||
mov r10,rdi
|
||||
mov r11,rsi
|
||||
|
||||
//
|
||||
// Compute the results mask for zeroing or accumulate mode.
|
||||
//
|
||||
|
||||
vxorps xmm1,xmm1,xmm1
|
||||
vcmpeqss xmm0,xmm1,xmm0
|
||||
vshufps xmm0,xmm0,xmm0,0
|
||||
|
||||
//
|
||||
// Compute the conditional load/store mask for an unaligned CountK.
|
||||
//
|
||||
|
||||
mov eax,ecx
|
||||
and eax,7
|
||||
vmovd xmm7,eax
|
||||
vshufps xmm7,xmm7,xmm7,0
|
||||
vpcmpgtd xmm6,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16]
|
||||
vpcmpgtd xmm7,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
|
||||
vinsertf128 ymm7,ymm7,xmm6,1
|
||||
|
||||
//
|
||||
// Process 4 rows of the matrices in a loop.
|
||||
//
|
||||
|
||||
sub r8,4
|
||||
jb .LProcessRemainingCountN
|
||||
|
||||
.LProcessRowLoop4:
|
||||
vxorps xmm2,xmm2,xmm2 # clear row accumulators
|
||||
vxorps xmm3,xmm3,xmm3
|
||||
vxorps xmm4,xmm4,xmm4
|
||||
vxorps xmm5,xmm5,xmm5
|
||||
mov rdi,r10 # reload matrix A
|
||||
mov rsi,r11 # reload matrix B
|
||||
mov rax,rcx # reload CountK
|
||||
lea r11,[rsi+r9*4] # advance matrix B by 4 rows
|
||||
sub rax,8
|
||||
jb .LProcessRemainingCountK4
|
||||
|
||||
.LProcessColumnLoop4:
|
||||
lea rbx,[rsi+r9*2] # compute matrix B plus 2 rows
|
||||
vmovups ymm1,YMMWORD PTR [rdi]
|
||||
vmulps ymm6,ymm1,YMMWORD PTR [rsi]
|
||||
vaddps ymm2,ymm2,ymm6
|
||||
vmulps ymm6,ymm1,YMMWORD PTR [rsi+r9]
|
||||
vaddps ymm3,ymm3,ymm6
|
||||
vmulps ymm6,ymm1,YMMWORD PTR [rbx]
|
||||
vaddps ymm4,ymm4,ymm6
|
||||
vmulps ymm6,ymm1,YMMWORD PTR [rbx+r9]
|
||||
vaddps ymm5,ymm5,ymm6
|
||||
add rdi,8*4 # advance matrix A by 8 columns
|
||||
add rsi,8*4 # advance matrix B by 8 columns
|
||||
sub rax,8
|
||||
jae .LProcessColumnLoop4
|
||||
|
||||
.LProcessRemainingCountK4:
|
||||
test al,7 # test for unaligned columns
|
||||
jz .LOutput4x1Block
|
||||
lea rbx,[rsi+r9*2] # compute matrix B plus 2 rows
|
||||
vmaskmovps ymm1,ymm7,YMMWORD PTR [rdi]
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi]
|
||||
vmulps ymm6,ymm1,ymm6
|
||||
vaddps ymm2,ymm2,ymm6
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi+r9]
|
||||
vmulps ymm6,ymm1,ymm6
|
||||
vaddps ymm3,ymm3,ymm6
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rbx]
|
||||
vmulps ymm6,ymm1,ymm6
|
||||
vaddps ymm4,ymm4,ymm6
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rbx+r9]
|
||||
vmulps ymm6,ymm1,ymm6
|
||||
vaddps ymm5,ymm5,ymm6
|
||||
|
||||
//
|
||||
// Reduce and output the row accumulators.
|
||||
//
|
||||
|
||||
.LOutput4x1Block:
|
||||
vunpcklps ymm6,ymm2,ymm3 # transpose row accumulators
|
||||
vunpckhps ymm1,ymm2,ymm3
|
||||
vunpcklps ymm2,ymm4,ymm5
|
||||
vunpckhps ymm3,ymm4,ymm5
|
||||
vunpcklpd ymm4,ymm6,ymm2
|
||||
vunpckhpd ymm5,ymm6,ymm2
|
||||
vaddps ymm4,ymm4,ymm5
|
||||
vunpcklpd ymm6,ymm1,ymm3
|
||||
vunpckhpd ymm2,ymm1,ymm3
|
||||
vaddps ymm4,ymm4,ymm6
|
||||
vaddps ymm4,ymm4,ymm2
|
||||
vextractf128 xmm5,ymm4,1
|
||||
vaddps xmm4,xmm4,xmm5
|
||||
vandnps xmm6,xmm0,XMMWORD PTR [rdx]
|
||||
vaddps xmm4,xmm4,xmm6
|
||||
vmovups XMMWORD PTR [rdx],xmm4
|
||||
add rdx,4*4 # advance matrix C by 4 columns
|
||||
sub r8,4
|
||||
jae .LProcessRowLoop4
|
||||
|
||||
.LProcessRemainingCountN:
|
||||
test r8d,2
|
||||
jnz .LProcessRowLoop2
|
||||
test r8d,1
|
||||
jnz .LProcessRowLoop1
|
||||
|
||||
.LExitKernel:
|
||||
vzeroupper
|
||||
pop rbx
|
||||
ret
|
||||
|
||||
//
|
||||
// Process 2 rows of the matrices.
|
||||
//
|
||||
|
||||
.LProcessRowLoop2:
|
||||
vxorps xmm2,xmm2,xmm2 # clear row accumulators
|
||||
vxorps xmm3,xmm3,xmm3
|
||||
mov rdi,r10 # reload matrix A
|
||||
mov rsi,r11 # reload matrix B
|
||||
mov rax,rcx # reload CountK
|
||||
lea r11,[rsi+r9*2] # advance matrix B by 2 rows
|
||||
sub rax,8
|
||||
jb .LProcessRemainingCountK2
|
||||
|
||||
.LProcessColumnLoop2:
|
||||
vmovups ymm1,YMMWORD PTR [rdi]
|
||||
vmulps ymm6,ymm1,YMMWORD PTR [rsi]
|
||||
vaddps ymm2,ymm2,ymm6
|
||||
vmulps ymm6,ymm1,YMMWORD PTR [rsi+r9]
|
||||
vaddps ymm3,ymm3,ymm6
|
||||
add rdi,8*4 # advance matrix A by 8 columns
|
||||
add rsi,8*4 # advance matrix B by 8 columns
|
||||
sub rax,8
|
||||
jae .LProcessColumnLoop2
|
||||
|
||||
.LProcessRemainingCountK2:
|
||||
test al,7 # test for unaligned columns
|
||||
jz .LOutput2x1Block
|
||||
vmaskmovps ymm1,ymm7,YMMWORD PTR [rdi]
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi]
|
||||
vmulps ymm6,ymm1,ymm6
|
||||
vaddps ymm2,ymm2,ymm6
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi+r9]
|
||||
vmulps ymm6,ymm1,ymm6
|
||||
vaddps ymm3,ymm3,ymm6
|
||||
|
||||
//
|
||||
// Reduce and output the row accumulators.
|
||||
//
|
||||
|
||||
.LOutput2x1Block:
|
||||
vunpcklps ymm4,ymm2,ymm3 # reduce row accumulators
|
||||
vunpckhps ymm2,ymm2,ymm3
|
||||
vaddps ymm2,ymm2,ymm4
|
||||
vextractf128 xmm4,ymm2,1
|
||||
vaddps xmm2,xmm2,xmm4
|
||||
vmovhlps xmm4,xmm2,xmm2
|
||||
vaddps xmm2,xmm2,xmm4
|
||||
vmovsd xmm3,QWORD PTR [rdx]
|
||||
vandnps xmm3,xmm0,xmm3
|
||||
vaddps xmm2,xmm2,xmm3
|
||||
vmovsd QWORD PTR [rdx],xmm2
|
||||
add rdx,2*4 # advance matrix C by 2 columns
|
||||
test r8d,1
|
||||
jz .LExitKernel
|
||||
|
||||
//
|
||||
// Process 1 row of the matrices.
|
||||
//
|
||||
|
||||
.LProcessRowLoop1:
|
||||
vxorps xmm2,xmm2,xmm2 # clear row accumulators
|
||||
mov rdi,r10 # reload matrix A
|
||||
mov rsi,r11 # reload matrix B
|
||||
mov rax,rcx # reload CountK
|
||||
sub rax,8
|
||||
jb .LProcessRemainingCountK1
|
||||
|
||||
.LProcessColumnLoop1:
|
||||
vmovups ymm1,YMMWORD PTR [rdi]
|
||||
vmulps ymm6,ymm1,YMMWORD PTR [rsi]
|
||||
vaddps ymm2,ymm2,ymm6
|
||||
add rdi,8*4 # advance matrix A by 8 columns
|
||||
add rsi,8*4 # advance matrix B by 8 columns
|
||||
sub rax,8
|
||||
jae .LProcessColumnLoop1
|
||||
|
||||
.LProcessRemainingCountK1:
|
||||
test al,7 # test for unaligned columns
|
||||
jz .LOutput1x1Block
|
||||
vmaskmovps ymm1,ymm7,YMMWORD PTR [rdi]
|
||||
vmaskmovps ymm6,ymm7,YMMWORD PTR [rsi]
|
||||
vmulps ymm6,ymm1,ymm6
|
||||
vaddps ymm2,ymm2,ymm6
|
||||
|
||||
//
|
||||
// Reduce and output the row accumulators.
|
||||
//
|
||||
|
||||
.LOutput1x1Block:
|
||||
vhaddps ymm2,ymm2,ymm2 # reduce row accumulators
|
||||
vhaddps ymm2,ymm2,ymm2
|
||||
vextractf128 xmm4,ymm2,1
|
||||
vaddss xmm2,xmm2,xmm4
|
||||
vmovss xmm3,DWORD PTR [rdx]
|
||||
vandnps xmm3,xmm0,xmm3
|
||||
vaddss xmm2,xmm2,xmm3
|
||||
vmovss DWORD PTR [rdx],xmm2
|
||||
jmp .LExitKernel
|
||||
|
||||
.end
|
||||
+273
@@ -0,0 +1,273 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelSse2.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM).
|
||||
|
||||
This implementation uses SSE2 instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
#include "SgemmKernelCommon.h"
|
||||
#include "FgemmKernelSse2Common.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
.text
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for a 16xN block of the output matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
Shuffle - Supplies the shuffle mask to extract the element from matrix A.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rsi - Supplies the address into the matrix B data.
|
||||
|
||||
xmm0-xmm1 - Supplies up to four elements loaded from matrix A and matrix A
|
||||
plus one row.
|
||||
|
||||
xmm8-xmm15 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockSseBy16 RowCount, VectorOffset, Shuffle
|
||||
|
||||
movaps xmm4,XMMWORD PTR [rsi+\VectorOffset\()]
|
||||
movaps xmm5,XMMWORD PTR [rsi+\VectorOffset\()+16]
|
||||
pshufd xmm2,xmm0,\Shuffle\()
|
||||
.if \RowCount\() == 2
|
||||
pshufd xmm3,xmm1,\Shuffle\()
|
||||
movaps xmm6,xmm4
|
||||
movaps xmm7,xmm5
|
||||
.endif
|
||||
mulps xmm4,xmm2
|
||||
mulps xmm5,xmm2
|
||||
addps xmm8,xmm4
|
||||
addps xmm9,xmm5
|
||||
.if \RowCount\() == 2
|
||||
mulps xmm6,xmm3
|
||||
mulps xmm7,xmm3
|
||||
addps xmm12,xmm6
|
||||
addps xmm13,xmm7
|
||||
.endif
|
||||
movaps xmm4,XMMWORD PTR [rsi+\VectorOffset\()+32]
|
||||
movaps xmm5,XMMWORD PTR [rsi+\VectorOffset\()+48]
|
||||
.if \RowCount\() == 2
|
||||
movaps xmm6,xmm4
|
||||
movaps xmm7,xmm5
|
||||
.endif
|
||||
mulps xmm4,xmm2
|
||||
mulps xmm5,xmm2
|
||||
addps xmm10,xmm4
|
||||
addps xmm11,xmm5
|
||||
.if \RowCount\() == 2
|
||||
mulps xmm6,xmm3
|
||||
mulps xmm7,xmm3
|
||||
addps xmm14,xmm6
|
||||
addps xmm15,xmm7
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to compute matrix multiplication for a fixed set
|
||||
of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
Fallthrough - Supplies a non-blank value if the macro may fall through to
|
||||
the ExitKernel label.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address of matrix A.
|
||||
|
||||
rsi - Supplies the address of matrix B.
|
||||
|
||||
r11 - Supplies the address of matrix A.
|
||||
|
||||
r9 - Supplies the number of columns from matrix B and matrix C to iterate
|
||||
over.
|
||||
|
||||
rdx - Supplies the address of matrix C.
|
||||
|
||||
rcx - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
r10 - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
rax - Supplies the length in bytes of a row from matrix C.
|
||||
|
||||
r15 - Stores the ZeroMode argument from the stack frame.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ProcessCountM RowCount, Fallthrough
|
||||
|
||||
.LProcessNextColumnLoop16xN\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "xorps xmm8,xmm8"
|
||||
EmitIfCountGE \RowCount\(), 1, "xorps xmm9,xmm9"
|
||||
EmitIfCountGE \RowCount\(), 1, "xorps xmm10,xmm10"
|
||||
EmitIfCountGE \RowCount\(), 1, "xorps xmm11,xmm11"
|
||||
EmitIfCountGE \RowCount\(), 2, "xorps xmm12,xmm12"
|
||||
EmitIfCountGE \RowCount\(), 2, "xorps xmm13,xmm13"
|
||||
EmitIfCountGE \RowCount\(), 2, "xorps xmm14,xmm14"
|
||||
EmitIfCountGE \RowCount\(), 2, "xorps xmm15,xmm15"
|
||||
mov rbp,rcx # reload CountK
|
||||
sub rbp,4
|
||||
jb .LProcessRemaining16xNBlocks\@
|
||||
|
||||
.LCompute16xNBlockBy4Loop\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "movups xmm0,XMMWORD PTR [rdi]"
|
||||
EmitIfCountGE \RowCount\(), 2, "movups xmm1,XMMWORD PTR [rdi+r10]"
|
||||
ComputeBlockSseBy16 2, 0, 0x00
|
||||
ComputeBlockSseBy16 2, 16*4, 0x55
|
||||
sub rsi,-32*4 # advance matrix B by 32 columns
|
||||
ComputeBlockSseBy16 2, 0, 0xAA
|
||||
ComputeBlockSseBy16 2, 16*4, 0xFF
|
||||
sub rsi,-32*4 # advance matrix B by 32 columns
|
||||
add rdi,4*4 # advance matrix A by 4 columns
|
||||
sub rbp,4
|
||||
jae .LCompute16xNBlockBy4Loop\@
|
||||
|
||||
.LProcessRemaining16xNBlocks\@:
|
||||
add rbp,4 # correct for over-subtract above
|
||||
jz .LOutput16xNBlock\@
|
||||
|
||||
.LCompute16xNBlockBy1Loop\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "movss xmm0,[rdi]"
|
||||
EmitIfCountGE \RowCount\(), 2, "movss xmm1,[rdi+r10]"
|
||||
ComputeBlockSseBy16 2, 0, 0x00
|
||||
add rsi,16*4 # advance matrix B by 16 columns
|
||||
add rdi,4 # advance matrix A by 1 column
|
||||
dec rbp
|
||||
jne .LCompute16xNBlockBy1Loop\@
|
||||
|
||||
.LOutput16xNBlock\@:
|
||||
movss xmm2,.LFgemmKernelFrame_alpha[rsp]
|
||||
shufps xmm2,xmm2,0
|
||||
EmitIfCountGE \RowCount\(), 1, "mulps xmm8,xmm2"
|
||||
# multiply by alpha
|
||||
EmitIfCountGE \RowCount\(), 1, "mulps xmm9,xmm2"
|
||||
EmitIfCountGE \RowCount\(), 1, "mulps xmm10,xmm2"
|
||||
EmitIfCountGE \RowCount\(), 1, "mulps xmm11,xmm2"
|
||||
EmitIfCountGE \RowCount\(), 2, "mulps xmm12,xmm2"
|
||||
EmitIfCountGE \RowCount\(), 2, "mulps xmm13,xmm2"
|
||||
EmitIfCountGE \RowCount\(), 2, "mulps xmm14,xmm2"
|
||||
EmitIfCountGE \RowCount\(), 2, "mulps xmm15,xmm2"
|
||||
sub r9,16
|
||||
jb .LOutputPartial16xNBlock\@
|
||||
AccumulateAndStoreBlock \RowCount\(), 4
|
||||
add rdx,16*4 # advance matrix C by 16 columns
|
||||
mov rdi,r11 # reload matrix A
|
||||
test r9,r9
|
||||
jnz .LProcessNextColumnLoop16xN\@
|
||||
jmp .LExitKernel
|
||||
|
||||
//
|
||||
// Output a partial 16xN block to the matrix.
|
||||
//
|
||||
|
||||
.LOutputPartial16xNBlock\@:
|
||||
add r9,16 # correct for over-subtract above
|
||||
cmp r9,4
|
||||
jb .LOutputPartialLessThan4xNBlock\@
|
||||
cmp r9,8
|
||||
jb .LOutputPartialLessThan8xNBlock\@
|
||||
cmp r9,12
|
||||
jb .LOutputPartialLessThan12xNBlock\@
|
||||
AccumulateAndStoreBlock \RowCount\(), 3
|
||||
and r9d,3 # check if remaining count is small
|
||||
jz .LExitKernel
|
||||
EmitIfCountGE \RowCount\(), 1, "movaps xmm8,xmm11"
|
||||
# shift remaining elements down
|
||||
EmitIfCountGE \RowCount\(), 2, "movaps xmm12,xmm15"
|
||||
add rdx,12*4 # advance matrix C by 12 columns
|
||||
jmp .LOutputPartialLessThan4xNBlock\@
|
||||
|
||||
.LOutputPartialLessThan12xNBlock\@:
|
||||
AccumulateAndStoreBlock \RowCount\(), 2
|
||||
and r9d,3 # check if remaining count is small
|
||||
jz .LExitKernel
|
||||
EmitIfCountGE \RowCount\(), 1, "movaps xmm8,xmm10"
|
||||
# shift remaining elements down
|
||||
EmitIfCountGE \RowCount\(), 2, "movaps xmm12,xmm14"
|
||||
add rdx,8*4 # advance matrix C by 8 columns
|
||||
jmp .LOutputPartialLessThan4xNBlock\@
|
||||
|
||||
.LOutputPartialLessThan8xNBlock\@:
|
||||
AccumulateAndStoreBlock \RowCount\(), 1
|
||||
and r9d,3 # check if remaining count is small
|
||||
jz .LExitKernel
|
||||
EmitIfCountGE \RowCount\(), 1, "movaps xmm8,xmm9"
|
||||
# shift remaining elements down
|
||||
EmitIfCountGE \RowCount\(), 2, "movaps xmm12,xmm13"
|
||||
add rdx,4*4 # advance matrix C by 4 columns
|
||||
|
||||
.LOutputPartialLessThan4xNBlock\@:
|
||||
test r9d,2
|
||||
jz .LOutputPartial1xNBlock\@
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LSkipAccumulateOutput2xN\@
|
||||
EmitIfCountGE \RowCount\(), 1, "movsd xmm0,QWORD PTR [rdx]"
|
||||
EmitIfCountGE \RowCount\(), 2, "movsd xmm1,QWORD PTR [rdx+rax]"
|
||||
EmitIfCountGE \RowCount\(), 1, "addps xmm8,xmm0"
|
||||
EmitIfCountGE \RowCount\(), 2, "addps xmm12,xmm1"
|
||||
|
||||
.LSkipAccumulateOutput2xN\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "movsd QWORD PTR [rdx],xmm8"
|
||||
EmitIfCountGE \RowCount\(), 2, "movsd QWORD PTR [rdx+rax],xmm12"
|
||||
test r9d,1 # check if remaining count is odd
|
||||
jz .LExitKernel
|
||||
EmitIfCountGE \RowCount\(), 1, "movhlps xmm8,xmm8"
|
||||
# shift third element down
|
||||
EmitIfCountGE \RowCount\(), 2, "movhlps xmm12,xmm12"
|
||||
add rdx,2*4 # advance matrix C by 2 columns
|
||||
|
||||
.LOutputPartial1xNBlock\@:
|
||||
test r15b,r15b # ZeroMode?
|
||||
jnz .LSkipAccumulateOutput1xN\@
|
||||
EmitIfCountGE \RowCount\(), 1, "addss xmm8,[rdx]"
|
||||
EmitIfCountGE \RowCount\(), 2, "addss xmm12,[rdx+rax]"
|
||||
|
||||
.LSkipAccumulateOutput1xN\@:
|
||||
EmitIfCountGE \RowCount\(), 1, "movss [rdx],xmm8"
|
||||
EmitIfCountGE \RowCount\(), 2, "movss [rdx+rax],xmm12"
|
||||
.ifb \Fallthrough\()
|
||||
jmp .LExitKernel
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
//
|
||||
// Generate the GEMM kernel.
|
||||
//
|
||||
|
||||
FgemmKernelSse2Function MlasGemmFloatKernelSse
|
||||
|
||||
.end
|
||||
@@ -0,0 +1,120 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmTransposePackB16x4Avx.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements routines for packing buffers for the single precision
|
||||
matrix/matrix multiply operation (SGEMM).
|
||||
|
||||
This implementation uses AVX instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
.text
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
4 columns of 8 rows from the source matrix are transposed to 8 columns of 4
|
||||
rows in the destination packed buffer.
|
||||
|
||||
Arguments:
|
||||
|
||||
StoreOffset - Supplies the relative byte offset into the destination packed
|
||||
buffer.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
rdi - Supplies the address of the destination packed buffer.
|
||||
|
||||
rsi - Supplies the address of the source matrix.
|
||||
|
||||
rdx - Supplies the number of elements per row of the source matrix.
|
||||
|
||||
--*/
|
||||
|
||||
.macro TransposePackB8x4BlockAvx StoreOffset
|
||||
|
||||
//
|
||||
// Load 4 columns from 8 rows of the source matrix into the lower and upper
|
||||
// halves of 4 YMM registers.
|
||||
//
|
||||
|
||||
lea rax,[rsi+rdx*2]
|
||||
vmovups xmm0,XMMWORD PTR [rsi]
|
||||
vmovups xmm1,XMMWORD PTR [rsi+rdx]
|
||||
lea rsi,[rax+rdx*2]
|
||||
vmovups xmm2,XMMWORD PTR [rax]
|
||||
vmovups xmm3,XMMWORD PTR [rax+rdx]
|
||||
lea rax,[rsi+rdx*2]
|
||||
vinsertf128 ymm0,ymm0,XMMWORD PTR [rsi],1
|
||||
vinsertf128 ymm1,ymm1,XMMWORD PTR [rsi+rdx],1
|
||||
vinsertf128 ymm2,ymm2,XMMWORD PTR [rax],1
|
||||
vinsertf128 ymm3,ymm3,XMMWORD PTR [rax+rdx],1
|
||||
|
||||
//
|
||||
// Transpose the lower and upper halves of the 4 YMM registers as two 4x4
|
||||
// matrices and store the output to the destination packed buffer.
|
||||
//
|
||||
|
||||
vunpcklps ymm4,ymm0,ymm1
|
||||
vunpckhps ymm5,ymm0,ymm1
|
||||
vunpcklps ymm0,ymm2,ymm3
|
||||
vunpckhps ymm1,ymm2,ymm3
|
||||
vunpcklpd ymm2,ymm4,ymm0
|
||||
vunpckhpd ymm3,ymm4,ymm0
|
||||
vmovaps YMMWORD PTR [rdi+16*4*0+\StoreOffset\()],ymm2
|
||||
vmovaps YMMWORD PTR [rdi+16*4*1+\StoreOffset\()],ymm3
|
||||
vunpcklpd ymm0,ymm5,ymm1
|
||||
vunpckhpd ymm4,ymm5,ymm1
|
||||
vmovaps YMMWORD PTR [rdi+16*4*2+\StoreOffset\()],ymm0
|
||||
vmovaps YMMWORD PTR [rdi+16*4*3+\StoreOffset\()],ymm4
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine transposes elements from the source matrix to the destination
|
||||
packed buffer.
|
||||
|
||||
4 columns of 16 rows from the source matrix are transposed to 16 columns of 4
|
||||
rows in the destination packed buffer.
|
||||
|
||||
Arguments:
|
||||
|
||||
D (rdi) - Supplies the address of the destination packed buffer.
|
||||
|
||||
B (rsi) - Supplies the address of the source matrix.
|
||||
|
||||
ldb (rdx) - Supplies the number of elements per row of the source matrix.
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY MlasSgemmTransposePackB16x4Avx
|
||||
|
||||
shl rdx,2 # convert ldb to bytes
|
||||
TransposePackB8x4BlockAvx 0*4
|
||||
lea rsi,[rax+rdx*2]
|
||||
TransposePackB8x4BlockAvx 8*4
|
||||
vzeroupper
|
||||
ret
|
||||
|
||||
.end
|
||||
@@ -0,0 +1,83 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmTransposePackB16x4Sse2.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements routines for packing buffers for the single precision
|
||||
matrix/matrix multiply operation (SGEMM).
|
||||
|
||||
This implementation uses SSE2 instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
.text
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine transposes elements from the source matrix to the destination
|
||||
packed buffer.
|
||||
|
||||
4 columns of 16 rows from the source matrix are transposed to 16 columns of 4
|
||||
rows in the destination packed buffer.
|
||||
|
||||
Arguments:
|
||||
|
||||
D (rdi) - Supplies the address of the destination packed buffer.
|
||||
|
||||
B (rsi) - Supplies the address of the source matrix.
|
||||
|
||||
ldb (rdx) - Supplies the number of elements per row of the source matrix.
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY MlasSgemmTransposePackB16x4Sse
|
||||
|
||||
shl rdx,2 # convert ldb to bytes
|
||||
mov ecx,4 # transpose four 4x4 blocks
|
||||
|
||||
.LTransposeBlockLoop:
|
||||
lea rax,[rsi+rdx*2]
|
||||
movups xmm0,XMMWORD PTR [rsi]
|
||||
movups xmm1,XMMWORD PTR [rsi+rdx]
|
||||
movups xmm2,XMMWORD PTR [rax]
|
||||
movups xmm3,XMMWORD PTR [rax+rdx]
|
||||
movaps xmm4,xmm0
|
||||
unpcklps xmm4,xmm1
|
||||
unpckhps xmm0,xmm1
|
||||
movaps xmm5,xmm2
|
||||
unpcklps xmm5,xmm3
|
||||
unpckhps xmm2,xmm3
|
||||
movaps xmm1,xmm4
|
||||
unpcklpd xmm1,xmm5
|
||||
unpckhpd xmm4,xmm5
|
||||
movaps xmm3,xmm0
|
||||
unpcklpd xmm3,xmm2
|
||||
unpckhpd xmm0,xmm2
|
||||
movaps XMMWORD PTR [rdi+16*4*0],xmm1
|
||||
movaps XMMWORD PTR [rdi+16*4*1],xmm4
|
||||
movaps XMMWORD PTR [rdi+16*4*2],xmm3
|
||||
movaps XMMWORD PTR [rdi+16*4*3],xmm0
|
||||
add rdi,4*4
|
||||
lea rsi,[rax+rdx*2]
|
||||
dec ecx
|
||||
jnz .LTransposeBlockLoop
|
||||
ret
|
||||
|
||||
.end
|
||||
Vendored
+172
@@ -0,0 +1,172 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
asmmacro.h
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements common macros for the assembly modules.
|
||||
|
||||
--*/
|
||||
|
||||
#if defined(__APPLE__)
|
||||
#define C_UNDERSCORE(symbol) _##symbol
|
||||
#else
|
||||
#define C_UNDERSCORE(symbol) symbol
|
||||
#endif
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro emits the assembler directives to annotate a new function.
|
||||
|
||||
Arguments:
|
||||
|
||||
FunctionName - Supplies the name of the function.
|
||||
|
||||
--*/
|
||||
|
||||
.macro FUNCTION_ENTRY FunctionName
|
||||
|
||||
.p2align 4
|
||||
#if defined(__APPLE__)
|
||||
.globl _\FunctionName\()
|
||||
_\FunctionName\():
|
||||
#else
|
||||
.globl \FunctionName\()
|
||||
.type \FunctionName\(),@function
|
||||
\FunctionName\():
|
||||
#endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates an optimization for "add reg,128" which can instead
|
||||
be encoded as "sub reg,-128" to reduce code size by using a signed 8-bit
|
||||
value.
|
||||
|
||||
Arguments:
|
||||
|
||||
Register - Supplies the register to be added to.
|
||||
|
||||
Immediate - Supplies the immediate to add to the register.
|
||||
|
||||
--*/
|
||||
|
||||
.macro add_immed Register, Immediate
|
||||
|
||||
.if (\Immediate\() != 128)
|
||||
add \Register\(),\Immediate\()
|
||||
.else
|
||||
sub \Register\(),-\Immediate\() # smaller encoding
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro conditionally emits the statement if Count is greater than or
|
||||
equal to Value.
|
||||
|
||||
Arguments:
|
||||
|
||||
Count - Supplies the variable used in the comparison.
|
||||
|
||||
Value - Supplies the static used in the comparison.
|
||||
|
||||
Statement - Supplies the statement to conditionally emit.
|
||||
|
||||
--*/
|
||||
|
||||
.macro EmitIfCountGE Count1, Value1, Statement
|
||||
|
||||
.if (\Count1\() >= \Value1\())
|
||||
\Statement\()
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
|
||||
/*++
|
||||
Macro Description:
|
||||
This macro conditionally emits the statement if Count1 is equal to Value1
|
||||
and Count2 is equal to Value2.
|
||||
Arguments:
|
||||
Count1 - Supplies the variable used in the comparison.
|
||||
Value1 - Supplies the static used in the comparison.
|
||||
Count2 - Supplies the variable used in the comparison.
|
||||
Value2 - Supplies the static used in the comparison.
|
||||
Statement - Supplies the statement to conditionally emit.
|
||||
--*/
|
||||
|
||||
.macro EmitIfCount2EQ Count1, Value1, Count2, Value2, Statement
|
||||
|
||||
.if (\Count1\() == \Value1\()) && (\Count2\() == \Value2\())
|
||||
\Statement\()
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro conditionally emits the statement if Count1 is greater than or
|
||||
equal to Value1 and Count2 is greater than or equal to Value2.
|
||||
|
||||
Arguments:
|
||||
|
||||
Count1 - Supplies the variable used in the comparison.
|
||||
|
||||
Value1 - Supplies the static used in the comparison.
|
||||
|
||||
Count2 - Supplies the variable used in the comparison.
|
||||
|
||||
Value2 - Supplies the static used in the comparison.
|
||||
|
||||
Statement - Supplies the statement to conditionally emit.
|
||||
|
||||
--*/
|
||||
|
||||
.macro EmitIfCount2GE Count1, Value1, Count2, Value2, Statement
|
||||
|
||||
.if (\Count1\() >= \Value1\()) && (\Count2\() >= \Value2\())
|
||||
\Statement\()
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro emits the statement for each register listed in the register
|
||||
list. The statement can use RegItem to access the current register.
|
||||
|
||||
Arguments:
|
||||
|
||||
RegList - Supplies the list of registers.
|
||||
|
||||
Statement - Supplies the statement to emit.
|
||||
|
||||
--*/
|
||||
|
||||
.macro EmitForEachRegister RegList, Statement
|
||||
|
||||
.irp RegItem, \RegList\()
|
||||
\Statement\()
|
||||
.endr
|
||||
|
||||
.endm
|
||||
Reference in New Issue
Block a user