vendor: OpenCV 5.0.0 snapshot at 40738fb16ceddb5fb3fea747585f7ce6abb0605b
This commit is contained in:
Gitea Mirror Bot
+435
@@ -0,0 +1,435 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelAvx.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM).
|
||||
|
||||
This implementation uses AVX instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
//
|
||||
// Stack frame layout for the SGEMM kernel.
|
||||
//
|
||||
|
||||
.equ .LSgemmKernelFrame_SavedEdi, 0
|
||||
.equ .LSgemmKernelFrame_SavedEsi, 4
|
||||
.equ .LSgemmKernelFrame_SavedEbx, 8
|
||||
.equ .LSgemmKernelFrame_SavedEbp, 12
|
||||
.equ .LSgemmKernelFrame_ReturnAddress, 16
|
||||
.equ .LSgemmKernelFrame_MatrixA, 20
|
||||
.equ .LSgemmKernelFrame_MatrixB, 24
|
||||
.equ .LSgemmKernelFrame_MatrixC, 28
|
||||
.equ .LSgemmKernelFrame_CountK, 32
|
||||
.equ .LSgemmKernelFrame_CountM, 36
|
||||
.equ .LSgemmKernelFrame_CountN, 40
|
||||
.equ .LSgemmKernelFrame_lda, 44
|
||||
.equ .LSgemmKernelFrame_ldc, 48
|
||||
.equ .LSgemmKernelFrame_alpha, 52
|
||||
.equ .LSgemmKernelFrame_ZeroMode, 56
|
||||
|
||||
.text
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for a 16xN block of the output matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
BroadcastOffset - Supplies the byte offset from matrix A to fetch elements.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
ebx - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ecx - Supplies the address into the matrix A data.
|
||||
|
||||
edx - Supplies the address into the matrix B data.
|
||||
|
||||
ymm4-ymm7 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvxBy16 RowCount, VectorOffset, BroadcastOffset
|
||||
|
||||
.if \RowCount\() == 1
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()]
|
||||
vmulps ymm1,ymm3,YMMWORD PTR [edx+\VectorOffset\()]
|
||||
vaddps ymm4,ymm1,ymm4
|
||||
vmulps ymm3,ymm3,YMMWORD PTR [edx+\VectorOffset\()+32]
|
||||
vaddps ymm5,ymm3,ymm5
|
||||
.else
|
||||
vmovaps ymm0,YMMWORD PTR [edx+\VectorOffset\()]
|
||||
vmovaps ymm1,YMMWORD PTR [edx+\VectorOffset\()+32]
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()]
|
||||
vmulps ymm2,ymm3,ymm0
|
||||
vaddps ymm4,ymm2,ymm4
|
||||
vmulps ymm2,ymm3,ymm1
|
||||
vaddps ymm5,ymm2,ymm5
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+ebx+\BroadcastOffset\()]
|
||||
vmulps ymm2,ymm3,ymm0
|
||||
vaddps ymm6,ymm2,ymm6
|
||||
vmulps ymm2,ymm3,ymm1
|
||||
vaddps ymm7,ymm2,ymm7
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for a 8xN block of the output matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
BroadcastOffset - Supplies the byte offset from matrix A to fetch elements.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
ebx - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ecx - Supplies the address into the matrix A data.
|
||||
|
||||
edx - Supplies the address into the matrix B data.
|
||||
|
||||
ymm4-ymm7 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvxBy8 RowCount, VectorOffset, BroadcastOffset
|
||||
|
||||
.if \RowCount\() == 1
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()]
|
||||
vmulps ymm3,ymm3,YMMWORD PTR [edx+\VectorOffset\()]
|
||||
vaddps ymm5,ymm3,ymm5
|
||||
.else
|
||||
vmovaps ymm0,YMMWORD PTR [edx+\VectorOffset\()]
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()]
|
||||
vmulps ymm3,ymm3,ymm0
|
||||
vaddps ymm5,ymm3,ymm5
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+ebx+\BroadcastOffset\()]
|
||||
vmulps ymm3,ymm3,ymm0
|
||||
vaddps ymm7,ymm3,ymm7
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to execute the block compute macro multiple
|
||||
times and advancing the matrix A and matrix B data pointers.
|
||||
|
||||
Arguments:
|
||||
|
||||
ComputeBlock - Supplies the macro to compute a single block.
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
ebx - Supplies the number of bytes to the next row of matrix A.
|
||||
|
||||
ecx - Supplies the address into the matrix A data.
|
||||
|
||||
edx - Supplies the address into the matrix B data.
|
||||
|
||||
edi - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
ymm4-ymm7 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvxLoop ComputeBlock, RowCount
|
||||
|
||||
sub edi,4
|
||||
jb .LProcessRemainingBlocks\@
|
||||
|
||||
.LComputeBlockBy4Loop\@:
|
||||
\ComputeBlock\() \RowCount\(), 0, 0
|
||||
\ComputeBlock\() \RowCount\(), 16*4, 4
|
||||
sub edx,-32*4 # advance matrix B by 32 columns
|
||||
\ComputeBlock\() \RowCount\(), 0, 8
|
||||
\ComputeBlock\() \RowCount\(), 16*4, 12
|
||||
sub edx,-32*4 # advance matrix B by 32 columns
|
||||
add ecx,4*4 # advance matrix A by 4 columns
|
||||
sub edi,4
|
||||
jae .LComputeBlockBy4Loop\@
|
||||
|
||||
.LProcessRemainingBlocks\@:
|
||||
add edi,4 # correct for over-subtract above
|
||||
jz .LOutputBlock\@
|
||||
|
||||
.LComputeBlockBy1Loop\@:
|
||||
\ComputeBlock\() \RowCount\(), 0, 0
|
||||
add edx,16*4 # advance matrix B by 16 columns
|
||||
add ecx,4 # advance matrix A by 1 column
|
||||
dec edi
|
||||
jne .LComputeBlockBy1Loop\@
|
||||
|
||||
.LOutputBlock\@:
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A - Supplies the address of matrix A.
|
||||
|
||||
B - Supplies the address of matrix B. The matrix data has been packed using
|
||||
MlasSgemmCopyPackB or MlasSgemmTransposePackB.
|
||||
|
||||
C - Supplies the address of matrix C.
|
||||
|
||||
CountK - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
CountM - Supplies the maximum number of rows that can be processed for
|
||||
matrix A and matrix C. The actual number of rows handled for this
|
||||
invocation depends on the kernel implementation.
|
||||
|
||||
CountN - Supplies the number of columns from matrix B and matrix C to iterate
|
||||
over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
Alpha - Supplies the scalar multiplier (see SGEMM definition).
|
||||
|
||||
ZeroMode - Supplies true if the output matrix must be zero initialized,
|
||||
else false if the output matrix is accumulated into.
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY MlasGemmFloatKernelAvx
|
||||
|
||||
push ebp
|
||||
push ebx
|
||||
push esi
|
||||
push edi
|
||||
mov edx,.LSgemmKernelFrame_MatrixB[esp]
|
||||
mov esi,.LSgemmKernelFrame_MatrixC[esp]
|
||||
mov ebp,.LSgemmKernelFrame_CountN[esp]
|
||||
|
||||
//
|
||||
// Process 2 rows of the matrices.
|
||||
//
|
||||
|
||||
cmp DWORD PTR .LSgemmKernelFrame_CountM[esp],2
|
||||
jb .LProcessCountMLessThan2
|
||||
mov BYTE PTR .LSgemmKernelFrame_CountM[esp],2
|
||||
mov eax,.LSgemmKernelFrame_ldc[esp]
|
||||
mov ebx,.LSgemmKernelFrame_lda[esp]
|
||||
shl eax,2 # convert ldc to bytes
|
||||
shl ebx,2 # convert lda to bytes
|
||||
cmp ebp,8
|
||||
jbe .LProcessRemainingCountN2
|
||||
|
||||
.LProcessNextColumnLoop16x2:
|
||||
mov edi,.LSgemmKernelFrame_CountK[esp]
|
||||
mov ecx,.LSgemmKernelFrame_MatrixA[esp]
|
||||
vxorps xmm4,xmm4,xmm4 # clear block accumulators
|
||||
vxorps xmm5,xmm5,xmm5
|
||||
vxorps xmm6,xmm6,xmm6
|
||||
vxorps xmm7,xmm7,xmm7
|
||||
ComputeBlockAvxLoop ComputeBlockAvxBy16, 2
|
||||
vbroadcastss ymm2,DWORD PTR .LSgemmKernelFrame_alpha[esp]
|
||||
vmulps ymm4,ymm4,ymm2 # multiply by alpha
|
||||
vmulps ymm5,ymm5,ymm2
|
||||
vmulps ymm6,ymm6,ymm2
|
||||
vmulps ymm7,ymm7,ymm2
|
||||
sub ebp,16
|
||||
jb .LOutputMasked16x2Block
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateOutput16x2
|
||||
vaddps ymm4,ymm4,YMMWORD PTR [esi]
|
||||
vaddps ymm5,ymm5,YMMWORD PTR [esi+32]
|
||||
vaddps ymm6,ymm6,YMMWORD PTR [esi+eax]
|
||||
vaddps ymm7,ymm7,YMMWORD PTR [esi+eax+32]
|
||||
|
||||
.LSkipAccumulateOutput16x2:
|
||||
vmovups YMMWORD PTR [esi],ymm4
|
||||
vmovups YMMWORD PTR [esi+32],ymm5
|
||||
vmovups YMMWORD PTR [esi+eax],ymm6
|
||||
vmovups YMMWORD PTR [esi+eax+32],ymm7
|
||||
add esi,16*4 # advance matrix C by 16 columns
|
||||
cmp ebp,8
|
||||
ja .LProcessNextColumnLoop16x2
|
||||
test ebp,ebp
|
||||
jz .LExitKernel
|
||||
|
||||
.LProcessRemainingCountN2:
|
||||
mov edi,.LSgemmKernelFrame_CountK[esp]
|
||||
mov ecx,.LSgemmKernelFrame_MatrixA[esp]
|
||||
vxorps xmm5,xmm5,xmm5 # clear block accumulators
|
||||
vxorps xmm7,xmm7,xmm7
|
||||
ComputeBlockAvxLoop ComputeBlockAvxBy8, 2
|
||||
vbroadcastss ymm2,DWORD PTR .LSgemmKernelFrame_alpha[esp]
|
||||
vmulps ymm5,ymm5,ymm2 # multiply by alpha
|
||||
vmulps ymm7,ymm7,ymm2
|
||||
cmp ebp,8
|
||||
jb .LOutputMasked8x2Block
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateOutput8x2
|
||||
vaddps ymm5,ymm5,YMMWORD PTR [esi]
|
||||
vaddps ymm7,ymm7,YMMWORD PTR [esi+eax]
|
||||
|
||||
.LSkipAccumulateOutput8x2:
|
||||
vmovups YMMWORD PTR [esi],ymm5
|
||||
vmovups YMMWORD PTR [esi+eax],ymm7
|
||||
|
||||
//
|
||||
// Restore non-volatile registers and return.
|
||||
//
|
||||
|
||||
.LExitKernel:
|
||||
movzx eax,BYTE PTR .LSgemmKernelFrame_CountM[esp]
|
||||
vzeroupper
|
||||
pop edi
|
||||
pop esi
|
||||
pop ebx
|
||||
pop ebp
|
||||
ret
|
||||
|
||||
.LOutputMasked16x2Block:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateMasked16x2Block
|
||||
vaddps ymm4,ymm4,YMMWORD PTR [esi]
|
||||
vaddps ymm6,ymm6,YMMWORD PTR [esi+eax]
|
||||
|
||||
.LSkipAccumulateMasked16x2Block:
|
||||
vmovups YMMWORD PTR [esi],ymm4
|
||||
vmovups YMMWORD PTR [esi+eax],ymm6
|
||||
add esi,8*4 # advance matrix C by 8 columns
|
||||
add ebp,8 # correct for over-subtract above
|
||||
|
||||
.LOutputMasked8x2Block:
|
||||
neg ebp
|
||||
LoadGlobalOffsetTable bx
|
||||
mov ebx,DWORD PTR C_UNDERSCORE(MlasMaskMoveTableAvx)@GOT[ebx]
|
||||
vmovdqu ymm0,YMMWORD PTR [ebx+ebp*4+8*4]
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateMasked8x2Block
|
||||
vmaskmovps ymm4,ymm0,YMMWORD PTR [esi]
|
||||
vmaskmovps ymm6,ymm0,YMMWORD PTR [esi+eax]
|
||||
vaddps ymm5,ymm5,ymm4
|
||||
vaddps ymm7,ymm7,ymm6
|
||||
|
||||
.LSkipAccumulateMasked8x2Block:
|
||||
vmaskmovps YMMWORD PTR [esi],ymm0,ymm5
|
||||
vmaskmovps YMMWORD PTR [esi+eax],ymm0,ymm7
|
||||
jmp .LExitKernel
|
||||
|
||||
//
|
||||
// Process 1 row of the matrices.
|
||||
//
|
||||
|
||||
.LProcessCountMLessThan2:
|
||||
mov BYTE PTR .LSgemmKernelFrame_CountM[esp],1
|
||||
mov ebx,.LSgemmKernelFrame_MatrixA[esp]
|
||||
vbroadcastss ymm2,DWORD PTR .LSgemmKernelFrame_alpha[esp]
|
||||
cmp ebp,8
|
||||
jbe .LProcessRemainingCountN1
|
||||
|
||||
.LProcessNextColumnLoop16x1:
|
||||
mov edi,.LSgemmKernelFrame_CountK[esp]
|
||||
mov ecx,ebx # reload matrix A
|
||||
vxorps xmm4,xmm4,xmm4 # clear block accumulators
|
||||
vxorps xmm5,xmm5,xmm5
|
||||
ComputeBlockAvxLoop ComputeBlockAvxBy16, 1
|
||||
vmulps ymm4,ymm4,ymm2 # multiply by alpha
|
||||
vmulps ymm5,ymm5,ymm2
|
||||
sub ebp,16
|
||||
jb .LOutputMasked16x1Block
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulate16x1Block
|
||||
vaddps ymm4,ymm4,YMMWORD PTR [esi]
|
||||
vaddps ymm5,ymm5,YMMWORD PTR [esi+32]
|
||||
|
||||
.LSkipAccumulate16x1Block:
|
||||
vmovups YMMWORD PTR [esi],ymm4
|
||||
vmovups YMMWORD PTR [esi+32],ymm5
|
||||
add esi,16*4 # advance matrix C by 16 columns
|
||||
cmp ebp,8
|
||||
ja .LProcessNextColumnLoop16x1
|
||||
test ebp,ebp
|
||||
jz .LExitKernel
|
||||
|
||||
.LProcessRemainingCountN1:
|
||||
mov edi,.LSgemmKernelFrame_CountK[esp]
|
||||
mov ecx,ebx # reload matrix A
|
||||
vxorps xmm5,xmm5,xmm5 # clear block accumulators
|
||||
ComputeBlockAvxLoop ComputeBlockAvxBy8, 1
|
||||
vmulps ymm5,ymm5,ymm2 # multiply by alpha
|
||||
cmp ebp,8
|
||||
jb .LOutputMasked8x1Block
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulate8x1Block
|
||||
vaddps ymm5,ymm5,YMMWORD PTR [esi]
|
||||
|
||||
.LSkipAccumulate8x1Block:
|
||||
vmovups YMMWORD PTR [esi],ymm5
|
||||
jmp .LExitKernel
|
||||
|
||||
.LOutputMasked16x1Block:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateMasked16x1Block
|
||||
vaddps ymm4,ymm4,YMMWORD PTR [esi]
|
||||
|
||||
.LSkipAccumulateMasked16x1Block:
|
||||
vmovups YMMWORD PTR [esi],ymm4
|
||||
add esi,8*4 # advance matrix C by 8 columns
|
||||
add ebp,8 # correct for over-subtract above
|
||||
|
||||
.LOutputMasked8x1Block:
|
||||
neg ebp
|
||||
LoadGlobalOffsetTable bx
|
||||
mov ebx,DWORD PTR C_UNDERSCORE(MlasMaskMoveTableAvx)@GOT[ebx]
|
||||
vmovdqu ymm0,YMMWORD PTR [ebx+ebp*4+8*4]
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateMasked8x1Block
|
||||
vmaskmovps ymm4,ymm0,YMMWORD PTR [esi]
|
||||
vaddps ymm5,ymm5,ymm4
|
||||
|
||||
.LSkipAccumulateMasked8x1Block:
|
||||
vmaskmovps YMMWORD PTR [esi],ymm0,ymm5
|
||||
jmp .LExitKernel
|
||||
|
||||
.end
|
||||
+406
@@ -0,0 +1,406 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelSse2.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM).
|
||||
|
||||
This implementation uses SSE2 instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
//
|
||||
// Stack frame layout for the SGEMM kernel.
|
||||
//
|
||||
|
||||
.equ .LSgemmKernelFrame_SavedEdi, 0
|
||||
.equ .LSgemmKernelFrame_SavedEsi, 4
|
||||
.equ .LSgemmKernelFrame_SavedEbx, 8
|
||||
.equ .LSgemmKernelFrame_SavedEbp, 12
|
||||
.equ .LSgemmKernelFrame_ReturnAddress, 16
|
||||
.equ .LSgemmKernelFrame_MatrixA, 20
|
||||
.equ .LSgemmKernelFrame_MatrixB, 24
|
||||
.equ .LSgemmKernelFrame_MatrixC, 28
|
||||
.equ .LSgemmKernelFrame_CountK, 32
|
||||
.equ .LSgemmKernelFrame_CountM, 36
|
||||
.equ .LSgemmKernelFrame_CountN, 40
|
||||
.equ .LSgemmKernelFrame_lda, 44
|
||||
.equ .LSgemmKernelFrame_ldc, 48
|
||||
.equ .LSgemmKernelFrame_alpha, 52
|
||||
.equ .LSgemmKernelFrame_ZeroMode, 56
|
||||
|
||||
.text
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for a Nx1 block of the output matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
Shuffle - Supplies the shuffle mask to extract the element from matrix A.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
ebx - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ecx - Supplies the address into the matrix A data.
|
||||
|
||||
edx - Supplies the address into the matrix B data.
|
||||
|
||||
xmm2 - Supplies up to four elements loaded from matrix A.
|
||||
|
||||
xmm4-xmm7 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockSseBy4 VectorOffset, Shuffle
|
||||
|
||||
pshufd xmm3,xmm1,\Shuffle\()
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm4,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+16]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm5,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+32]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm6,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+48]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm7,xmm0
|
||||
|
||||
.endm
|
||||
|
||||
.macro ComputeBlockSseBy3 VectorOffset, Shuffle
|
||||
|
||||
pshufd xmm3,xmm1,\Shuffle\()
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm5,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+16]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm6,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+32]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm7,xmm0
|
||||
|
||||
.endm
|
||||
|
||||
.macro ComputeBlockSseBy2 VectorOffset, Shuffle
|
||||
|
||||
pshufd xmm3,xmm1,\Shuffle\()
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm6,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+16]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm7,xmm0
|
||||
|
||||
.endm
|
||||
|
||||
.macro ComputeBlockSseBy1 VectorOffset, Shuffle
|
||||
|
||||
pshufd xmm3,xmm1,\Shuffle\()
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm7,xmm0
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to execute the block compute macro multiple
|
||||
times and advancing the matrix A and matrix B data pointers.
|
||||
|
||||
Arguments:
|
||||
|
||||
ComputeBlock - Supplies the macro to compute a single block.
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
ebx - Supplies the number of bytes to the next row of matrix A.
|
||||
|
||||
ecx - Supplies the address into the matrix A data.
|
||||
|
||||
edx - Supplies the address into the matrix B data.
|
||||
|
||||
edi - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
xmm4-xmm7 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockSseLoop RowCount
|
||||
|
||||
sub edi,4
|
||||
jb .LProcessRemainingBlocks\@
|
||||
|
||||
.LComputeBlockBy4Loop\@:
|
||||
movups xmm1,XMMWORD PTR [ecx]
|
||||
ComputeBlockSseBy\RowCount\() 0, 0x00
|
||||
ComputeBlockSseBy\RowCount\() 16*4, 0x55
|
||||
sub edx,-32*4 # advance matrix B by 32 columns
|
||||
ComputeBlockSseBy\RowCount\() 0, 0xAA
|
||||
ComputeBlockSseBy\RowCount\() 16*4, 0xFF
|
||||
sub edx,-32*4 # advance matrix B by 32 columns
|
||||
add ecx,4*4 # advance matrix A by 4 columns
|
||||
sub edi,4
|
||||
jae .LComputeBlockBy4Loop\@
|
||||
|
||||
.LProcessRemainingBlocks\@:
|
||||
add edi,4 # correct for over-subtract above
|
||||
jz .LOutputBlock\@
|
||||
|
||||
.LComputeBlockBy1Loop\@:
|
||||
movss xmm1,DWORD PTR [ecx]
|
||||
ComputeBlockSseBy\RowCount\() 0, 0x00
|
||||
add edx,16*4 # advance matrix B by 16 columns
|
||||
add ecx,4 # advance matrix A by 1 column
|
||||
dec edi
|
||||
jne .LComputeBlockBy1Loop\@
|
||||
|
||||
.LOutputBlock\@:
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A - Supplies the address of matrix A.
|
||||
|
||||
B - Supplies the address of matrix B. The matrix data has been packed using
|
||||
MlasSgemmCopyPackB or MlasSgemmTransposePackB.
|
||||
|
||||
C - Supplies the address of matrix C.
|
||||
|
||||
CountK - Supplies the number of columns from matrix A and the number of
|
||||
rows from matrix B to iterate over.
|
||||
|
||||
CountM - Supplies the maximum number of rows that can be processed for
|
||||
matrix A and matrix C. The actual number of rows handled for this
|
||||
invocation depends on the kernel implementation.
|
||||
|
||||
CountN - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
Alpha - Supplies the scalar multiplier (see SGEMM definition).
|
||||
|
||||
ZeroMode - Supplies true if the output matrix must be zero initialized,
|
||||
else false if the output matrix is accumulated into.
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY MlasGemmFloatKernelSse
|
||||
|
||||
push ebp
|
||||
push ebx
|
||||
push esi
|
||||
push edi
|
||||
mov edx,.LSgemmKernelFrame_MatrixB[esp]
|
||||
mov esi,.LSgemmKernelFrame_MatrixC[esp]
|
||||
mov ebp,.LSgemmKernelFrame_CountN[esp]
|
||||
|
||||
//
|
||||
// Process 1 row of the matrices.
|
||||
//
|
||||
|
||||
mov eax,.LSgemmKernelFrame_CountK[esp]
|
||||
mov ebx,.LSgemmKernelFrame_MatrixA[esp]
|
||||
cmp ebp,12
|
||||
jbe .LProcessRemainingCountN
|
||||
|
||||
.LProcessNextColumnLoop16x1:
|
||||
mov edi,eax # reload CountK
|
||||
mov ecx,ebx # reload matrix A
|
||||
xorps xmm4,xmm4 # clear block accumulators
|
||||
xorps xmm5,xmm5
|
||||
xorps xmm6,xmm6
|
||||
xorps xmm7,xmm7
|
||||
ComputeBlockSseLoop 4
|
||||
movss xmm2,DWORD PTR .LSgemmKernelFrame_alpha[esp]
|
||||
shufps xmm2,xmm2,0
|
||||
mulps xmm4,xmm2 # multiply by alpha
|
||||
mulps xmm5,xmm2
|
||||
mulps xmm6,xmm2
|
||||
mulps xmm7,xmm2
|
||||
sub ebp,16
|
||||
jb .LOutputMasked16x1Block
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateOutput16x1
|
||||
movups xmm0,XMMWORD PTR [esi]
|
||||
movups xmm1,XMMWORD PTR [esi+16]
|
||||
movups xmm2,XMMWORD PTR [esi+32]
|
||||
movups xmm3,XMMWORD PTR [esi+48]
|
||||
addps xmm4,xmm0
|
||||
addps xmm5,xmm1
|
||||
addps xmm6,xmm2
|
||||
addps xmm7,xmm3
|
||||
|
||||
.LSkipAccumulateOutput16x1:
|
||||
movups XMMWORD PTR [esi],xmm4
|
||||
movups XMMWORD PTR [esi+16],xmm5
|
||||
movups XMMWORD PTR [esi+32],xmm6
|
||||
movups XMMWORD PTR [esi+48],xmm7
|
||||
add esi,16*4 # advance matrix C by 16 columns
|
||||
cmp ebp,12
|
||||
ja .LProcessNextColumnLoop16x1
|
||||
test ebp,ebp
|
||||
jnz .LProcessRemainingCountN
|
||||
|
||||
//
|
||||
// Restore non-volatile registers and return.
|
||||
//
|
||||
|
||||
.LExitKernel:
|
||||
mov eax,1 # return 1 row handled
|
||||
pop edi
|
||||
pop esi
|
||||
pop ebx
|
||||
pop ebp
|
||||
ret
|
||||
|
||||
//
|
||||
// Process the remaining 1 to 12 columns of the matrices.
|
||||
//
|
||||
|
||||
.LProcessRemainingCountN:
|
||||
mov edi,eax # reload CountK
|
||||
mov ecx,ebx # reload matrix A
|
||||
movss xmm4,DWORD PTR .LSgemmKernelFrame_alpha[esp]
|
||||
shufps xmm4,xmm4,0
|
||||
xorps xmm5,xmm5 # clear block accumulators
|
||||
xorps xmm6,xmm6
|
||||
xorps xmm7,xmm7
|
||||
cmp ebp,4
|
||||
jbe .LProcessRemainingCountN4OrLess
|
||||
cmp ebp,8
|
||||
jbe .LProcessRemainingCountN8OrLess
|
||||
|
||||
.LProcessRemainingCountN12OrLess:
|
||||
ComputeBlockSseLoop 3
|
||||
mulps xmm5,xmm4 # multiply by alpha
|
||||
mulps xmm6,xmm4
|
||||
mulps xmm7,xmm4
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateLeadingN12OrLess
|
||||
movups xmm0,XMMWORD PTR [esi]
|
||||
movups xmm1,XMMWORD PTR [esi+16]
|
||||
addps xmm5,xmm0
|
||||
addps xmm6,xmm1
|
||||
|
||||
.LSkipAccumulateLeadingN12OrLess:
|
||||
movups XMMWORD PTR [esi],xmm5
|
||||
movups XMMWORD PTR [esi+16],xmm6
|
||||
add esi,8*4 # advance matrix C by 8 columns
|
||||
jmp .LOutputTrailingBlock
|
||||
|
||||
.LProcessRemainingCountN8OrLess:
|
||||
ComputeBlockSseLoop 2
|
||||
mulps xmm6,xmm4 # multiply by alpha
|
||||
mulps xmm7,xmm4
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateLeadingN8OrLess
|
||||
movups xmm0,XMMWORD PTR [esi]
|
||||
addps xmm6,xmm0
|
||||
|
||||
.LSkipAccumulateLeadingN8OrLess:
|
||||
movups XMMWORD PTR [esi],xmm6
|
||||
add esi,4*4 # advance matrix C by 4 columns
|
||||
jmp .LOutputTrailingBlock
|
||||
|
||||
.LProcessRemainingCountN4OrLess:
|
||||
ComputeBlockSseLoop 1
|
||||
mulps xmm7,xmm4 # multiply by alpha
|
||||
jmp .LOutputTrailingBlock
|
||||
|
||||
.LOutputMasked16x1Block:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateLeading16x1Block
|
||||
movups xmm0,XMMWORD PTR [esi]
|
||||
movups xmm1,XMMWORD PTR [esi+16]
|
||||
movups xmm2,XMMWORD PTR [esi+32]
|
||||
addps xmm4,xmm0
|
||||
addps xmm5,xmm1
|
||||
addps xmm6,xmm2
|
||||
|
||||
.LSkipAccumulateLeading16x1Block:
|
||||
movups XMMWORD PTR [esi],xmm4
|
||||
movups XMMWORD PTR [esi+16],xmm5
|
||||
movups XMMWORD PTR [esi+32],xmm6
|
||||
add esi,12*4 # advance matrix C by 12 columns
|
||||
|
||||
.LOutputTrailingBlock:
|
||||
test ebp,3
|
||||
jz .LOutputTrailingBlock4Elements
|
||||
test ebp,2
|
||||
jz .LOutputTrailingBlock1Element
|
||||
|
||||
.LOutputTrailingBlock2Elements:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateTrailingBlock2Elements
|
||||
movsd xmm0,QWORD PTR [esi]
|
||||
addps xmm7,xmm0
|
||||
|
||||
.LSkipAccumulateTrailingBlock2Elements:
|
||||
movsd QWORD PTR [esi],xmm7
|
||||
test ebp,1
|
||||
jz .LExitKernel
|
||||
shufps xmm7,xmm7,0xAA # shuffle third float down
|
||||
add esi,2*4 # advance matrix C by 2 columns
|
||||
|
||||
.LOutputTrailingBlock1Element:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateTrailingBlock1Element
|
||||
movss xmm0,DWORD PTR [esi]
|
||||
addss xmm7,xmm0
|
||||
|
||||
.LSkipAccumulateTrailingBlock1Element:
|
||||
movss DWORD PTR [esi],xmm7
|
||||
jmp .LExitKernel
|
||||
|
||||
.LOutputTrailingBlock4Elements:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateTrailingBlock4Elements
|
||||
movups xmm0,XMMWORD PTR [esi]
|
||||
addps xmm7,xmm0
|
||||
|
||||
.LSkipAccumulateTrailingBlock4Elements:
|
||||
movups XMMWORD PTR [esi],xmm7
|
||||
jmp .LExitKernel
|
||||
|
||||
.end
|
||||
Vendored
+79
@@ -0,0 +1,79 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
asmmacro.h
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements common macros for the assembly modules.
|
||||
|
||||
--*/
|
||||
|
||||
#if defined(__APPLE__)
|
||||
#define C_UNDERSCORE(symbol) _##symbol
|
||||
#else
|
||||
#define C_UNDERSCORE(symbol) symbol
|
||||
#endif
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro emits the assembler directives to annotate a new function.
|
||||
|
||||
Arguments:
|
||||
|
||||
FunctionName - Supplies the name of the function.
|
||||
|
||||
--*/
|
||||
|
||||
.macro FUNCTION_ENTRY FunctionName
|
||||
|
||||
.p2align 4
|
||||
#if defined(__APPLE__)
|
||||
.globl _\FunctionName\()
|
||||
_\FunctionName\():
|
||||
#else
|
||||
.globl \FunctionName\()
|
||||
.type \FunctionName\(),@function
|
||||
\FunctionName\():
|
||||
#endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro emits the code to load the global offset table address into the
|
||||
supplied register.
|
||||
|
||||
Arguments:
|
||||
|
||||
TargetReg - Specifies the target register.
|
||||
|
||||
--*/
|
||||
|
||||
.macro LoadGlobalOffsetTable, TargetReg
|
||||
|
||||
//
|
||||
// The LLVM integrated assembler doesn't support the Intel syntax for OFFSET:
|
||||
//
|
||||
// add ebx,OFFSET _GLOBAL_OFFSET_TABLE_
|
||||
//
|
||||
// Workaround this by temporarily switching to AT&T syntax.
|
||||
//
|
||||
|
||||
.att_syntax
|
||||
|
||||
calll __x86.get_pc_thunk.\TargetReg\()
|
||||
addl $_GLOBAL_OFFSET_TABLE_,%e\TargetReg\()
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
.endm
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
x86.get_pc_thunk.S
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements __x86.get_pc_thunk.* to avoid external dependency.
|
||||
|
||||
--*/
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
The routine loads its return address -- which is the address of the
|
||||
instruction that immediately follows -- into the ebx register.
|
||||
|
||||
--*/
|
||||
|
||||
.p2align 4
|
||||
.weak __x86.get_pc_thunk.bx
|
||||
// Hidden = non-preemptible, so the PC-relative call in the SGEMM
|
||||
// kernels binds locally in the .so (no R_386_PC32 link error).
|
||||
.hidden __x86.get_pc_thunk.bx
|
||||
.type __x86.get_pc_thunk.bx,@function
|
||||
__x86.get_pc_thunk.bx:
|
||||
mov ebx, [esp]
|
||||
ret
|
||||
Reference in New Issue
Block a user