mirrors/opencv
1
0
Fork 0
Code Pull Requests Packages Releases Activity

vendor: OpenCV 5.0.0 snapshot at 40738fb16ceddb5fb3fea747585f7ce6abb0605b

Browse Source
This commit is contained in:
Gitea Mirror Bot
2026-08-22 00:10:33 +08:00
commit f7f077da11
6933 changed files with 2335208 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
3rdparty/mlas/lib/aarch64/SgemmKernelNeon.S
Vendored
+482
View File
@@ -0,0 +1,482 @@
/*++
Copyright (c) Microsoft Corporation. All rights reserved.
Licensed under the MIT License.
Module Name:
SgemmKernelNeon.s
Abstract:
This module implements the kernels for the single precision matrix/matrix
multiply operation (SGEMM).
--*/
#include "asmmacro.h"
.text
//
// ClearRowAccumulators
//
// Generates the code to clear the accumulators for a single row of the output
// block.
//
.macro ClearRowAccumulators Columns, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg
movi v\Vec1Reg\().16b,#0
movi v\Vec2Reg\().16b,#0
.if \Columns\() > 8
movi v\Vec3Reg\().16b,#0
movi v\Vec4Reg\().16b,#0
.endif
.endm
//
// ClearBlockAccumulators
//
// Generates the code to clear the accumulators for a single row of the output
// block.
//
.macro ClearBlockAccumulators Columns, Rows
ClearRowAccumulators \Columns\(),16,17,18,19
.if \Rows\() >= 2
ClearRowAccumulators \Columns\(),20,21,22,23
.endif
.if \Rows\() >= 4
ClearRowAccumulators \Columns\(),24,25,26,27
ClearRowAccumulators \Columns\(),28,29,30,31
.endif
.endm
//
// LoadMatrixAElementsBy4
// LoadMatrixAElementsBy1
//
// Generates the code to load 1 or 4 elements from matrix A.
//
.macro LoadMatrixAElementsBy4 Rows
ldr q8,[x0],#16
.if \Rows\() >= 2
ldr q9,[x10],#16
.endif
.if \Rows\() >= 4
ldr q10,[x11],#16
ldr q11,[x12],#16
.endif
.endm
.macro LoadMatrixAElementsBy1 Rows
ldr s8,[x0],#4
.if \Rows\() >= 2
ldr s9,[x10],#4
.endif
.if \Rows\() >= 4
ldr s10,[x11],#4
ldr s11,[x12],#4
.endif
.endm
//
// MultiplyAccumulateRow
//
// Generates the code to multiply and accumulate a single row of the output
// block.
//
.macro MultiplyAccumulateRow Columns, MatrixAReg, Broadcast, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg
fmla v\Vec1Reg\().4s,v4.4s,\MatrixAReg\().s[\Broadcast\()]
fmla v\Vec2Reg\().4s,v5.4s,\MatrixAReg\().s[\Broadcast\()]
.if \Columns\() > 8
fmla v\Vec3Reg\().4s,v6.4s,\MatrixAReg\().s[\Broadcast\()]
fmla v\Vec4Reg\().4s,v7.4s,\MatrixAReg\().s[\Broadcast\()]
.endif
.endm
//
// MultiplyAccumulateBlock
//
// Generates the code to multiply and accumulate into the output block.
//
.macro MultiplyAccumulateBlock Columns, Rows, Broadcast
MultiplyAccumulateRow \Columns\(),v8,\Broadcast\(),16,17,18,19
.if \Rows\() >= 2
MultiplyAccumulateRow \Columns\(),v9,\Broadcast\(),20,21,22,23
.endif
.if \Rows\() >= 4
MultiplyAccumulateRow \Columns\(),v10,\Broadcast\(),24,25,26,27
MultiplyAccumulateRow \Columns\(),v11,\Broadcast\(),28,29,30,31
.endif
.endm
//
// ComputeBlockLoop
//
// Generates the code to loop over K entries of the input matrices to produce
// the output block.
//
.macro ComputeBlockLoop Mode, Columns, Rows
ClearBlockAccumulators \Columns\(),\Rows\()
.if \Rows\() >= 2
add x10,x0,x6,lsl #2 // compute matrix A plus 1 row
.endif
.if \Rows\() >= 4
add x11,x10,x6,lsl #2 // compute matrix A plus 2 rows
add x12,x11,x6,lsl #2 // compute matrix A plus 3 rows
.endif
sub x9,x3,#4 // decrement block count to process
tbnz x9,#63,.L\Mode\().ProcessRemaining\Columns\().x\Rows\().Blocks
.L\Mode\().Compute\Columns\().x\Rows\().BlockBy4Loop:
LoadMatrixAElementsBy4 \Rows\()
ldp q4,q5,[x1],#64*4
.if \Columns\() > 8
ldp q6,q7,[x1,#-56*4]
.endif
MultiplyAccumulateBlock \Columns\(),\Rows\(),0
ldp q4,q5,[x1,#-48*4]
.if \Columns\() > 8
ldp q6,q7,[x1,#-40*4]
.endif
MultiplyAccumulateBlock \Columns\(),\Rows\(),1
ldp q4,q5,[x1,#-32*4]
.if \Columns\() > 8
ldp q6,q7,[x1,#-24*4]
.endif
MultiplyAccumulateBlock \Columns\(),\Rows\(),2
ldp q4,q5,[x1,#-16*4]
.if \Columns\() > 8
ldp q6,q7,[x1,#-8*4]
.endif
MultiplyAccumulateBlock \Columns\(),\Rows\(),3
sub x9,x9,#4
tbz x9,#63,.L\Mode\().Compute\Columns\().x\Rows\().BlockBy4Loop
.L\Mode\().ProcessRemaining\Columns\().x\Rows\().Blocks:
add x9,x9,#4 // correct for over-subtract above
cbz x9,.L\Mode\().Output\Columns\().x\Rows\().Block
.L\Mode\().Compute\Columns\().x\Rows\().BlockBy1Loop:
LoadMatrixAElementsBy1 \Rows\()
ldp q4,q5,[x1],#16*4
.if \Columns\() > 8
ldp q6,q7,[x1,#-8*4]
.endif
MultiplyAccumulateBlock \Columns\(),\Rows\(),0
sub x9,x9,#1
cbnz x9,.L\Mode\().Compute\Columns\().x\Rows\().BlockBy1Loop
.L\Mode\().Output\Columns\().x\Rows\().Block:
.endm
//
// MultiplyAlphaRow
//
// Generates the code to multiply a single row of the output block by the alpha
// value.
//
.macro MultiplyAlphaRow Columns, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg
.if \Columns\() <= 4
fmul v\Vec1Reg\().4s,v\Vec1Reg\().4s,v0.s[0]
.elif \Columns\() <= 8
fmul v\Vec1Reg\().4s,v\Vec1Reg\().4s,v0.s[0]
fmul v\Vec2Reg\().4s,v\Vec2Reg\().4s,v0.s[0]
.elif \Columns\() <= 12
fmul v\Vec1Reg\().4s,v\Vec1Reg\().4s,v0.s[0]
fmul v\Vec2Reg\().4s,v\Vec2Reg\().4s,v0.s[0]
fmul v\Vec3Reg\().4s,v\Vec3Reg\().4s,v0.s[0]
.else
fmul v\Vec1Reg\().4s,v\Vec1Reg\().4s,v0.s[0]
fmul v\Vec2Reg\().4s,v\Vec2Reg\().4s,v0.s[0]
fmul v\Vec3Reg\().4s,v\Vec3Reg\().4s,v0.s[0]
fmul v\Vec4Reg\().4s,v\Vec4Reg\().4s,v0.s[0]
.endif
.endm
//
// MultiplyAlphaBlock
//
// Generates the code to multiply the output block by the alpha value.
//
.macro MultiplyAlphaBlock Columns, Rows
MultiplyAlphaRow \Columns\(),16,17,18,19
.if \Rows\() >= 2
MultiplyAlphaRow \Columns\(),20,21,22,23
.endif
.if \Rows\() >= 4
MultiplyAlphaRow \Columns\(),24,25,26,27
MultiplyAlphaRow \Columns\(),28,29,30,31
.endif
.endm
//
// OutputRow1Element
// OutputRow2Element
// OutputRow4Element
// OutputRow8Element
// OutputRow16Element
//
// Generates the code to store elements to the output block.
//
.macro OutputRow1Element Mode, AddrReg, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg
.ifeqs "\Mode\()","Add"
ld1 {v4.s}[0],[\AddrReg\()]
fmla v4.2s,v\Vec1Reg\().2s,v0.s[0]
st1 {v4.s}[0],[\AddrReg\()] // post-increment not needed for last element
.else
st1 {v\Vec1Reg\().s}[0],[\AddrReg\()]// post-increment not needed for last element
.endif
.endm
.macro OutputRow2Element Mode, AddrReg, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg
.ifeqs "\Mode\()","Add"
ld1 {v4.2s},[\AddrReg\()]
fmla v4.2s,v\Vec1Reg\().2s,v0.s[0]
st1 {v4.2s},[\AddrReg\()],#2*4
.else
st1 {v\Vec1Reg\().2s},[\AddrReg\()],#2*4
.endif
dup v\Vec1Reg\().4s,v\Vec1Reg\().s[2] // shift remaining elements down
.endm
.macro OutputRow4Element Mode, AddrReg, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg
.ifeqs "\Mode\()","Add"
ld1 {v4.4s},[\AddrReg\()]
fmla v4.4s,v\Vec1Reg\().4s,v0.s[0]
st1 {v4.4s},[\AddrReg\()],#4*4
.else
st1 {v\Vec1Reg\().4s},[\AddrReg\()],#4*4
.endif
mov v\Vec1Reg\().16b,v\Vec2Reg\().16b // shift remaining elements down
.endm
.macro OutputRow8Element Mode, AddrReg, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg
.ifeqs "\Mode\()","Add"
ldp q4,q5,[\AddrReg\()]
fmla v4.4s,v\Vec1Reg\().4s,v0.s[0]
fmla v5.4s,v\Vec2Reg\().4s,v0.s[0]
stp q4,q5,[\AddrReg\()],#8*4
.else
stp q\Vec1Reg\(),q\Vec2Reg\(),[\AddrReg\()],#8*4
.endif
mov v\Vec1Reg\().16b,v\Vec3Reg\().16b // shift remaining elements down
mov v\Vec2Reg\().16b,v\Vec4Reg\().16b
.endm
.macro OutputRow16Element Mode, AddrReg, Vec1Reg, Vec2Reg, Vec3Reg, Vec4Reg
.ifeqs "\Mode\()","Add"
ldp q4,q5,[\AddrReg\()]
ldp q6,q7,[\AddrReg\(),#8*4]
fmla v4.4s,v\Vec1Reg\().4s,v0.s[0]
fmla v5.4s,v\Vec2Reg\().4s,v0.s[0]
fmla v6.4s,v\Vec3Reg\().4s,v0.s[0]
fmla v7.4s,v\Vec4Reg\().4s,v0.s[0]
stp q4,q5,[\AddrReg\()],#16*4
stp q6,q7,[\AddrReg\(),#-8*4]
.else
stp q\Vec1Reg\(),q\Vec2Reg\(),[\AddrReg\()],#16*4
stp q\Vec3Reg\(),q\Vec4Reg\(),[\AddrReg\(),#-8*4]
.endif
.endm
//
// OutputBlock
//
// Generates the code to store the output block.
//
.macro OutputBlock Mode, Columns, Rows
OutputRow\Columns\()Element \Mode\(),x2,16,17,18,19
.if \Rows\() >= 2
OutputRow\Columns\()Element \Mode\(),x13,20,21,22,23
.endif
.if \Rows\() >= 4
OutputRow\Columns\()Element \Mode\(),x14,24,25,26,27
OutputRow\Columns\()Element \Mode\(),x15,28,29,30,31
.endif
.endm
//
// ProcessRows
//
// Generates the code to process a compute and store the output block for a
// fixed number of rows.
//
.macro ProcessRows Mode, Rows
mov x4,#\Rows\() // return number of rows handled
cmp x5,#8
ble .L\Mode\().ProcessRemainingCountN\Rows\()
.L\Mode\().ProcessNextColumnLoop16x\Rows\():
ComputeBlockLoop \Mode\(),16,\Rows\()
.ifeqs "\Mode\()","Zero"
MultiplyAlphaBlock 16,\Rows\()
.endif
sub x5,x5,#16
tbnz x5,#63,.L\Mode\().OutputMasked16x\Rows\().Block
OutputBlock \Mode\(),16,\Rows\()
mov x0,x8 // reload matrix A
cmp x5,#8
bgt .L\Mode\().ProcessNextColumnLoop16x\Rows\()
cbz x5,.L\Mode\().ExitKernel
.L\Mode\().ProcessRemainingCountN\Rows\():
ComputeBlockLoop \Mode\(),8,\Rows\()
.ifeqs "\Mode\()","Zero"
MultiplyAlphaBlock 8,\Rows\()
.endif
.L\Mode\().OutputMasked16x\Rows\().Block:
tbz x5,#3,.L\Mode\().OutputRemaining7x\Rows\().Block
OutputBlock \Mode\(),8,\Rows\()
.L\Mode\().OutputRemaining7x\Rows\().Block:
tbz x5,#2,.L\Mode\().OutputRemaining3x\Rows\().Block
OutputBlock \Mode\(),4,\Rows\()
.L\Mode\().OutputRemaining3x\Rows\().Block:
tbz x5,#1,.L\Mode\().OutputRemaining1x\Rows\().Block
OutputBlock \Mode\(),2,\Rows\()
.L\Mode\().OutputRemaining1x\Rows\().Block:
tbz x5,#0,.L\Mode\().ExitKernel
OutputBlock \Mode\(),1,\Rows\()
.endm
/*++
Routine Description:
This routine is an inner kernel to compute matrix multiplication for a
set of rows.
Arguments:
A (x0) - Supplies the address of matrix A.
B (x1) - Supplies the address of matrix B. The matrix data has been packed
using MlasSgemmCopyPackB or MlasSgemmTransposePackB.
C (x2) - Supplies the address of matrix C.
CountK (x3) - Supplies the number of columns from matrix A and the number
of rows from matrix B to iterate over.
CountM (x4) - Supplies the maximum number of rows that can be processed for
matrix A and matrix C. The actual number of rows handled for this
invocation depends on the kernel implementation.
CountN (x5) - Supplies the number of columns from matrix B and matrix C to
iterate over.
lda (x6) - Supplies the first dimension of matrix A.
ldc (x7) - Supplies the first dimension of matrix C.
Alpha (s0) - Supplies the scalar multiplier (see SGEMM definition).
Return Value:
Returns the number of rows handled.
--*/
.macro SgemmKernelNeonFunction Mode
FUNCTION_ENTRY MlasSgemmKernel\Mode\()
stp d8,d9,[sp,#-32]!
stp d10,d11,[sp,#16]
add x13,x2,x7,lsl #2 // compute matrix C plus 1 row
add x14,x13,x7,lsl #2 // compute matrix C plus 2 rows
add x15,x14,x7,lsl #2 // compute matrix C plus 3 rows
mov x8,x0 // save matrix A
//
// Process 4 rows of the matrices.
//
cmp x4,#4
blt .L\Mode\().ProcessCountMLessThan4
ProcessRows \Mode\(),4
//
// Restore non-volatile registers and return.
//
.L\Mode\().ExitKernel:
mov x0,x4
ldp d10,d11,[sp,#16]
ldp d8,d9,[sp],#32
ret
//
// Process 2 rows of the matrices.
//
.L\Mode\().ProcessCountMLessThan4:
cmp x4,#2
blt .L\Mode\().ProcessCountMLessThan2
ProcessRows \Mode\(),2
b .L\Mode\().ExitKernel
//
// Process 1 row of the matrices.
//
.L\Mode\().ProcessCountMLessThan2:
ProcessRows \Mode\(),1
b .L\Mode\().ExitKernel
.endm
SgemmKernelNeonFunction Zero
SgemmKernelNeonFunction Add
.end
3rdparty/mlas/lib/aarch64/SgemvKernelNeon.S
Vendored
+303
View File
@@ -0,0 +1,303 @@
/*++
Copyright (c) Microsoft Corporation. All rights reserved.
Licensed under the MIT License.
Module Name:
SgemvKernelNeon.s
Abstract:
This module implements the kernels for the single precision matrix/vector
multiply operation (SGEMV).
--*/
#include "asmmacro.h"
.text
/*++
Routine Description:
This routine is an inner kernel to compute matrix multiplication for a
set of rows. This handles the special case of M=1.
The elements in matrix B are not transposed.
Arguments:
A (x0) - Supplies the address of matrix A.
B (x1) - Supplies the address of matrix B.
C (x2) - Supplies the address of matrix C.
CountK (x3) - Supplies the number of columns from matrix A and the number
of rows from matrix B to iterate over.
CountN (x4) - Supplies the number of columns from matrix B and matrix C to
iterate over.
ldb (x5) - Supplies the first dimension of matrix B.
ZeroMode (x6) - Supplies true if the output matrix must be zero initialized,
else false if the output matrix is accumulated into.
Return Value:
None.
--*/
FUNCTION_ENTRY MlasGemvFloatKernel
cmp x4,#64
blo .LSgemvN.ProcessRemainingCountN
mov x14,x0 // preserve vector A
//
// Process 64 columns at a time in a loop.
//
.LSgemvN.ProcessColumnLoopBy64:
ldr q4,[x1]
add x15,x1,#256 // compute next matrix B
ldr q5,[x1,#16]
tst w6,0xFF // ZeroMode?
mov x13,x3 // reload CountK
ldr q6,[x1,#32]
beq .LSgemvN.LoadOutputBy64
movi v16.4s,#0
movi v17.4s,#0
movi v18.4s,#0
movi v19.4s,#0
movi v20.4s,#0
movi v21.4s,#0
movi v22.4s,#0
movi v23.4s,#0
movi v24.4s,#0
movi v25.4s,#0
movi v26.4s,#0
movi v27.4s,#0
movi v28.4s,#0
movi v29.4s,#0
movi v30.4s,#0
movi v31.4s,#0
b .LSgemvN.MultiplyAccumulateBy64
.LSgemvN.LoadOutputBy64:
ldp q16,q17,[x2]
ldp q18,q19,[x2,#32]
ldp q20,q21,[x2,#64]
ldp q22,q23,[x2,#96]
ldp q24,q25,[x2,#128]
ldp q26,q27,[x2,#160]
ldp q28,q29,[x2,#192]
ldp q30,q31,[x2,#224]
.LSgemvN.MultiplyAccumulateBy64:
ld1r {v0.4s},[x0] // broadcast next vector A element
add x0,x0,4 // advance vector A by 1 element
sub x13,x13,#1 // decrement K remaining
fmla v16.4s,v4.4s,v0.4s
ldr q7,[x1,#48]
fmla v17.4s,v5.4s,v0.4s
ldr q4,[x1,#64]
fmla v18.4s,v6.4s,v0.4s
ldr q5,[x1,#80]
fmla v19.4s,v7.4s,v0.4s
ldr q6,[x1,#96]
fmla v20.4s,v4.4s,v0.4s
ldr q7,[x1,#112]
fmla v21.4s,v5.4s,v0.4s
ldr q4,[x1,#128]
fmla v22.4s,v6.4s,v0.4s
ldr q5,[x1,#144]
fmla v23.4s,v7.4s,v0.4s
ldr q6,[x1,#160]
fmla v24.4s,v4.4s,v0.4s
ldr q7,[x1,#176]
fmla v25.4s,v5.4s,v0.4s
ldr q4,[x1,#192]
fmla v26.4s,v6.4s,v0.4s
ldr q5,[x1,#208]
fmla v27.4s,v7.4s,v0.4s
ldr q6,[x1,#224]
fmla v28.4s,v4.4s,v0.4s
ldr q7,[x1,#240]
add x1,x1,x5,lsl #2 // compute next matrix B row address
cbz x13,.LSgemvN.StoreOutputBy64
ldr q4,[x1] // load data for next iteration
fmla v29.4s,v5.4s,v0.4s
ldr q5,[x1,#16]
fmla v30.4s,v6.4s,v0.4s
ldr q6,[x1,#32]
fmla v31.4s,v7.4s,v0.4s
b .LSgemvN.MultiplyAccumulateBy64
.LSgemvN.StoreOutputBy64:
stp q16,q17,[x2]
fmla v29.4s,v5.4s,v0.4s // finish computing tail vectors
stp q18,q19,[x2,#32]
fmla v30.4s,v6.4s,v0.4s
stp q20,q21,[x2,#64]
fmla v31.4s,v7.4s,v0.4s
stp q22,q23,[x2,#96]
sub x4,x4,#64 // subtract 64 columns
stp q24,q25,[x2,#128]
mov x0,x14 // reload vector A
stp q26,q27,[x2,#160]
mov x1,x15 // load next matrix B
stp q28,q29,[x2,#192]
stp q30,q31,[x2,#224]
add x2,x2,#256 // advance vector C by 64 columns
cbz x4,.LSgemvN.ExitKernel
cmp x4,#64
bhs .LSgemvN.ProcessColumnLoopBy64
//
// Process the remaining 1 to 63 columns.
//
.LSgemvN.ProcessRemainingCountN:
tst w6,0xFF // ZeroMode?
beq .LSgemvN.LoadOutputPartial32
movi v16.4s,#0
movi v17.4s,#0
movi v18.4s,#0
movi v19.4s,#0
movi v20.4s,#0
movi v21.4s,#0
movi v22.4s,#0
movi v23.4s,#0
movi v24.4s,#0
movi v25.4s,#0
movi v26.4s,#0
movi v27.4s,#0
movi v28.4s,#0
movi v29.4s,#0
movi v30.4s,#0
movi v31.4s,#0 // trailing float[2]
movi v1.4s,#0 // trailing float[1]
b .LSgemvN.ProcessNextPartialRow
.LSgemvN.LoadOutputPartial32:
mov x15,x2
tbz x4,#5,.LSgemvN.LoadOutputPartial16
ldp q16,q17,[x15],#128
ldp q18,q19,[x15,#-96]
ldp q20,q21,[x15,#-64]
ldp q22,q23,[x15,#-32]
.LSgemvN.LoadOutputPartial16:
tbz x4,#4,.LSgemvN.LoadOutputPartial8
ldp q24,q25,[x15],#64
ldp q26,q27,[x15,#-32]
.LSgemvN.LoadOutputPartial8:
tbz x4,#3,.LSgemvN.LoadOutputPartial4
ldp q28,q29,[x15],#32
.LSgemvN.LoadOutputPartial4:
tbz x4,#2,.LSgemvN.LoadOutputPartial2
ldr q30,[x15],#16
.LSgemvN.LoadOutputPartial2:
tbz x4,#1,.LSgemvN.LoadOutputPartial1
ldr d31,[x15],#8
.LSgemvN.LoadOutputPartial1:
tbz x4,#0,.LSgemvN.ProcessNextPartialRow
ldr s1,[x15]
.LSgemvN.ProcessNextPartialRow:
ld1r {v0.4s},[x0]
add x0,x0,4
sub x3,x3,#1 // decrement K remaining
mov x15,x1
.LSgemvN.MultiplyAccumulatePartial32:
tbz x4,#5,.LSgemvN.MultiplyAccumulatePartial16
ldp q4,q5,[x15],#128
fmla v16.4s,v4.4s,v0.4s
ldp q6,q7,[x15,#-96]
fmla v17.4s,v5.4s,v0.4s
ldp q4,q5,[x15,#-64]
fmla v18.4s,v6.4s,v0.4s
fmla v19.4s,v7.4s,v0.4s
ldp q6,q7,[x15,#-32]
fmla v20.4s,v4.4s,v0.4s
fmla v21.4s,v5.4s,v0.4s
fmla v22.4s,v6.4s,v0.4s
fmla v23.4s,v7.4s,v0.4s
.LSgemvN.MultiplyAccumulatePartial16:
tbz x4,#4,.LSgemvN.MultiplyAccumulatePartial8
ldp q4,q5,[x15],#64
fmla v24.4s,v4.4s,v0.4s
ldp q6,q7,[x15,#-32]
fmla v25.4s,v5.4s,v0.4s
fmla v26.4s,v6.4s,v0.4s
fmla v27.4s,v7.4s,v0.4s
.LSgemvN.MultiplyAccumulatePartial8:
tbz x4,#3,.LSgemvN.MultiplyAccumulatePartial4
ldp q4,q5,[x15],#32
fmla v28.4s,v4.4s,v0.4s
fmla v29.4s,v5.4s,v0.4s
.LSgemvN.MultiplyAccumulatePartial4:
tbz x4,#2,.LSgemvN.MultiplyAccumulatePartial2
ldr q4,[x15],#16
fmla v30.4s,v4.4s,v0.4s
.LSgemvN.MultiplyAccumulatePartial2:
tbz x4,#1,.LSgemvN.MultiplyAccumulatePartial1
ldr d4,[x15],#8
fmla v31.4s,v4.4s,v0.4s
.LSgemvN.MultiplyAccumulatePartial1:
tbz x4,#0,.LSgemvN.AdvancePartialRow
ldr s4,[x15]
fmla v1.4s,v4.4s,v0.4s
.LSgemvN.AdvancePartialRow:
add x1,x1,x5,lsl #2 // compute next matrix B row address
cbnz x3,.LSgemvN.ProcessNextPartialRow
.LSgemvN.StoreOutputPartial32:
tbz x4,#5,.LSgemvN.StoreOutputPartial16
stp q16,q17,[x2],#128
stp q18,q19,[x2,#-96]
stp q20,q21,[x2,#-64]
stp q22,q23,[x2,#-32]
.LSgemvN.StoreOutputPartial16:
tbz x4,#4,.LSgemvN.StoreOutputPartial8
stp q24,q25,[x2],#64
stp q26,q27,[x2,#-32]
.LSgemvN.StoreOutputPartial8:
tbz x4,#3,.LSgemvN.StoreOutputPartial4
stp q28,q29,[x2],#32
.LSgemvN.StoreOutputPartial4:
tbz x4,#2,.LSgemvN.StoreOutputPartial2
str q30,[x2],#16
.LSgemvN.StoreOutputPartial2:
tbz x4,#1,.LSgemvN.StoreOutputPartial1
str d31,[x2],#8
.LSgemvN.StoreOutputPartial1:
tbz x4,#0,.LSgemvN.ExitKernel
str s1,[x2]
.LSgemvN.ExitKernel:
ret
.end
3rdparty/mlas/lib/aarch64/asmmacro.h
Vendored
+95
View File
@@ -0,0 +1,95 @@
/*++
Copyright (c) Microsoft Corporation. All rights reserved.
Licensed under the MIT License.
Module Name:
asmmacro.h
Abstract:
This module implements common macros for the assembly modules.
--*/
/*++
Macro Description:
This macro emits the assembler directives to annotate a new function.
Arguments:
FunctionName - Supplies the name of the function.
--*/
.macro FUNCTION_ENTRY FunctionName
.p2align 2
#if defined(__APPLE__)
.globl _\FunctionName\()
_\FunctionName\():
#else
.globl \FunctionName\()
.type \FunctionName\(),%function
\FunctionName\():
#endif
.endm
/*++
Macro Description:
This macro conditionally emits the statement if Count is greater than or
equal to Value.
Arguments:
Count - Supplies the variable used in the comparison.
Value - Supplies the static used in the comparison.
Statement - Supplies the statement to conditionally emit.
--*/
.macro EmitIfCountGE Count1, Value1, Statement
.if (\Count1\() >= \Value1\())
\Statement\()
.endif
.endm
/*++
Macro Description:
This macro conditionally emits the statement if Count1 is greater than or
equal to Value1 and Count2 is greater than or equal to Value2.
Arguments:
Count1 - Supplies the variable used in the comparison.
Value1 - Supplies the static used in the comparison.
Count2 - Supplies the variable used in the comparison.
Value2 - Supplies the static used in the comparison.
Statement - Supplies the statement to conditionally emit.
--*/
.macro EmitIfCount2GE Count1, Value1, Count2, Value2, Statement
.if (\Count1\() >= \Value1\()) && (\Count2\() >= \Value2\())
\Statement\()
.endif
.endm