vendor: OpenCV 5.0.0 snapshot at 40738fb16ceddb5fb3fea747585f7ce6abb0605b
This commit is contained in:
Gitea Mirror Bot
@@ -0,0 +1,193 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SconvDepthwiseKernelScalar.cpp
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision direct
|
||||
convolution kernels.
|
||||
|
||||
--*/
|
||||
|
||||
#include "mlasi.h"
|
||||
|
||||
static
|
||||
void
|
||||
MlasConv2dSingleChannel_CHW_Kernel3x3_Pad01_Dilation1(
|
||||
const MLAS_CONV_PARAMETERS* Parameters,
|
||||
const float* Input,
|
||||
const float* Filter,
|
||||
float* Output,
|
||||
const float* Zeros
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute convolution on one channel input with one filter channel.
|
||||
|
||||
Arguments:
|
||||
|
||||
Parameters - conv parameters calculated based on conv parameters like padding, strides, dilations, etc.
|
||||
|
||||
Input - input channel data start. Input is NCHW, so this pointer point to single H x W image data.
|
||||
|
||||
Filter - Whole filters are of F x CpG x FH x FW, this filter point to single FH x FW filter data.
|
||||
|
||||
Output - whole output are of N x F x OH x OW. This pointer point to single OH x OW output image data.
|
||||
|
||||
Zeroes - Point to working buffer where all 0.0f are filled.
|
||||
|
||||
--*/
|
||||
{
|
||||
const size_t W = Parameters->InputShape[1];
|
||||
const float beta = Parameters->Beta;
|
||||
|
||||
if (W > 1) {
|
||||
|
||||
const float w00 = Filter[0];
|
||||
const float w01 = Filter[1];
|
||||
const float w02 = Filter[2];
|
||||
const float w10 = Filter[3];
|
||||
const float w11 = Filter[4];
|
||||
const float w12 = Filter[5];
|
||||
const float w20 = Filter[6];
|
||||
const float w21 = Filter[7];
|
||||
const float w22 = Filter[8];
|
||||
|
||||
const size_t H = Parameters->InputShape[0];
|
||||
const size_t pad_top = Parameters->Padding[0];
|
||||
const size_t pad_left = Parameters->Padding[1];
|
||||
const size_t stride_h = Parameters->StrideShape[0];
|
||||
const size_t stride_w = Parameters->StrideShape[1];
|
||||
|
||||
// We treat pad_left, pad_top are hard require.
|
||||
// While pad_right and pad_bottom could be adjusted if they do not 100% match other parameters.
|
||||
const size_t pad_right = (((Parameters->OutputShape[1] - 1) * stride_w + 3) > (pad_left + W)) ? 1 : 0;
|
||||
|
||||
const float* row0 = (pad_top > 0) ? Zeros : (Input - pad_left);
|
||||
// Need to handle effective pad_bottom is 2 when H == 1
|
||||
const float* row1 = (H + pad_top <= 1) ? Zeros : (Input + (1 - pad_top) * W) - pad_left;
|
||||
const float* row2 = (H + pad_top <= 2) ? Zeros : (row1 + W);
|
||||
|
||||
for (size_t h = 0, out_row = Parameters->OutputShape[0]; out_row > 0; --out_row) {
|
||||
auto out_col = Parameters->OutputShape[1];
|
||||
|
||||
if (pad_left == 1) {
|
||||
float dotsum = w01 * row0[1] + w02 * row0[2] + w11 * row1[1] + w12 * row1[2] +
|
||||
w21 * row2[1] + w22 * row2[2] + (beta == 0.f ? 0.f : *Output * beta);
|
||||
*Output++ = dotsum;
|
||||
out_col--;
|
||||
row0 += stride_w;
|
||||
row1 += stride_w;
|
||||
row2 += stride_w;
|
||||
}
|
||||
|
||||
for (; out_col > pad_right; out_col--) {
|
||||
float dotsum = w00 * row0[0] + w01 * row0[1] + w02 * row0[2] + w10 * row1[0] +
|
||||
w11 * row1[1] + w12 * row1[2] + w20 * row2[0] + w21 * row2[1] +
|
||||
w22 * row2[2] + (beta == 0.f ? 0.f : *Output * beta);
|
||||
*Output++ = dotsum;
|
||||
row0 += stride_w;
|
||||
row1 += stride_w;
|
||||
row2 += stride_w;
|
||||
}
|
||||
|
||||
if (out_col == 1) { // pad_right == 1
|
||||
float dotsum = w00 * row0[0] + w01 * row0[1] + w10 * row1[0] + w11 * row1[1] +
|
||||
w20 * row2[0] + w21 * row2[1] + (beta == 0.f ? 0.f : *Output * beta);
|
||||
*Output++ = dotsum;
|
||||
}
|
||||
|
||||
h += stride_h;
|
||||
row0 = (Input + (h - pad_top) * W) - pad_left;
|
||||
row1 = row0 + W;
|
||||
row2 = (h + 2 >= H + pad_top) ? Zeros : (row1 + W);
|
||||
}
|
||||
|
||||
} else { // W == 1
|
||||
|
||||
const size_t H = Parameters->InputShape[0];
|
||||
const size_t pad_left = Parameters->Padding[1];
|
||||
const size_t pad_top = Parameters->Padding[0];
|
||||
const size_t stride_h = Parameters->StrideShape[0];
|
||||
size_t out_row = Parameters->OutputShape[0];
|
||||
|
||||
// Make sure pad_bottom is consistent with other parameters.
|
||||
size_t pad_bottom = ((out_row - 1) * stride_h + 3) > (pad_top + H) ?
|
||||
((out_row - 1) * stride_h + 3) - (pad_top + H) : 0;
|
||||
|
||||
const float w0 = Filter[pad_left ? 1 : 0];
|
||||
const float w1 = Filter[pad_left ? 4 : 3];
|
||||
const float w2 = Filter[pad_left ? 7 : 6];
|
||||
auto init_v = (beta == 0.f ? 0.f : *Output * beta);
|
||||
|
||||
if (pad_top == 1) {
|
||||
*Output++ = w1 * Input[0] + w2 * ((H + pad_top <= 2) ? 0.0f : Input[1]) + init_v;
|
||||
out_row--;
|
||||
}
|
||||
|
||||
for (const float* row = Input + pad_top * stride_h - pad_top; out_row > pad_bottom; --out_row) {
|
||||
// All pixels are in the input col
|
||||
auto init = (beta == 0.f ? 0.f : *Output * beta);
|
||||
*Output++ = w0 * row[0] + w1 * row[1] + w2 * row[2] + init;
|
||||
row += stride_h;
|
||||
}
|
||||
|
||||
if (out_row > 0) {
|
||||
// last 1 or 2 rows are from the padding zero row.
|
||||
// out_row == 1 when arrive here
|
||||
if (pad_bottom == 1) {
|
||||
const float* row = Input + H - 2;
|
||||
*Output++ = w0 * row[0] + w1 * row[1] + init_v;
|
||||
} else { // pad_bottom == 2 and H == 1 and padding_top == 0
|
||||
*Output++ = w0 * Input[0] + init_v;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
MlasConvDepthwiseFloat_CHW(
|
||||
const MLAS_CONV_PARAMETERS* Parameters,
|
||||
const float* Input,
|
||||
const float* Filter,
|
||||
float* Output,
|
||||
const float* Zeros
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute depthwise convolution for one filter channel on one input channel.
|
||||
|
||||
Arguments:
|
||||
|
||||
Parameters - conv parameters calculated based on conv parameters like padding, strides, dilations, etc.
|
||||
|
||||
Input - input channel data start. Input is NCHW, so this pointer point to single H x W image data.
|
||||
|
||||
Filter - Whole filters are of F x CpG x FH x FW, this filter point to single FH x FW filter data.
|
||||
|
||||
Output - whole output are of N x F x OH x OW. This pointer point to single OH x OW output image data.
|
||||
|
||||
Zeroes - Point to working buffer where all 0.0f are filled.
|
||||
|
||||
Note:
|
||||
No checking here as it is inner loop. Logic in generating Parameters controls the check.
|
||||
|
||||
Currently only support 2d kernel 3x3.
|
||||
Will add general case and more special case if needed later.
|
||||
|
||||
--*/
|
||||
{
|
||||
MlasConv2dSingleChannel_CHW_Kernel3x3_Pad01_Dilation1(Parameters, Input, Filter, Output, Zeros);
|
||||
}
|
||||
+480
@@ -0,0 +1,480 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelScalar.cpp
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM).
|
||||
|
||||
--*/
|
||||
|
||||
#include "mlasi.h"
|
||||
|
||||
template<bool ZeroMode, bool ProcessTwoRows>
|
||||
size_t
|
||||
MlasSgemmKernel(
|
||||
const float* A,
|
||||
const float* B,
|
||||
float* C,
|
||||
size_t CountK,
|
||||
size_t CountN,
|
||||
size_t lda,
|
||||
size_t ldc,
|
||||
float alpha
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A - Supplies the address of matrix A.
|
||||
|
||||
B - Supplies the address of matrix B. The matrix data has been packed using
|
||||
MlasSgemmCopyPackB or MlasSgemmTransposePackB with a packing width
|
||||
of 16.
|
||||
|
||||
C - Supplies the address of matrix C.
|
||||
|
||||
CountK - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
CountN - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
alpha - Supplies the scaler multiplier (see SGEMM definition).
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
{
|
||||
float Row0Block00;
|
||||
float Row0Block01;
|
||||
float Row0Block02;
|
||||
float Row0Block03;
|
||||
|
||||
float Row1Block00;
|
||||
float Row1Block01;
|
||||
float Row1Block02;
|
||||
float Row1Block03;
|
||||
|
||||
#if defined(_WIN32)
|
||||
|
||||
if (!ProcessTwoRows) {
|
||||
UNREFERENCED_PARAMETER(lda);
|
||||
UNREFERENCED_PARAMETER(ldc);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
int countb = 0;
|
||||
|
||||
do {
|
||||
|
||||
float BElements00;
|
||||
float BElements01;
|
||||
float BElements02;
|
||||
float BElements03;
|
||||
|
||||
float Row0AElements0;
|
||||
float Row0AElements1;
|
||||
float Row1AElements0;
|
||||
float Row1AElements1;
|
||||
|
||||
//
|
||||
// Clear the block accumulators.
|
||||
//
|
||||
|
||||
Row0Block00 = 0.0f;
|
||||
Row0Block01 = 0.0f;
|
||||
Row0Block02 = 0.0f;
|
||||
Row0Block03 = 0.0f;
|
||||
|
||||
if (ProcessTwoRows) {
|
||||
Row1Block00 = 0.0f;
|
||||
Row1Block01 = 0.0f;
|
||||
Row1Block02 = 0.0f;
|
||||
Row1Block03 = 0.0f;
|
||||
}
|
||||
|
||||
//
|
||||
// Compute the 4x1 or 4x2 output block.
|
||||
//
|
||||
|
||||
const float* a = A;
|
||||
const float* b = B;
|
||||
size_t k = CountK;
|
||||
|
||||
while (k >= 2) {
|
||||
|
||||
Row0AElements0 = a[0];
|
||||
Row0AElements1 = a[1];
|
||||
|
||||
if (ProcessTwoRows) {
|
||||
Row1AElements0 = a[lda];
|
||||
Row1AElements1 = a[lda + 1];
|
||||
}
|
||||
|
||||
BElements00 = b[0];
|
||||
BElements01 = b[1];
|
||||
BElements02 = b[2];
|
||||
BElements03 = b[3];
|
||||
Row0Block00 = Row0Block00 + BElements00 * Row0AElements0;
|
||||
Row0Block01 = Row0Block01 + BElements01 * Row0AElements0;
|
||||
Row0Block02 = Row0Block02 + BElements02 * Row0AElements0;
|
||||
Row0Block03 = Row0Block03 + BElements03 * Row0AElements0;
|
||||
|
||||
if (ProcessTwoRows) {
|
||||
Row1Block00 = Row1Block00 + BElements00 * Row1AElements0;
|
||||
Row1Block01 = Row1Block01 + BElements01 * Row1AElements0;
|
||||
Row1Block02 = Row1Block02 + BElements02 * Row1AElements0;
|
||||
Row1Block03 = Row1Block03 + BElements03 * Row1AElements0;
|
||||
}
|
||||
|
||||
BElements00 = b[16];
|
||||
BElements01 = b[17];
|
||||
BElements02 = b[18];
|
||||
BElements03 = b[19];
|
||||
Row0Block00 = Row0Block00 + BElements00 * Row0AElements1;
|
||||
Row0Block01 = Row0Block01 + BElements01 * Row0AElements1;
|
||||
Row0Block02 = Row0Block02 + BElements02 * Row0AElements1;
|
||||
Row0Block03 = Row0Block03 + BElements03 * Row0AElements1;
|
||||
|
||||
if (ProcessTwoRows) {
|
||||
Row1Block00 = Row1Block00 + BElements00 * Row1AElements1;
|
||||
Row1Block01 = Row1Block01 + BElements01 * Row1AElements1;
|
||||
Row1Block02 = Row1Block02 + BElements02 * Row1AElements1;
|
||||
Row1Block03 = Row1Block03 + BElements03 * Row1AElements1;
|
||||
}
|
||||
|
||||
a += 2;
|
||||
b += 32;
|
||||
k -= 2;
|
||||
}
|
||||
|
||||
if (k > 0) {
|
||||
|
||||
Row0AElements0 = a[0];
|
||||
|
||||
if (ProcessTwoRows) {
|
||||
Row1AElements0 = a[lda];
|
||||
}
|
||||
|
||||
BElements00 = b[0];
|
||||
BElements01 = b[1];
|
||||
BElements02 = b[2];
|
||||
BElements03 = b[3];
|
||||
Row0Block00 = Row0Block00 + BElements00 * Row0AElements0;
|
||||
Row0Block01 = Row0Block01 + BElements01 * Row0AElements0;
|
||||
Row0Block02 = Row0Block02 + BElements02 * Row0AElements0;
|
||||
Row0Block03 = Row0Block03 + BElements03 * Row0AElements0;
|
||||
|
||||
if (ProcessTwoRows) {
|
||||
Row1Block00 = Row1Block00 + BElements00 * Row1AElements0;
|
||||
Row1Block01 = Row1Block01 + BElements01 * Row1AElements0;
|
||||
Row1Block02 = Row1Block02 + BElements02 * Row1AElements0;
|
||||
Row1Block03 = Row1Block03 + BElements03 * Row1AElements0;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Multiply by the alpha value.
|
||||
//
|
||||
|
||||
Row0Block00 = Row0Block00 * alpha;
|
||||
Row0Block01 = Row0Block01 * alpha;
|
||||
Row0Block02 = Row0Block02 * alpha;
|
||||
Row0Block03 = Row0Block03 * alpha;
|
||||
|
||||
if (ProcessTwoRows) {
|
||||
Row1Block00 = Row1Block00 * alpha;
|
||||
Row1Block01 = Row1Block01 * alpha;
|
||||
Row1Block02 = Row1Block02 * alpha;
|
||||
Row1Block03 = Row1Block03 * alpha;
|
||||
}
|
||||
|
||||
if (CountN >= 4) {
|
||||
|
||||
//
|
||||
// Store the entire output block.
|
||||
//
|
||||
|
||||
if (!ZeroMode) {
|
||||
Row0Block00 = Row0Block00 + C[0];
|
||||
Row0Block01 = Row0Block01 + C[1];
|
||||
Row0Block02 = Row0Block02 + C[2];
|
||||
Row0Block03 = Row0Block03 + C[3];
|
||||
}
|
||||
|
||||
C[0] = Row0Block00;
|
||||
C[1] = Row0Block01;
|
||||
C[2] = Row0Block02;
|
||||
C[3] = Row0Block03;
|
||||
|
||||
if (ProcessTwoRows) {
|
||||
|
||||
if (!ZeroMode) {
|
||||
Row1Block00 = Row1Block00 + C[ldc];
|
||||
Row1Block01 = Row1Block01 + C[ldc + 1];
|
||||
Row1Block02 = Row1Block02 + C[ldc + 2];
|
||||
Row1Block03 = Row1Block03 + C[ldc + 3];
|
||||
}
|
||||
|
||||
C[ldc] = Row1Block00;
|
||||
C[ldc + 1] = Row1Block01;
|
||||
C[ldc + 2] = Row1Block02;
|
||||
C[ldc + 3] = Row1Block03;
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
//
|
||||
// Store the partial output block.
|
||||
//
|
||||
if ((CountN & 2) != 0) {
|
||||
|
||||
if (!ZeroMode) {
|
||||
Row0Block00 = Row0Block00 + C[0];
|
||||
Row0Block01 = Row0Block01 + C[1];
|
||||
}
|
||||
|
||||
C[0] = Row0Block00;
|
||||
C[1] = Row0Block01;
|
||||
Row0Block00 = Row0Block02;
|
||||
Row0Block01 = Row0Block03;
|
||||
|
||||
if (ProcessTwoRows) {
|
||||
|
||||
if (!ZeroMode) {
|
||||
Row1Block00 = Row1Block00 + C[ldc];
|
||||
Row1Block01 = Row1Block01 + C[ldc + 1];
|
||||
}
|
||||
|
||||
C[ldc] = Row1Block00;
|
||||
C[ldc + 1] = Row1Block01;
|
||||
Row1Block00 = Row1Block02;
|
||||
Row1Block01 = Row1Block03;
|
||||
}
|
||||
|
||||
C += 2;
|
||||
}
|
||||
|
||||
if ((CountN & 1) != 0) {
|
||||
|
||||
if (!ZeroMode) {
|
||||
Row0Block00 = Row0Block00 + C[0];
|
||||
}
|
||||
|
||||
C[0] = Row0Block00;
|
||||
|
||||
if (ProcessTwoRows) {
|
||||
|
||||
if (!ZeroMode) {
|
||||
Row1Block00 = Row1Block00 + C[ldc];
|
||||
}
|
||||
|
||||
C[ldc] = Row1Block00;
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
B += 4;
|
||||
C += 4;
|
||||
CountN -= 4;
|
||||
|
||||
countb = (countb + 1) % 4;
|
||||
if (countb == 0) {
|
||||
B += CountK * 16 - 16;
|
||||
}
|
||||
} while (CountN > 0);
|
||||
|
||||
return ProcessTwoRows ? 2 : 1;
|
||||
}
|
||||
|
||||
template<bool ZeroMode>
|
||||
size_t
|
||||
MlasSgemmKernel(
|
||||
const float* A,
|
||||
const float* B,
|
||||
float* C,
|
||||
size_t CountK,
|
||||
size_t CountM,
|
||||
size_t CountN,
|
||||
size_t lda,
|
||||
size_t ldc,
|
||||
float alpha
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A - Supplies the address of matrix A.
|
||||
|
||||
B - Supplies the address of matrix B. The matrix data has been packed using
|
||||
MlasSgemmCopyPackB or MlasSgemmTransposePackB.
|
||||
|
||||
C - Supplies the address of matrix C.
|
||||
|
||||
CountK - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
CountM - Supplies the maximum number of rows that can be processed for
|
||||
matrix A and matrix C. The actual number of rows handled for this
|
||||
invocation depends on the kernel implementation.
|
||||
|
||||
CountN - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
alpha - Supplies the scaler multiplier (see SGEMM definition).
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
{
|
||||
size_t RowsHandled;
|
||||
|
||||
if (CountM >= 2) {
|
||||
RowsHandled = MlasSgemmKernel<ZeroMode, true>(A, B, C, CountK, CountN, lda, ldc, alpha);
|
||||
} else {
|
||||
RowsHandled = MlasSgemmKernel<ZeroMode, false>(A, B, C, CountK, CountN, lda, ldc, alpha);
|
||||
}
|
||||
|
||||
return RowsHandled;
|
||||
}
|
||||
|
||||
size_t
|
||||
MLASCALL
|
||||
MlasSgemmKernelZero(
|
||||
const float* A,
|
||||
const float* B,
|
||||
float* C,
|
||||
size_t CountK,
|
||||
size_t CountM,
|
||||
size_t CountN,
|
||||
size_t lda,
|
||||
size_t ldc,
|
||||
float alpha
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A - Supplies the address of matrix A.
|
||||
|
||||
B - Supplies the address of matrix B. The matrix data has been packed using
|
||||
MlasSgemmCopyPackB or MlasSgemmTransposePackB.
|
||||
|
||||
C - Supplies the address of matrix C.
|
||||
|
||||
CountK - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
CountM - Supplies the maximum number of rows that can be processed for
|
||||
matrix A and matrix C. The actual number of rows handled for this
|
||||
invocation depends on the kernel implementation.
|
||||
|
||||
CountN - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
alpha - Supplies the scaler multiplier (see SGEMM definition).
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
{
|
||||
return MlasSgemmKernel<true>(A, B, C, CountK, CountM, CountN, lda, ldc, alpha);
|
||||
}
|
||||
|
||||
size_t
|
||||
MLASCALL
|
||||
MlasSgemmKernelAdd(
|
||||
const float* A,
|
||||
const float* B,
|
||||
float* C,
|
||||
size_t CountK,
|
||||
size_t CountM,
|
||||
size_t CountN,
|
||||
size_t lda,
|
||||
size_t ldc,
|
||||
float alpha
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A - Supplies the address of matrix A.
|
||||
|
||||
B - Supplies the address of matrix B. The matrix data has been packed using
|
||||
MlasSgemmCopyPackB or MlasSgemmTransposePackB.
|
||||
|
||||
C - Supplies the address of matrix C.
|
||||
|
||||
CountK - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
CountM - Supplies the maximum number of rows that can be processed for
|
||||
matrix A and matrix C. The actual number of rows handled for this
|
||||
invocation depends on the kernel implementation.
|
||||
|
||||
CountN - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
alpha - Supplies the scaler multiplier (see SGEMM definition).
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
{
|
||||
return MlasSgemmKernel<false>(A, B, C, CountK, CountM, CountN, lda, ldc, alpha);
|
||||
}
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemvKernelScalar.cpp
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/vector
|
||||
multiply operation (SGEMV).
|
||||
|
||||
--*/
|
||||
|
||||
#include "mlasi.h"
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasGemvFloatKernel(
|
||||
const float* A,
|
||||
const float* B,
|
||||
float* C,
|
||||
size_t CountK,
|
||||
size_t CountN,
|
||||
size_t ldb,
|
||||
bool ZeroMode
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows. This handles the special case of M=1.
|
||||
|
||||
The elements in matrix B are not transposed.
|
||||
|
||||
Arguments:
|
||||
|
||||
A - Supplies the address of matrix A.
|
||||
|
||||
B - Supplies the address of matrix B.
|
||||
|
||||
C - Supplies the address of matrix C.
|
||||
|
||||
CountK - Supplies the number of columns from matrix A and the number
|
||||
of rows from matrix B to iterate over.
|
||||
|
||||
CountN - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
ldb - Supplies the first dimension of matrix B.
|
||||
|
||||
ZeroMode - Supplies true if the output matrix must be zero initialized,
|
||||
else false if the output matrix is accumulated into.
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
{
|
||||
if (ZeroMode && CountK > 0) {
|
||||
float* c = C;
|
||||
const float* b = B;
|
||||
const float A0 = A[0];
|
||||
auto N = CountN;
|
||||
constexpr size_t kWidth = 4;
|
||||
for (; N >= kWidth; N -= kWidth) {
|
||||
c[0] = A0 * b[0];
|
||||
c[1] = A0 * b[1];
|
||||
c[2] = A0 * b[2];
|
||||
c[3] = A0 * b[3];
|
||||
c += kWidth;
|
||||
b += kWidth;
|
||||
}
|
||||
|
||||
for (; N > 0; N--) {
|
||||
c[0] = A0 * b[0];
|
||||
c++;
|
||||
b++;
|
||||
}
|
||||
A++;
|
||||
B += ldb;
|
||||
|
||||
CountK--;
|
||||
}
|
||||
|
||||
for (; CountK >= 4; CountK -= 4) {
|
||||
float* c = C;
|
||||
const float* b = B;
|
||||
const float* b2 = B + ldb * 2;
|
||||
|
||||
const float A0 = A[0];
|
||||
const float A1 = A[1];
|
||||
const float A2 = A[2];
|
||||
const float A3 = A[3];
|
||||
|
||||
constexpr size_t kWidth = 4;
|
||||
auto N = CountN;
|
||||
for (; N >= kWidth; N -= kWidth) {
|
||||
float c0 = c[0] + A0 * b[0];
|
||||
float c1 = c[1] + A0 * b[1];
|
||||
float c2 = c[2] + A0 * b[2];
|
||||
float c3 = c[3] + A0 * b[3];
|
||||
|
||||
c0 += A1 * b[ldb + 0];
|
||||
c1 += A1 * b[ldb + 1];
|
||||
c2 += A1 * b[ldb + 2];
|
||||
c3 += A1 * b[ldb + 3];
|
||||
|
||||
c0 += A2 * b2[0];
|
||||
c1 += A2 * b2[1];
|
||||
c2 += A2 * b2[2];
|
||||
c3 += A2 * b2[3];
|
||||
|
||||
c0 += A3 * b2[ldb + 0];
|
||||
c1 += A3 * b2[ldb + 1];
|
||||
c2 += A3 * b2[ldb + 2];
|
||||
c3 += A3 * b2[ldb + 3];
|
||||
|
||||
c[0] = c0;
|
||||
c[1] = c1;
|
||||
c[2] = c2;
|
||||
c[3] = c3;
|
||||
|
||||
c += kWidth;
|
||||
b += kWidth;
|
||||
b2 += kWidth;
|
||||
}
|
||||
|
||||
for (; N > 0; N--) {
|
||||
c[0] += A0 * b[0] + A1 * b[ldb] + A2 * b2[0] + A3 * b2[ldb];
|
||||
c++;
|
||||
b++;
|
||||
b2++;
|
||||
}
|
||||
|
||||
B += 4 * ldb;
|
||||
A += 4;
|
||||
}
|
||||
|
||||
for (; CountK > 0; CountK--) {
|
||||
float* c = C;
|
||||
const float* b = B;
|
||||
const float A0 = A[0];
|
||||
constexpr size_t kWidth = 4;
|
||||
auto N = CountN;
|
||||
for (; N >= kWidth; N -= kWidth) {
|
||||
c[0] += A0 * b[0];
|
||||
c[1] += A0 * b[1];
|
||||
c[2] += A0 * b[2];
|
||||
c[3] += A0 * b[3];
|
||||
|
||||
c += kWidth;
|
||||
b += kWidth;
|
||||
}
|
||||
|
||||
for (; N > 0; N--) {
|
||||
c[0] += A0 * b[0];
|
||||
c++;
|
||||
b++;
|
||||
}
|
||||
B += ldb;
|
||||
A++;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user