2
* Automatically Tuned Linear Algebra Software v3.8.3
3
* (C) Copyright 2007 R. Clint Whaley
5
* Redistribution and use in source and binary forms, with or without
6
* modification, are permitted provided that the following conditions
8
* 1. Redistributions of source code must retain the above copyright
9
* notice, this list of conditions and the following disclaimer.
10
* 2. Redistributions in binary form must reproduce the above copyright
11
* notice, this list of conditions, and the following disclaimer in the
12
* documentation and/or other materials provided with the distribution.
13
* 3. The name of the ATLAS group or the names of its contributers may
14
* not be used to endorse or promote products derived from this
15
* software without specific written permission.
17
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ATLAS GROUP OR ITS CONTRIBUTORS
21
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
* POSSIBILITY OF SUCH DAMAGE.
30
#include "atlas_misc.h"
31
#include "atlas_lvl3.h"
33
void Mjoin(PATL,mmK)(int M, /* true # of rows in row-panel, M <= MB */
34
int m, /* # of rows to operate on, m >= M */
35
int N, /* true # of cols in col-panel, N < = NB */
36
int n, /* # of cols to operate on, n >= N */
37
int nblk, /* # of blocks in K dimension */
38
int kr, /* kr = K - nKb*KB; */
39
int KR, /* 0 : do not do full KB-call to avoid cleanup */
40
const SCALAR alphaA, /* alpha to apply during A copy */
41
const SCALAR alphaB, /* alpha to apply during B copy */
42
const SCALAR beta, /* beta to apply to C */
43
const TYPE *A, /* array to copy from, NULL if already cp */
44
const int lda, /* leading dimension of A */
45
const int incA, /* inc to next blk in A */
46
TYPE *pA, /* wrkspace to copy A to */
47
const int incAW, /* 0 : keep using same KBxMB space */
48
const TYPE *B, /* array to copy from, NULL if already cp */
49
const int ldb, /* leading dimension of B */
50
const int incB, /* inc to next blk in B */
51
TYPE *pB, /* wrkspace to copy B to */
52
const int incBW, /* 0 : keep using same KBxNB space */
53
TYPE *C, /* output matrix */
55
MAT2BLK2 A2blk, /* rout to copy A */
56
MAT2BLK2 B2blk, /* rout to copy B */
57
NBMM0 NBmm0, /* rout to do first mul (applies beta) */
58
NBMM0 NBmm1) /* rout to do later muls (beta=1) */
60
* Performs a K-inner-loop matmul, while copying A & B if necessary.
61
* If M > m, we are doing extra flops so we don't call cleanup (same for N)
67
if (B) { B2blk(KB, N, alphaB, B, ldb, pB, KB); B += incB; }
68
if (A) { A2blk(KB, M, alphaA, A, lda, pA, KB); A += incA; }
69
NBmm0(m, n, KB, ATL_rone, pA, KB, pB, KB, beta, C, ldc);
70
pA += incAW; pB += incBW;
71
for (k = nblk-1; k; k--)
73
if (B) { B2blk(KB, N, alphaB, B, ldb, pB, KB); B += incB; }
74
if (A) { A2blk(KB, M, alphaA, A, lda, pA, KB); A += incA; }
75
NBmm1(m, n, KB, ATL_rone, pA, KB, pB, KB, ATL_rone, C, ldc);
76
pA += incAW; pB += incBW;
79
if (kr) /* need to cleanup K loop */
85
B2blk(kr, N, alphaB, B, ldb, pB, KB);
86
Mjoin(PATL,gezero)(KB-kr, n, pB+kr, KB);
90
A2blk(kr, M, alphaA, A, lda, pA, KB);
91
Mjoin(PATL,gezero)(KB-kr, m, pA+kr, KB);
94
NBmm1(m, n, KB, ATL_rone, pA, KB, pB, KB, ATL_rone, C, ldc);
96
NBmm0(m, n, KB, ATL_rone, pA, KB, pB, KB, beta, C, ldc);
100
if (B) B2blk(kr, N, alphaB, B, ldb, pB, kr);
101
if (A) A2blk(kr, M, alphaA, A, lda, pA, kr);
102
Mjoin(PATL,pKBmm)(M, N, kr, ATL_rone, pA, kr, pB, kr,
103
nblk ? ATL_rone : beta, C, ldc);
added
removed
src/blas/gemm/ATL_mmK.c
