~ubuntu-branches/ubuntu/vivid/atlas/vivid

(const int M, const int N, const int K, const TYPE alpha, const TYPE *A, const int lda, const TYPE *B, const int ldb, const TYPE beta, TYPE *C, const int ldc)

* matmul with TA=T, TB=N, MB=0, NB=0, KB=0,

* lda=0, ldb=0, ldc=0, mu=1, nu=1, ku=1

{

#define Mb M

#define Nb N

#define Kb K

const TYPE *stM = A + (lda*Mb);

const TYPE *stN = B + (ldb*Nb);

#define incAk 1

const int incAm = ((lda) - Kb), incAn = -(Mb*lda);

#define incBk 1

const int incBm = -(Kb), incBn = (ldb);

#ifdef TREAL

#define incCm 1

const int incCn = (ldc) - (Mb);

#else

#define incCm 2

const int incCn = ((ldc) - (Mb))<<1;

#endif

TYPE *pC0=C;

const TYPE *pA0=A;

const TYPE *pB0=B;

do /* N-loop */

{

do /* M-loop */

{

#ifdef BETA0

rC0_0 = ATL_rzero;

#elif defined(BETAX)

rC0_0 = beta * *pC0;

#else

rC0_0 = *pC0;

#endif

for (k=K; k; k--) /* easy loop to unroll */

{

rA0 = *pA0;

rB0 = *pB0;

rC0_0 += rA0 * rB0;

pA0 += incAk;

pB0 += incBk;

}

*pC0 = rC0_0;

pC0 += incCm;

pA0 += incAm;

pB0 += incBm;

}

while(pA0 != stM);

pC0 += incCn;

pA0 += incAn;

pB0 += incBn;

}

while(pB0 != stN);

}

#ifdef incAm

#undef incAm

#endif

#ifdef incAn

#undef incAn

#endif

#ifdef incAk

#undef incAk

#endif

#ifdef incBm

#undef incBm

#endif

#ifdef incBn

#undef incBn

#endif

#ifdef incBk

#undef incBk

#endif

#ifdef incCm

#undef incCm

#endif

#ifdef incCn

#undef incCn

#endif

#ifdef incCk

#undef incCk

#endif

#ifdef Mb

#undef Mb

#endif

#ifdef Nb

#undef Nb

#endif

#ifdef Kb

#undef Kb

#endif

100

static void ATL_dJIK0x0x0TN6x1x1_a1_b1

101

(const int M, const int N, const int K, const TYPE alpha, const TYPE *A, const int lda, const TYPE *B, const int ldb, const TYPE beta, TYPE *C, const int ldc)

102

103

* matmul with TA=T, TB=N, MB=0, NB=0, KB=0,

104

* lda=0, ldb=0, ldc=0, mu=6, nu=1, ku=1

105

106

{

107

const int Mb = (M/6)*6;

108

#define Nb N

109

#define Kb K

110

const TYPE *ca=A, *cb=B;

111

TYPE *cc=C;

112

const TYPE *stM = A + (lda*Mb);

113

const TYPE *stN = B + (ldb*Nb);

114

#define incAk 1

115

const int incAm = ((((lda) << 2)+((lda) << 1)) - Kb), incAn = -(Mb*lda);

116

#define incBk 1

117

const int incBm = -(Kb), incBn = (ldb);

118

#ifdef TREAL

119

#define incCm 6

120

const int incCn = (ldc) - (Mb);

121

#else

122

#define incCm 12

123

const int incCn = ((ldc) - (Mb))<<1;

124

#endif

125

TYPE *pC0=C;

126

const TYPE *pA0=A, *pA1=pA0+(lda), *pA2=pA1+(lda), *pA3=pA2+(lda), *pA4=pA3+(lda), *pA5=pA4+(lda);

127

const TYPE *pB0=B;

128

129

130

131

132

if (pA0 != stM)

133

{

134

do /* N-loop */

135

{

136

do /* M-loop */

137

{

138

#ifdef BETA0

139

rC0_0 = rC1_0 = rC2_0 = rC3_0 = rC4_0 = rC5_0 = ATL_rzero;

140

#else

141

rC0_0 = *pC0;

142

#ifdef TREAL

143

rC1_0 = pC0[1]; rC2_0 = pC0[2]; rC3_0 = pC0[3];

144

rC4_0 = pC0[4]; rC5_0 = pC0[5];

145

#else

146

rC1_0 = pC0[2]; rC2_0 = pC0[4]; rC3_0 = pC0[6];

147

rC4_0 = pC0[8]; rC5_0 = pC0[10];

148

#endif

149

#ifdef BETAX

150

rA5 = beta;

151

rC0_0 *= rA5; rC1_0 *= rA5; rC2_0 *= rA5;

152

rC3_0 *= rA5; rC4_0 *= rA5; rC5_0 *= rA5;

153

#endif

154

#endif

155

for (k=K; k; k--) /* easy loop to unroll */

156

{

157

rA0 = *pA0;

158

rB0 = *pB0;

159

rA1 = *pA1;

160

rA2 = *pA2;

161

rA3 = *pA3;

162

rA4 = *pA4;

163

rA5 = *pA5;

164

rC0_0 += rA0 * rB0;

165

rC1_0 += rA1 * rB0;

166

rC2_0 += rA2 * rB0;

167

rC3_0 += rA3 * rB0;

168

rC4_0 += rA4 * rB0;

169

rC5_0 += rA5 * rB0;

170

pA0 += incAk;

171

pA1 += incAk;

172

pA2 += incAk;

173

pA3 += incAk;

174

pA4 += incAk;

175

pA5 += incAk;

176

pB0 += incBk;

177

}

178

*pC0 = rC0_0;

179

#ifdef TREAL

180

pC0[1] = rC1_0; pC0[2] = rC2_0; pC0[3] = rC3_0;

181

pC0[4] = rC4_0; pC0[5] = rC5_0;

182

#else

183

pC0[2] = rC1_0; pC0[4] = rC2_0; pC0[6] = rC3_0;

184

pC0[8] = rC4_0; pC0[10] = rC5_0;

185

#endif

186

pC0 += incCm;

187

pA0 += incAm;

188

pA1 += incAm;

189

pA2 += incAm;

190

pA3 += incAm;

191

pA4 += incAm;

192

pA5 += incAm;

193

pB0 += incBm;

194

}

195

while(pA0 != stM);

196

pC0 += incCn;

197

pA0 += incAn;

198

pA1 += incAn;

199

pA2 += incAn;

200

pA3 += incAn;

201

pA4 += incAn;

202

pA5 += incAn;

203

pB0 += incBn;

204

}

205

while(pB0 != stN);

206

}

207

if (k=M-Mb)

208

ATL_dJIK0x0x0TN1x1x1_a1_b1(k, N, K, alpha, ca + (Mb*lda), lda, cb, ldb, beta, cc + (Mb SHIFT), ldc);

209

}

210

#ifdef incAm

211

#undef incAm

212

#endif

213

#ifdef incAn

214

#undef incAn

215

#endif

216

#ifdef incAk

217

#undef incAk

218

#endif

219

#ifdef incBm

220

#undef incBm

221

#endif

222

#ifdef incBn

223

#undef incBn

224

#endif

225

#ifdef incBk

226

#undef incBk

227

#endif

228

#ifdef incCm

229

#undef incCm

230

#endif

231

#ifdef incCn

232

#undef incCn

233

#endif

234

#ifdef incCk

235

#undef incCk

236

#endif

237

#ifdef Mb

238

#undef Mb

239

#endif

240

#ifdef Nb

241

#undef Nb

242

#endif

243

#ifdef Kb

244

#undef Kb

245

#endif

246

static void ATL_dJIK0x0x0TN1x8x1_a1_b1

247

(const int M, const int N, const int K, const TYPE alpha, const TYPE *A, const int lda, const TYPE *B, const int ldb, const TYPE beta, TYPE *C, const int ldc)

248

249

* matmul with TA=T, TB=N, MB=0, NB=0, KB=0,

250

* lda=0, ldb=0, ldc=0, mu=1, nu=8, ku=1

251

252

{

253

#define Mb M

254

const int Nb = (N>>3)<<3;

255

#define Kb K

256

const TYPE *ca=A, *cb=B;

257

TYPE *cc=C;

258

const TYPE *stM = A + (lda*Mb);

259

const TYPE *stN = B + (ldb*Nb);

260

#define incAk 1

261

const int incAm = ((lda) - Kb), incAn = -(Mb*lda);

262

#define incBk 1

263

const int incBm = -(Kb), incBn = (((ldb) << 3));

264

#ifdef TREAL

265

#define incCm 1

266

#define ldc2 ldc

267

const int incCn = (((ldc) << 3)) - (Mb);

268

#else

269

#define incCm 2

270

const int incCn = ((((ldc) << 3)) - (Mb))<<1, ldc2 = ldc<<1;

271

#endif

272

TYPE *pC0=C, *pC1=pC0+(ldc2), *pC2=pC1+(ldc2), *pC3=pC2+(ldc2),

273

*pC4=pC3+(ldc2), *pC5=pC4+(ldc2), *pC6=pC5+(ldc2), *pC7=pC6+(ldc2);

274

const TYPE *pA0=A;

275

const TYPE *pB0=B, *pB1=pB0+(ldb), *pB2=pB1+(ldb), *pB3=pB2+(ldb), *pB4=pB3+(ldb), *pB5=pB4+(ldb), *pB6=pB5+(ldb), *pB7=pB6+(ldb);

276

277

278

279

280

if (pB0 != stN)

281

{

282

do /* N-loop */

283

{

284

do /* M-loop */

285

{

286

#ifdef BETA0

287

rC0_0 = rC0_1 = rC0_2 = rC0_3 = rC0_4 = rC0_5 =

288

rC0_6 = rC0_7 = ATL_rzero;

289

#else

290

rC0_0 = *pC0; rC0_1 = *pC1; rC0_2 = *pC2;

291

rC0_3 = *pC3; rC0_4 = *pC4; rC0_5 = *pC5;

292

rC0_6 = *pC6; rC0_7 = *pC7;

293

#ifdef BETAX

294

rB7 = beta;

295

rC0_0 *= rB7; rC0_1 *= rB7; rC0_2 *= rB7; rC0_3 *= rB7;

296

rC0_4 *= rB7; rC0_5 *= rB7; rC0_6 *= rB7; rC0_7 *= rB7;

297

#endif

298

#endif

299

for (k=K; k; k--) /* easy loop to unroll */

300

{

301

rA0 = *pA0;

302

rB0 = *pB0;

303

rB1 = *pB1;

304

rB2 = *pB2;

305

rB3 = *pB3;

306

rB4 = *pB4;

307

rB5 = *pB5;

308

rB6 = *pB6;

309

rB7 = *pB7;

310

rC0_0 += rA0 * rB0;

311

rC0_1 += rA0 * rB1;

312

rC0_2 += rA0 * rB2;

313

rC0_3 += rA0 * rB3;

314

rC0_4 += rA0 * rB4;

315

rC0_5 += rA0 * rB5;

316

rC0_6 += rA0 * rB6;

317

rC0_7 += rA0 * rB7;

318

pA0 += incAk;

319

pB0 += incBk;

320

pB1 += incBk;

321

pB2 += incBk;

322

pB3 += incBk;

323

pB4 += incBk;

324

pB5 += incBk;

325

pB6 += incBk;

326

pB7 += incBk;

327

}

328

*pC0 = rC0_0;

329

*pC1 = rC0_1;

330

*pC2 = rC0_2;

331

*pC3 = rC0_3;

332

*pC4 = rC0_4;

333

*pC5 = rC0_5;

334

*pC6 = rC0_6;

335

*pC7 = rC0_7;

336

pC0 += incCm;

337

pC1 += incCm;

338

pC2 += incCm;

339

pC3 += incCm;

340

pC4 += incCm;

341

pC5 += incCm;

342

pC6 += incCm;

343

pC7 += incCm;

344

pA0 += incAm;

345

pB0 += incBm;

346

pB1 += incBm;

347

pB2 += incBm;

348

pB3 += incBm;

349

pB4 += incBm;

350

pB5 += incBm;

351

pB6 += incBm;

352

pB7 += incBm;

353

}

354

while(pA0 != stM);

355

pC0 += incCn;

356

pC1 += incCn;

357

pC2 += incCn;

358

pC3 += incCn;

359

pC4 += incCn;

360

pC5 += incCn;

361

pC6 += incCn;

362

pC7 += incCn;

363

pA0 += incAn;

364

pB0 += incBn;

365

pB1 += incBn;

366

pB2 += incBn;

367

pB3 += incBn;

368

pB4 += incBn;

369

pB5 += incBn;

370

pB6 += incBn;

371

pB7 += incBn;

372

}

373

while(pB0 != stN);

374

}

375

if (k=N-Nb)

376

ATL_dJIK0x0x0TN1x1x1_a1_b1(M, k, K, alpha, ca, lda, cb + (Nb*ldb), ldb, beta, cc + (Nb*ldc2), ldc);

377

}

378

#ifdef ldc2

379

#undef ldc2

380

#endif

381

#ifdef incAm

382

#undef incAm

383

#endif

384

#ifdef incAn

385

#undef incAn

386

#endif

387

#ifdef incAk

388

#undef incAk

389

#endif

390

#ifdef incBm

391

#undef incBm

392

#endif

393

#ifdef incBn

394

#undef incBn

395

#endif

396

#ifdef incBk

397

#undef incBk

398

#endif

399

#ifdef incCm

400

#undef incCm

401

#endif

402

#ifdef incCn

403

#undef incCn

404

#endif

405

#ifdef incCk

406

#undef incCk

407

#endif

408

#ifdef Mb

409

#undef Mb

410

#endif

411

#ifdef Nb

412

#undef Nb

413

#endif

414

#ifdef Kb

415

#undef Kb

416

#endif

417

void ATL_USERMM

418

(const int M, const int N, const int K, const TYPE alpha, const TYPE *A, const int lda, const TYPE *B, const int ldb, const TYPE beta, TYPE *C, const int ldc)

419

420

* matmul with TA=T, TB=N, MB=0, NB=0, KB=0,

421

* lda=0, ldb=0, ldc=0, mu=6, nu=8, ku=2

422

423

{

424

const int Mb = (M/6)*6;

425

const int Nb = (N>>3)<<3;

426

const int Kb = (K>>1)<<1;

427

#if K == 0

428

const int Kstart = (K-1)>>1;

429

#else

430

const int Kstart = (K>>1) - 1;

431

#endif

432

const TYPE *stM = A + (lda*Mb);

433

const TYPE *stN = B + (ldb*Nb), *ca=A, *cb=B;

434

TYPE *cc=C;

435

#define incAk 2

436

const int incAm = ((((lda) << 2)+((lda) << 1)) - Kb), incAn = -(Mb*lda);

437

#define incBk 2

438

const int incBm = -(Kb), incBn = (((ldb) << 3));

439

#ifdef TREAL

440

#define incCm 6

441

#define ldc2 ldc

442

const int incCn = (((ldc) << 3)) - (Mb);

443

#else

444

#define incCm 12

445

const int incCn = ((((ldc) << 3)) - (Mb))<<1, ldc2=ldc<<1;

446

#endif

447

TYPE *pC0=C, *pC1=pC0+(ldc2), *pC2=pC1+(ldc2), *pC3=pC2+(ldc2),

448

*pC4=pC3+(ldc2), *pC5=pC4+(ldc2), *pC6=pC5+(ldc2), *pC7=pC6+(ldc2);

449

const TYPE *pA0=A, *pA1=pA0+(lda), *pA2=pA1+(lda), *pA3=pA2+(lda),

450

*pA4=pA3+(lda), *pA5=pA4+(lda);

451

const TYPE *pB0=B, *pB1=pB0+(ldb), *pB2=pB1+(ldb), *pB3=pB2+(ldb),

452

*pB4=pB3+(ldb), *pB5=pB4+(ldb), *pB6=pB5+(ldb), *pB7=pB6+(ldb);

453

454

#ifdef BETAX

455

TYPE *bp = (TYPE *) β

456

#endif

457

458

459

460

461

462

rC0_1, rC1_1, rC2_1, rC3_1, rC4_1, rC5_1,

463

rC0_2, rC1_2, rC2_2, rC3_2, rC4_2, rC5_2,

464

rC0_3, rC1_3, rC2_3, rC3_3, rC4_3, rC5_3,

465

rC0_4, rC1_4, rC2_4, rC3_4, rC4_4, rC5_4,

466

rC0_5, rC1_5, rC2_5, rC3_5, rC4_5, rC5_5,

467

rC0_6, rC1_6, rC2_6, rC3_6, rC4_6, rC5_6,

468

rC0_7, rC1_7, rC2_7, rC3_7, rC4_7, rC5_7;

469

if (pA0 != stM && pB0 != stN)

470

{

471

do /* N-loop */

472

{

473

do /* M-loop */

474

{

475

#ifdef BETA0

476

rC0_0 = rC1_0 = rC2_0 = rC3_0 = rC4_0 = rC5_0 =

477

rC0_1 = rC1_1 = rC2_1 = rC3_1 = rC4_1 = rC5_1 =

478

rC0_2 = rC1_2 = rC2_2 = rC3_2 = rC4_2 = rC5_2 =

479

rC0_3 = rC1_3 = rC2_3 = rC3_3 = rC4_3 = rC5_3 =

480

rC0_4 = rC1_4 = rC2_4 = rC3_4 = rC4_4 = rC5_4 =

481

rC0_5 = rC1_5 = rC2_5 = rC3_5 = rC4_5 = rC5_5 =

482

rC0_6 = rC1_6 = rC2_6 = rC3_6 = rC4_6 = rC5_6 =

483

rC0_7 = rC1_7 = rC2_7 = rC3_7 = rC4_7 = rC5_7 = ATL_rzero;

484

#else

485

#ifdef TREAL

486

rC0_0 = *pC0; rC1_0 = pC0[1]; rC2_0 = pC0[ 2];

487

rC3_0 = pC0[3]; rC4_0 = pC0[4]; rC5_0 = pC0[ 5];

488

rC0_1 = *pC1; rC1_1 = pC1[1]; rC2_1 = pC1[ 2];

489

rC3_1 = pC1[3]; rC4_1 = pC1[4]; rC5_1 = pC1[ 5];

490

rC0_2 = *pC2; rC1_2 = pC2[1]; rC2_2 = pC2[ 2];

491

rC3_2 = pC2[3]; rC4_2 = pC2[4]; rC5_2 = pC2[ 5];

492

rC0_3 = *pC3; rC1_3 = pC3[1]; rC2_3 = pC3[ 2];

493

rC3_3 = pC3[3]; rC4_3 = pC3[4]; rC5_3 = pC3[ 5];

494

rC0_4 = *pC4; rC1_4 = pC4[1]; rC2_4 = pC4[ 2];

495

rC3_4 = pC4[3]; rC4_4 = pC4[4]; rC5_4 = pC4[ 5];

496

rC0_5 = *pC5; rC1_5 = pC5[1]; rC2_5 = pC5[ 2];

497

rC3_5 = pC5[3]; rC4_5 = pC5[4]; rC5_5 = pC5[ 5];

498

rC0_6 = *pC6; rC1_6 = pC6[1]; rC2_6 = pC6[ 2];

499

rC3_6 = pC6[3]; rC4_6 = pC6[4]; rC5_6 = pC6[ 5];

500

rC0_7 = *pC7; rC1_7 = pC7[1]; rC2_7 = pC7[ 2];

501

rC3_7 = pC7[3]; rC4_7 = pC7[4]; rC5_7 = pC7[ 5];

502

#else

503

rC0_0 = *pC0; rC1_0 = pC0[2]; rC2_0 = pC0[ 4];

504

rC3_0 = pC0[6]; rC4_0 = pC0[8]; rC5_0 = pC0[10];

505

rC0_1 = *pC1; rC1_1 = pC1[2]; rC2_1 = pC1[ 4];

506

rC3_1 = pC1[6]; rC4_1 = pC1[8]; rC5_1 = pC1[10];

507

rC0_2 = *pC2; rC1_2 = pC2[2]; rC2_2 = pC2[ 4];

508

rC3_2 = pC2[6]; rC4_2 = pC2[8]; rC5_2 = pC2[10];

509

rC0_3 = *pC3; rC1_3 = pC3[2]; rC2_3 = pC3[ 4];

510

rC3_3 = pC3[6]; rC4_3 = pC3[8]; rC5_3 = pC3[10];

511

rC0_4 = *pC4; rC1_4 = pC4[2]; rC2_4 = pC4[ 4];

512

rC3_4 = pC4[6]; rC4_4 = pC4[8]; rC5_4 = pC4[10];

513

rC0_5 = *pC5; rC1_5 = pC5[2]; rC2_5 = pC5[ 4];

514

rC3_5 = pC5[6]; rC4_5 = pC5[8]; rC5_5 = pC5[10];

515

rC0_6 = *pC6; rC1_6 = pC6[2]; rC2_6 = pC6[ 4];

516

rC3_6 = pC6[6]; rC4_6 = pC6[8]; rC5_6 = pC6[10];

517

rC0_7 = *pC7; rC1_7 = pC7[2]; rC2_7 = pC7[ 4];

518

rC3_7 = pC7[6]; rC4_7 = pC7[8]; rC5_7 = pC7[10];

519

#endif

520

#ifdef BETAX

521

rb7 = *bp;

522

rC0_0 *= rb7; rC1_0 *= rb7; rC2_0 *= rb7;

523

rC3_0 *= rb7; rC4_0 *= rb7; rC5_0 *= rb7;

524

rC0_1 *= rb7; rC1_1 *= rb7; rC2_1 *= rb7;

525

rC3_1 *= rb7; rC4_1 *= rb7; rC5_1 *= rb7;

526

rC0_2 *= rb7; rC1_2 *= rb7; rC2_2 *= rb7;

527

rC3_2 *= rb7; rC4_2 *= rb7; rC5_2 *= rb7;

528

rC0_3 *= rb7; rC1_3 *= rb7; rC2_3 *= rb7;

529

rC3_3 *= rb7; rC4_3 *= rb7; rC5_3 *= rb7;

530

rC0_4 *= rb7; rC1_4 *= rb7; rC2_4 *= rb7;

531

rC3_4 *= rb7; rC4_4 *= rb7; rC5_4 *= rb7;

532

rC0_5 *= rb7; rC1_5 *= rb7; rC2_5 *= rb7;

533

rC3_5 *= rb7; rC4_5 *= rb7; rC5_5 *= rb7;

534

rC0_6 *= rb7; rC1_6 *= rb7; rC2_6 *= rb7;

535

rC3_6 *= rb7; rC4_6 *= rb7; rC5_6 *= rb7;

536

rC0_7 *= rb7; rC1_7 *= rb7; rC2_7 *= rb7;

537

rC3_7 *= rb7; rC4_7 *= rb7; rC5_7 *= rb7;

538

#endif

539

#endif

540

rA0 = *pA0++; rA1 = *pA1++; rA2 = *pA2++; rA3 = *pA3++;

541

rA4 = *pA4++; rA5 = *pA5++;

542

rB0 = *pB0++; rB1 = *pB1++; rB2 = *pB2++; rB3 = *pB3++;

543

rB4 = *pB4++; rB5 = *pB5++; rB6 = *pB6++; rB7 = *pB7++;

544

for (k=Kstart; k; k--) /* easy loop to unroll */

545

{

546

rC0_0 += rA0 * rB0;

547

rC1_0 += rA1 * rB0;

548

rC2_0 += rA2 * rB0;

549

rC3_0 += rA3 * rB0;

550

rC4_0 += rA4 * rB0; rb0 = *pB0++;

551

rC5_0 += rA5 * rB0;

552

rC0_1 += rA0 * rB1;

553

rC1_1 += rA1 * rB1; ra0 = *pA0++;

554

rC2_1 += rA2 * rB1;

555

rC3_1 += rA3 * rB1;

556

rC4_1 += rA4 * rB1; ra1 = *pA1++;

557

rC5_1 += rA5 * rB1;

558

rC0_2 += rA0 * rB2;

559

rC1_2 += rA1 * rB2; ra2 = *pA2++;

560

rC2_2 += rA2 * rB2;

561

rC3_2 += rA3 * rB2;

562

rC4_2 += rA4 * rB2; ra3 = *pA3++;

563

rC5_2 += rA5 * rB2;

564

rC0_3 += rA0 * rB3;

565

rC1_3 += rA1 * rB3; ra4 = *pA4++;

566

rC2_3 += rA2 * rB3;

567

rC3_3 += rA3 * rB3;

568

rC4_3 += rA4 * rB3; ra5 = *pA5++;

569

rC5_3 += rA5 * rB3;

570

rC0_4 += rA0 * rB4;

571

rC1_4 += rA1 * rB4; rb1 = *pB1++;

572

rC2_4 += rA2 * rB4;

573

rC3_4 += rA3 * rB4;

574

rC4_4 += rA4 * rB4; rb2 = *pB2++;

575

rC5_4 += rA5 * rB4;

576

rC0_5 += rA0 * rB5;

577

rC1_5 += rA1 * rB5; rb3 = *pB3++;

578

rC2_5 += rA2 * rB5;

579

rC3_5 += rA3 * rB5;

580

rC4_5 += rA4 * rB5; rb4 = *pB4++;

581

rC5_5 += rA5 * rB5;

582

rC0_6 += rA0 * rB6;

583

rC1_6 += rA1 * rB6; rb5 = *pB5++;

584

rC2_6 += rA2 * rB6;

585

rC3_6 += rA3 * rB6;

586

rC4_6 += rA4 * rB6; rb6 = *pB6++;

587

rC5_6 += rA5 * rB6;

588

rC0_7 += rA0 * rB7;

589

rC1_7 += rA1 * rB7; rb7 = *pB7++;

590

rC2_7 += rA2 * rB7;

591

rC3_7 += rA3 * rB7;

592

rC4_7 += rA4 * rB7; rB0 = *pB0++;

593

rC5_7 += rA5 * rB7;

594

595

rC0_0 += ra0 * rb0;

596

rC1_0 += ra1 * rb0; rA0 = *pA0++;

597

rC2_0 += ra2 * rb0;

598

rC3_0 += ra3 * rb0;

599

rC4_0 += ra4 * rb0; rA1 = *pA1++;

600

rC5_0 += ra5 * rb0;

601

rC0_1 += ra0 * rb1;

602

rC1_1 += ra1 * rb1; rA2 = *pA2++;

603

rC2_1 += ra2 * rb1;

604

rC3_1 += ra3 * rb1;

605

rC4_1 += ra4 * rb1; rA3 = *pA3++;

606

rC5_1 += ra5 * rb1;

607

rC0_2 += ra0 * rb2;

608

rC1_2 += ra1 * rb2; rA4 = *pA4++;

609

rC2_2 += ra2 * rb2;

610

rC3_2 += ra3 * rb2;

611

rC4_2 += ra4 * rb2; rA5 = *pA5++;

612

rC5_2 += ra5 * rb2;

613

rC0_3 += ra0 * rb3;

614

rC1_3 += ra1 * rb3; rB1 = *pB1++;

615

rC2_3 += ra2 * rb3;

616

rC3_3 += ra3 * rb3;

617

rC4_3 += ra4 * rb3; rB2 = *pB2++;

618

rC5_3 += ra5 * rb3;

619

rC0_4 += ra0 * rb4;

620

rC1_4 += ra1 * rb4; rB3 = *pB3++;

621

rC2_4 += ra2 * rb4;

622

rC3_4 += ra3 * rb4;

623

rC4_4 += ra4 * rb4; rB4 = *pB4++;

624

rC5_4 += ra5 * rb4;

625

rC0_5 += ra0 * rb5;

626

rC1_5 += ra1 * rb5; rB5 = *pB5++;

627

rC2_5 += ra2 * rb5;

628

rC3_5 += ra3 * rb5;

629

rC4_5 += ra4 * rb5; rB6 = *pB6++;

630

rC5_5 += ra5 * rb5;

631

rC0_6 += ra0 * rb6;

632

rC1_6 += ra1 * rb6;

633

rC2_6 += ra2 * rb6; rB7 = *pB7++;

634

rC3_6 += ra3 * rb6;

635

rC4_6 += ra4 * rb6;

636

rC5_6 += ra5 * rb6;

637

rC0_7 += ra0 * rb7;

638

rC1_7 += ra1 * rb7;

639

rC2_7 += ra2 * rb7;

640

rC3_7 += ra3 * rb7;

641

rC4_7 += ra4 * rb7;

642

rC5_7 += ra5 * rb7;

643

}

644

645

#if KB == 0

646

if (K != Kb)

647

{

648

rC0_0 += rA0 * rB0;

649

rC1_0 += rA1 * rB0; pA0--;

650

rC2_0 += rA2 * rB0;

651

rC3_0 += rA3 * rB0; pA1--;

652

rC4_0 += rA4 * rB0;

653

rC5_0 += rA5 * rB0; pA2--;

654

rC0_1 += rA0 * rB1;

655

rC1_1 += rA1 * rB1; pA3--;

656

rC2_1 += rA2 * rB1;

657

rC3_1 += rA3 * rB1; pA4--;

658

rC4_1 += rA4 * rB1;

659

rC5_1 += rA5 * rB1; pA5--;

660

rC0_2 += rA0 * rB2;

661

rC1_2 += rA1 * rB2; pB0--;

662

rC2_2 += rA2 * rB2;

663

rC3_2 += rA3 * rB2; pB1--;

664

rC4_2 += rA4 * rB2;

665

rC5_2 += rA5 * rB2; pB2--;

666

rC0_3 += rA0 * rB3;

667

rC1_3 += rA1 * rB3; pB3--;

668

rC2_3 += rA2 * rB3;

669

rC3_3 += rA3 * rB3; pB4--;

670

rC4_3 += rA4 * rB3;

671

rC5_3 += rA5 * rB3; pB5--;

672

rC0_4 += rA0 * rB4;

673

rC1_4 += rA1 * rB4; pB6--;

674

rC2_4 += rA2 * rB4;

675

rC3_4 += rA3 * rB4; pB7--;

676

rC4_4 += rA4 * rB4;

677

rC5_4 += rA5 * rB4;

678

rC0_5 += rA0 * rB5;

679

rC1_5 += rA1 * rB5;

680

rC2_5 += rA2 * rB5;

681

rC3_5 += rA3 * rB5;

682

rC4_5 += rA4 * rB5;

683

rC5_5 += rA5 * rB5;

684

rC0_6 += rA0 * rB6;

685

rC1_6 += rA1 * rB6;

686

rC2_6 += rA2 * rB6;

687

rC3_6 += rA3 * rB6;

688

rC4_6 += rA4 * rB6;

689

rC5_6 += rA5 * rB6;

690

rC0_7 += rA0 * rB7;

691

rC1_7 += rA1 * rB7;

692

rC2_7 += rA2 * rB7;

693

rC3_7 += rA3 * rB7;

694

rC4_7 += rA4 * rB7;

695

rC5_7 += rA5 * rB7;

696

}

697

else

698

{

699

#endif

700

rC0_0 += rA0 * rB0;

701

rC1_0 += rA1 * rB0;

702

rC2_0 += rA2 * rB0;

703

rC3_0 += rA3 * rB0;

704

rC4_0 += rA4 * rB0; rb0 = *pB0++;

705

rC5_0 += rA5 * rB0;

706

rC0_1 += rA0 * rB1;

707

rC1_1 += rA1 * rB1; ra0 = *pA0++;

708

rC2_1 += rA2 * rB1;

709

rC3_1 += rA3 * rB1;

710

rC4_1 += rA4 * rB1; ra1 = *pA1++;

711

rC5_1 += rA5 * rB1;

712

rC0_2 += rA0 * rB2;

713

rC1_2 += rA1 * rB2; ra2 = *pA2++;

714

rC2_2 += rA2 * rB2;

715

rC3_2 += rA3 * rB2;

716

rC4_2 += rA4 * rB2; ra3 = *pA3++;

717

rC5_2 += rA5 * rB2;

718

rC0_3 += rA0 * rB3;

719

rC1_3 += rA1 * rB3; ra4 = *pA4++;

720

rC2_3 += rA2 * rB3;

721

rC3_3 += rA3 * rB3;

722

rC4_3 += rA4 * rB3; ra5 = *pA5++;

723

rC5_3 += rA5 * rB3;

724

rC0_4 += rA0 * rB4;

725

rC1_4 += rA1 * rB4; rb1 = *pB1++;

726

rC2_4 += rA2 * rB4;

727

rC3_4 += rA3 * rB4;

728

rC4_4 += rA4 * rB4; rb2 = *pB2++;

729

rC5_4 += rA5 * rB4;

730

rC0_5 += rA0 * rB5;

731

rC1_5 += rA1 * rB5; rb3 = *pB3++;

732

rC2_5 += rA2 * rB5;

733

rC3_5 += rA3 * rB5;

734

rC4_5 += rA4 * rB5; rb4 = *pB4++;

735

rC5_5 += rA5 * rB5;

736

rC0_6 += rA0 * rB6;

737

rC1_6 += rA1 * rB6; rb5 = *pB5++;

738

rC2_6 += rA2 * rB6;

739

rC3_6 += rA3 * rB6;

740

rC4_6 += rA4 * rB6; rb6 = *pB6++;

741

rC5_6 += rA5 * rB6;

742

rC0_7 += rA0 * rB7;

743

rC1_7 += rA1 * rB7; rb7 = *pB7++;

744

rC2_7 += rA2 * rB7;

745

rC3_7 += rA3 * rB7;

746

rC4_7 += rA4 * rB7;

747

rC5_7 += rA5 * rB7;

748

749

rC0_0 += ra0 * rb0;

750

rC1_0 += ra1 * rb0;

751

rC2_0 += ra2 * rb0;

752

rC3_0 += ra3 * rb0;

753

rC4_0 += ra4 * rb0;

754

rC5_0 += ra5 * rb0;

755

rC0_1 += ra0 * rb1;

756

rC1_1 += ra1 * rb1;

757

rC2_1 += ra2 * rb1;

758

rC3_1 += ra3 * rb1;

759

rC4_1 += ra4 * rb1;

760

rC5_1 += ra5 * rb1;

761

rC0_2 += ra0 * rb2;

762

rC1_2 += ra1 * rb2;

763

rC2_2 += ra2 * rb2;

764

rC3_2 += ra3 * rb2;

765

rC4_2 += ra4 * rb2;

766

rC5_2 += ra5 * rb2;

767

rC0_3 += ra0 * rb3;

768

rC1_3 += ra1 * rb3;

769

rC2_3 += ra2 * rb3;

770

rC3_3 += ra3 * rb3;

771

rC4_3 += ra4 * rb3;

772

rC5_3 += ra5 * rb3;

773

rC0_4 += ra0 * rb4;

774

rC1_4 += ra1 * rb4;

775

rC2_4 += ra2 * rb4;

776

rC3_4 += ra3 * rb4;

777

rC4_4 += ra4 * rb4;

778

rC5_4 += ra5 * rb4;

779

rC0_5 += ra0 * rb5;

780

rC1_5 += ra1 * rb5;

781

rC2_5 += ra2 * rb5;

782

rC3_5 += ra3 * rb5;

783

rC4_5 += ra4 * rb5;

784

rC5_5 += ra5 * rb5;

785

rC0_6 += ra0 * rb6;

786

rC1_6 += ra1 * rb6;

787

rC2_6 += ra2 * rb6;

788

rC3_6 += ra3 * rb6;

789

rC4_6 += ra4 * rb6;

790

rC5_6 += ra5 * rb6;

791

rC0_7 += ra0 * rb7;

792

rC1_7 += ra1 * rb7;

793

rC2_7 += ra2 * rb7;

794

rC3_7 += ra3 * rb7;

795

rC4_7 += ra4 * rb7;

796

rC5_7 += ra5 * rb7;

797

#if KB == 0

798

}

799

#endif

800

801

#ifdef TREAL

802

*pC0 = rC0_0; pC0[1] = rC1_0; pC0[ 2] = rC2_0;

803

pC0[3] = rC3_0; pC0[4] = rC4_0; pC0[ 5] = rC5_0;

804

*pC1 = rC0_1; pC1[1] = rC1_1; pC1[ 2] = rC2_1;

805

pC1[3] = rC3_1; pC1[4] = rC4_1; pC1[ 5] = rC5_1;

806

*pC2 = rC0_2; pC2[1] = rC1_2; pC2[ 2] = rC2_2;

807

pC2[3] = rC3_2; pC2[4] = rC4_2; pC2[ 5] = rC5_2;

808

*pC3 = rC0_3; pC3[1] = rC1_3; pC3[ 2] = rC2_3;

809

pC3[3] = rC3_3; pC3[4] = rC4_3; pC3[ 5] = rC5_3;

810

*pC4 = rC0_4; pC4[1] = rC1_4; pC4[ 2] = rC2_4;

811

pC4[3] = rC3_4; pC4[4] = rC4_4; pC4[ 5] = rC5_4;

812

*pC5 = rC0_5; pC5[1] = rC1_5; pC5[ 2] = rC2_5;

813

pC5[3] = rC3_5; pC5[4] = rC4_5; pC5[ 5] = rC5_5;

814

*pC6 = rC0_6; pC6[1] = rC1_6; pC6[ 2] = rC2_6;

815

pC6[3] = rC3_6; pC6[4] = rC4_6; pC6[ 5] = rC5_6;

816

*pC7 = rC0_7; pC7[1] = rC1_7; pC7[ 2] = rC2_7;

817

pC7[3] = rC3_7; pC7[4] = rC4_7; pC7[ 5] = rC5_7;

818

#else

819

*pC0 = rC0_0; pC0[2] = rC1_0; pC0[ 4] = rC2_0;

820

pC0[6] = rC3_0; pC0[8] = rC4_0; pC0[10] = rC5_0;

821

*pC1 = rC0_1; pC1[2] = rC1_1; pC1[ 4] = rC2_1;

822

pC1[6] = rC3_1; pC1[8] = rC4_1; pC1[10] = rC5_1;

823

*pC2 = rC0_2; pC2[2] = rC1_2; pC2[ 4] = rC2_2;

824

pC2[6] = rC3_2; pC2[8] = rC4_2; pC2[10] = rC5_2;

825

*pC3 = rC0_3; pC3[2] = rC1_3; pC3[ 4] = rC2_3;

826

pC3[6] = rC3_3; pC3[8] = rC4_3; pC3[10] = rC5_3;

827

*pC4 = rC0_4; pC4[2] = rC1_4; pC4[ 4] = rC2_4;

828

pC4[6] = rC3_4; pC4[8] = rC4_4; pC4[10] = rC5_4;

829

*pC5 = rC0_5; pC5[2] = rC1_5; pC5[ 4] = rC2_5;

830

pC5[6] = rC3_5; pC5[8] = rC4_5; pC5[10] = rC5_5;

831

*pC6 = rC0_6; pC6[2] = rC1_6; pC6[ 4] = rC2_6;

832

pC6[6] = rC3_6; pC6[8] = rC4_6; pC6[10] = rC5_6;

833

*pC7 = rC0_7; pC7[2] = rC1_7; pC7[ 4] = rC2_7;

834

pC7[6] = rC3_7; pC7[8] = rC4_7; pC7[10] = rC5_7;

835

#endif

836

pC0 += incCm; pC1 += incCm; pC2 += incCm; pC3 += incCm;

837

pC4 += incCm; pC5 += incCm; pC6 += incCm; pC7 += incCm;

838

pA0 += incAm; pA1 += incAm; pA2 += incAm; pA3 += incAm;

839

pA4 += incAm; pA5 += incAm;

840

pB0 += incBm; pB1 += incBm; pB2 += incBm; pB3 += incBm;

841

pB4 += incBm; pB5 += incBm; pB6 += incBm; pB7 += incBm;

842

}

843

while(pA0 != stM);

844

pC0 += incCn;

845

pC1 += incCn;

846

pC2 += incCn;

847

pC3 += incCn;

848

pC4 += incCn;

849

pC5 += incCn;

850

pC6 += incCn;

851

pC7 += incCn;

852

pA0 += incAn;

853

pA1 += incAn;

854

pA2 += incAn;

855

pA3 += incAn;

856

pA4 += incAn;

857

pA5 += incAn;

858

pB0 += incBn;

859

pB1 += incBn;

860

pB2 += incBn;

861

pB3 += incBn;

862

pB4 += incBn;

863

pB5 += incBn;

864

pB6 += incBn;

865

pB7 += incBn;

866

}

867

while(pB0 != stN);

868

}

869

if (k=N-Nb)

870

ATL_dJIK0x0x0TN6x1x1_a1_b1(M, k, K, alpha, ca, lda, cb + (Nb*ldb), ldb,

871

beta, cc + (Nb*ldc2), ldc);

872

if (Nb && (k=M-Mb))

873

ATL_dJIK0x0x0TN1x8x1_a1_b1(k, Nb, K, alpha, ca + (Mb*lda), lda, cb, ldb, beta, cc + (Mb SHIFT), ldc);

874

}

875

#ifdef incAm

876

#undef incAm

877

#endif

878

#ifdef incAn

879

#undef incAn

880

#endif

881

#ifdef incAk

882

#undef incAk

883

#endif

884

#ifdef incBm

885

#undef incBm

886

#endif

887

#ifdef incBn

888

#undef incBn

889

#endif

890

#ifdef incBk

891

#undef incBk

892

#endif

893

#ifdef incCm

894

#undef incCm

895

#endif

896

#ifdef incCn

897

#undef incCn

898

#endif

899

#ifdef incCk

900

#undef incCk

901

#endif

902

#ifdef Mb

903

#undef Mb

904

#endif

905

#ifdef Nb

906

#undef Nb

907

#endif

908

#ifdef Kb

909

#undef Kb

910

#endif

911

#ifdef ldc2

912

#undef ldc2

913

#endif

Older »