1
/* Copyright 2013-2014 IBM Corp.
3
* Licensed under the Apache License, Version 2.0 (the "License");
4
* you may not use this file except in compliance with the License.
5
* You may obtain a copy of the License at
7
* http://www.apache.org/licenses/LICENSE-2.0
9
* Unless required by applicable law or agreed to in writing, software
10
* distributed under the License is distributed on an "AS IS" BASIS,
11
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13
* See the License for the specific language governing permissions and
14
* limitations under the License.
21
#include <fsi-master.h>
24
* FSI Masters sit on OPB busses behind PIB2OPB bridges
26
* There are two cMFSI behind two different bridges at
27
* different XSCOM addresses. For now we don't have them in
28
* the device-tree so we hard code the address
30
#define PIB2OPB_MFSI0_ADDR 0x20000
31
#define PIB2OPB_MFSI1_ADDR 0x30000
34
* Bridge registers on XSCOM that allow generatoin
37
#define PIB2OPB_REG_CMD 0x0
38
#define OPB_CMD_WRITE 0x80000000
39
#define OPB_CMD_READ 0x00000000
40
#define OPB_CMD_8BIT 0x00000000
41
#define OPB_CMD_16BIT 0x20000000
42
#define OPB_CMD_32BIT 0x60000000
43
#define PIB2OPB_REG_STAT 0x1
44
#define OPB_STAT_ANY_ERR 0x80000000
45
#define OPB_STAT_ERR_OPB 0x7FEC0000
46
#define OPB_STAT_ERRACK 0x00100000
47
#define OPB_STAT_BUSY 0x00010000
48
#define OPB_STAT_READ_VALID 0x00020000
49
#define OPB_STAT_ERR_CMFSI 0x0000FC00
50
#define OPB_STAT_ERR_HMFSI 0x000000FC
51
#define OPB_STAT_ERR_BASE (OPB_STAT_ANY_ERR | \
54
#define PIB2OPB_REG_LSTAT 0x2
55
#define PIB2OPB_REG_RESET 0x4
56
#define PIB2OPB_REG_cRSIC 0x5
57
#define PIB2OPB_REG_cRSIM 0x6
58
#define PIB2OPB_REG_cRSIS 0x7
59
#define PIB2OPB_REG_hRSIC 0x8
60
#define PIB2OPB_REG_hRSIM 0x9
61
#define PIB2OPB_REG_hRSIS 0xA
63
/* Low level errors from OPB contain the status in the bottom 32-bit
64
* and one of these in the top 32-bit
66
#define OPB_ERR_XSCOM_ERR 0x100000000ull
67
#define OPB_ERR_TIMEOUT_ERR 0x200000000ull
68
#define OPB_ERR_BAD_OPB_ADDR 0x400000000ull
71
* PIB2OPB 0 has 2 MFSIs, cMFSI and hMFSI, PIB2OPB 1 only
74
#define cMFSI_OPB_PORTS_BASE 0x40000
75
#define cMFSI_OPB_REG_BASE 0x03000
76
#define hMFSI_OPB_PORTS_BASE 0x80000
77
#define hMFSI_OPB_REG_BASE 0x03400
78
#define MFSI_OPB_PORT_STRIDE 0x08000
80
/* MFSI control registers */
81
#define MFSI_REG_MSTAP(__n) (0x0D0 + (__n) * 4)
82
#define MFSI_REG_MATRB0 0x1D8
83
#define MFSI_REG_MDTRB0 0x1DC
84
#define MFSI_REG_MESRB0 0x1D0
85
#define MFSI_REG_MAESP0 0x050
86
#define MFSI_REG_MAEB 0x070
87
#define MFSI_REG_MSCSB0 0x1D4
89
/* FSI Slave registers */
90
#define FSI_SLAVE_REGS 0x000800 /**< FSI Slave Register */
91
#define FSI_SMODE (FSI_SLAVE_REGS | 0x00)
92
#define FSI_SLBUS (FSI_SLAVE_REGS | 0x30)
93
#define FSI_SLRES (FSI_SLAVE_REGS | 0x34)
95
#define FSI2PIB_ENGINE 0x001000 /**< FSI2PIB Engine (SCOM) */
96
#define FSI2PIB_RESET (FSI2PIB_ENGINE | 0x18)
97
#define FSI2PIB_STATUS (FSI2PIB_ENGINE | 0x1C)
98
#define FSI2PIB_COMPMASK (FSI2PIB_ENGINE | 0x30)
99
#define FSI2PIB_TRUEMASK (FSI2PIB_ENGINE | 0x34)
110
#define mfsi_log(__lev, __m, __fmt, ...) \
111
prlog(__lev, "MFSI %x:%x: " __fmt, __m->chip_id, __m->unit, ##__VA_ARGS__)
113
* Use a global FSI lock for now. Beware of re-entrancy
114
* if we ever add support for normal chip XSCOM via FSI, in
115
* which case we'll probably have to consider either per chip
116
* lock (which can have AB->BA deadlock issues) or a re-entrant
117
* global lock or something else. ...
119
static struct lock fsi_lock = LOCK_UNLOCKED;
125
/* We try up to 1.2ms for an OPB access */
126
#define MFSI_OPB_MAX_TRIES 1200
128
static uint64_t mfsi_opb_poll(struct mfsi *mfsi, uint32_t *read_data)
130
unsigned long retries = MFSI_OPB_MAX_TRIES;
135
/* We try again every 10us for a bit more than 1ms */
137
/* Read OPB status register */
138
rc = xscom_read(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_STAT, &sval);
140
/* Do something here ? */
141
mfsi_log(PR_ERR, mfsi, "XSCOM error %lld read OPB STAT\n", rc);
142
return OPB_ERR_XSCOM_ERR;
144
mfsi_log(PR_INSANE, mfsi, " STAT=0x%16llx...\n", sval);
149
if (!(stat & OPB_STAT_BUSY))
151
if (retries-- == 0) {
152
/* This isn't supposed to happen (HW timeout) */
153
mfsi_log(PR_ERR, mfsi, "OPB POLL timeout !\n");
154
return OPB_ERR_TIMEOUT_ERR | (stat & mfsi->err_bits);
159
/* Did we have an error ? */
160
if (stat & mfsi->err_bits)
161
return stat & mfsi->err_bits;
164
if (!(stat & OPB_STAT_READ_VALID)) {
165
mfsi_log(PR_ERR, mfsi, "Read successful but no data !\n");
167
/* What do do here ? can it actually happen ? */
170
*read_data = sval & 0xffffffff;
176
static uint64_t mfsi_opb_read(struct mfsi *mfsi, uint32_t opb_addr, uint32_t *data)
178
uint64_t opb_cmd = OPB_CMD_READ | OPB_CMD_32BIT;
181
if (opb_addr > 0x00ffffff)
182
return OPB_ERR_BAD_OPB_ADDR;
187
mfsi_log(PR_INSANE, mfsi, "MFSI_OPB_READ: Writing 0x%16llx to XSCOM %x\n",
188
opb_cmd, mfsi->xscom_base);
190
rc = xscom_write(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_CMD, opb_cmd);
192
mfsi_log(PR_ERR, mfsi, "XSCOM error %lld writing OPB CMD\n", rc);
193
return OPB_ERR_XSCOM_ERR;
195
return mfsi_opb_poll(mfsi, data);
198
static uint64_t mfsi_opb_write(struct mfsi *mfsi, uint32_t opb_addr, uint32_t data)
200
uint64_t opb_cmd = OPB_CMD_WRITE | OPB_CMD_32BIT;
203
if (opb_addr > 0x00ffffff)
204
return OPB_ERR_BAD_OPB_ADDR;
210
mfsi_log(PR_INSANE, mfsi, "MFSI_OPB_WRITE: Writing 0x%16llx to XSCOM %x\n",
211
opb_cmd, mfsi->xscom_base);
213
rc = xscom_write(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_CMD, opb_cmd);
215
mfsi_log(PR_ERR, mfsi, "XSCOM error %lld writing OPB CMD\n", rc);
216
return OPB_ERR_XSCOM_ERR;
218
return mfsi_opb_poll(mfsi, NULL);
221
static struct mfsi *mfsi_get(uint32_t chip_id, uint32_t unit)
223
struct proc_chip *chip = get_chip(chip_id);
226
if (!chip || unit > MFSI_hMFSI0)
228
mfsi = &chip->fsi_masters[unit];
229
if (mfsi->xscom_base == 0)
234
static int64_t mfsi_reset_pib2opb(struct mfsi *mfsi)
239
rc = xscom_write(mfsi->chip_id,
240
mfsi->xscom_base + PIB2OPB_REG_RESET, (1ul << 63));
242
mfsi_log(PR_ERR, mfsi, "XSCOM error %lld resetting PIB2OPB\n", rc);
245
rc = xscom_write(mfsi->chip_id,
246
mfsi->xscom_base + PIB2OPB_REG_STAT, (1ul << 63));
248
mfsi_log(PR_ERR, mfsi, "XSCOM error %lld resetting status\n", rc);
251
rc = xscom_read(mfsi->chip_id,
252
mfsi->xscom_base + PIB2OPB_REG_STAT, &stat);
254
mfsi_log(PR_ERR, mfsi, "XSCOM error %lld reading status\n", rc);
261
static void mfsi_dump_pib2opb_state(struct mfsi *mfsi)
265
/* Dump a bunch of registers */
266
if (xscom_read(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_CMD, &val))
268
mfsi_log(PR_ERR, mfsi, " PIB2OPB CMD = %016llx\n", val);
269
if (xscom_read(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_STAT, &val))
271
mfsi_log(PR_ERR, mfsi, " PIB2OPB STAT = %016llx\n", val);
272
if (xscom_read(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_LSTAT, &val))
274
mfsi_log(PR_ERR, mfsi, " PIB2OPB LSTAT = %016llx\n", val);
276
if (mfsi->unit == MFSI_cMFSI0 || mfsi->unit == MFSI_cMFSI1) {
277
if (xscom_read(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_cRSIC, &val))
279
mfsi_log(PR_ERR, mfsi, " PIB2OPB cRSIC = %016llx\n", val);
280
if (xscom_read(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_cRSIM, &val))
282
mfsi_log(PR_ERR, mfsi, " PIB2OPB cRSIM = %016llx\n", val);
283
if (xscom_read(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_cRSIS, &val))
285
mfsi_log(PR_ERR, mfsi, " PIB2OPB cRSIS = %016llx\n", val);
286
} else if (mfsi->unit == MFSI_hMFSI0) {
287
if (xscom_read(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_hRSIC, &val))
289
mfsi_log(PR_ERR, mfsi, " PIB2OPB hRSIC = %016llx\n", val);
290
if (xscom_read(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_hRSIM, &val))
292
mfsi_log(PR_ERR, mfsi, " PIB2OPB hRSIM = %016llx\n", val);
293
if (xscom_read(mfsi->chip_id, mfsi->xscom_base + PIB2OPB_REG_hRSIS, &val))
295
mfsi_log(PR_ERR, mfsi, " PIB2OPB hRSIS = %016llx\n", val);
299
mfsi_log(PR_ERR, mfsi, "XSCOM error reading PIB2OPB registers\n");
302
static int64_t mfsi_dump_ctrl_regs(struct mfsi *mfsi)
307
/* List of registers to dump (from HB) */
308
static uint32_t dump_regs[] = {
316
static const char * dump_regs_names[] = {
324
for (i = 0; i < ARRAY_SIZE(dump_regs); i++) {
327
opb_stat = mfsi_opb_read(mfsi, mfsi->reg_base + dump_regs[i], &val);
329
/* Error on dump, give up */
330
mfsi_log(PR_ERR, mfsi, " OPB stat 0x%016llx dumping reg %x\n",
331
opb_stat, dump_regs[i]);
332
return OPAL_HARDWARE;
334
mfsi_log(PR_ERR, mfsi, " %s = %08x\n", dump_regs_names[i], val);
336
for (i = 0; i < 8; i++) {
339
opb_stat = mfsi_opb_read(mfsi, mfsi->reg_base + MFSI_REG_MSTAP(i), &val);
341
/* Error on dump, give up */
342
mfsi_log(PR_ERR, mfsi, " OPB stat 0x%016llx dumping reg %x\n",
343
opb_stat, MFSI_REG_MSTAP(i));
344
return OPAL_HARDWARE;
346
mfsi_log(PR_ERR, mfsi, " MFSI_REG_MSTAP%d = %08x\n", i, val);
351
static int64_t mfsi_master_cleanup(struct mfsi *mfsi, uint32_t port)
354
uint32_t port_base, compmask, truemask;
356
/* Reset the bridge to clear up the residual errors */
358
/* bit0 = Bridge: General reset */
359
opb_stat = mfsi_opb_write(mfsi, mfsi->reg_base + MFSI_REG_MESRB0, 0x80000000u);
361
mfsi_log(PR_ERR, mfsi, " OPB stat 0x%016llx writing reset to MESRB0\n",
363
return OPAL_HARDWARE;
366
/* Calculate base address of port */
367
port_base = mfsi->ports_base + port * MFSI_OPB_PORT_STRIDE;
369
/* Perform error reset on Centaur fsi slave: */
370
/* write 0x4000000 to addr=834 */
371
opb_stat = mfsi_opb_write(mfsi, port_base + FSI_SLRES, 0x04000000);
373
mfsi_log(PR_ERR, mfsi,
374
" OPB stat 0x%016llx writing reset to FSI slave\n",
376
return OPAL_HARDWARE;
379
/* Further step is to issue a PIB reset to the FSI2PIB engine
380
* in busy state, i.e. write arbitrary data to 101c
381
* (putcfam 1007) register of the previously failed FSI2PIB
384
* XXX BenH: Should that be done by the upper FSI XSCOM layer ?
386
opb_stat = mfsi_opb_write(mfsi, port_base + FSI2PIB_STATUS, 0xFFFFFFFF);
388
mfsi_log(PR_ERR, mfsi,
389
" OPB stat 0x%016llx clearing FSI2PIB_STATUS\n",
391
return OPAL_HARDWARE;
394
/* Need to save/restore the true/comp masks or the FSP (PRD ?) will
397
opb_stat = mfsi_opb_read(mfsi, port_base + FSI2PIB_COMPMASK, &compmask);
399
mfsi_log(PR_ERR, mfsi,
400
" OPB stat 0x%016llx reading FSI2PIB_COMPMASK\n",
402
return OPAL_HARDWARE;
404
opb_stat = mfsi_opb_read(mfsi, port_base + FSI2PIB_TRUEMASK, &truemask);
406
mfsi_log(PR_ERR, mfsi,
407
" OPB stat 0x%016llx reading FSI2PIB_TRUEMASK\n",
409
return OPAL_HARDWARE;
412
/* Then, write arbitrary data to 1018 (putcfam 1006) to
413
* reset any pending FSI2PIB errors.
415
opb_stat = mfsi_opb_write(mfsi, port_base + FSI2PIB_RESET, 0xFFFFFFFF);
417
mfsi_log(PR_ERR, mfsi,
418
" OPB stat 0x%016llx writing FSI2PIB_RESET\n",
420
return OPAL_HARDWARE;
423
/* Restore the true/comp masks */
424
opb_stat = mfsi_opb_write(mfsi, port_base + FSI2PIB_COMPMASK, compmask);
426
mfsi_log(PR_ERR, mfsi,
427
" OPB stat 0x%016llx writing FSI2PIB_COMPMASK\n",
429
return OPAL_HARDWARE;
431
opb_stat = mfsi_opb_write(mfsi, port_base + FSI2PIB_TRUEMASK, truemask);
433
mfsi_log(PR_ERR, mfsi,
434
" OPB stat 0x%016llx writing FSI2PIB_TRUEMASK\n",
436
return OPAL_HARDWARE;
441
static int64_t mfsi_analyse_fsi_error(struct mfsi *mfsi)
446
/* Most of the code below is adapted from HB. The main difference is
451
opb_stat = mfsi_opb_read(mfsi, mfsi->reg_base + MFSI_REG_MESRB0, &mesrb0);
453
mfsi_log(PR_ERR, mfsi, " OPB stat 0x%016llx reading MESRB0\n", opb_stat);
454
return OPAL_HARDWARE;
456
mfsi_log(PR_ERR, mfsi, " MESRB0=%08x\n", mesrb0);
458
/* bits 8:15 are internal parity errors in the master */
459
if (mesrb0 & 0x00FF0000) {
460
mfsi_log(PR_ERR, mfsi, " Master parity error !\n");
462
/* bits 0:3 are a specific error code */
463
switch ((mesrb0 & 0xF0000000) >> 28 ) {
464
case 0x1: /* OPB error */
465
case 0x2: /* Invalid state of OPB state machine */
466
/* error is inside the OPB logic */
467
mfsi_log(PR_ERR, mfsi, " OPB logic error !\n");
469
case 0x3: /* Port access error */
470
/* probably some kind of code collision */
471
/* could also be something weird in the chip */
472
mfsi_log(PR_ERR, mfsi, " Port access error !\n");
474
case 0x4: /* ID mismatch */
475
mfsi_log(PR_ERR, mfsi, " Port ID mismatch !\n");
477
case 0x6: /* port timeout error */
478
mfsi_log(PR_ERR, mfsi, " Port timeout !\n");
480
case 0x7: /* master timeout error */
481
mfsi_log(PR_ERR, mfsi, " Master timeout !\n");
483
case 0x9: /* Any error response from Slave */
484
mfsi_log(PR_ERR, mfsi, " Slave error response !\n");
486
case 0xC: /* bridge parity error */
487
mfsi_log(PR_ERR, mfsi, " Bridge parity error !\n");
489
case 0xB: /* protocol error */
490
mfsi_log(PR_ERR, mfsi, " Protocol error !\n");
492
case 0x8: /* master CRC error */
493
mfsi_log(PR_ERR, mfsi, " Master CRC error !\n");
495
case 0xA: /* Slave CRC error */
496
mfsi_log(PR_ERR, mfsi, " Slave CRC error !\n");
499
mfsi_log(PR_ERR, mfsi, " Unknown error !\n");
506
static int64_t mfsi_handle_error(struct mfsi *mfsi, uint32_t port,
507
uint64_t opb_stat, uint32_t fsi_addr)
510
bool found_root_cause = false;
512
mfsi_log(PR_ERR, mfsi, "Access error on port %d, stat=%012llx\n",
515
/* First handle stat codes we synthetized */
516
if (opb_stat & OPB_ERR_XSCOM_ERR)
517
return OPAL_HARDWARE;
518
if (opb_stat & OPB_ERR_BAD_OPB_ADDR)
519
return OPAL_PARAMETER;
521
/* Dump a bunch of regisers from PIB2OPB and reset it */
522
mfsi_dump_pib2opb_state(mfsi);
525
mfsi_reset_pib2opb(mfsi);
527
/* This one is not supposed to happen but ... */
528
if (opb_stat & OPB_ERR_TIMEOUT_ERR)
529
return OPAL_HARDWARE;
531
/* Dump some FSI control registers */
532
rc = mfsi_dump_ctrl_regs(mfsi);
534
/* If that failed, reset PIB2OPB again and return */
536
mfsi_dump_pib2opb_state(mfsi);
537
mfsi_reset_pib2opb(mfsi);
538
return OPAL_HARDWARE;
541
/* Now check for known root causes (from HB) */
543
/* First check if it's a ctrl register access error and we got an OPB NACK,
544
* which means an out of bounds control reg
546
if ((opb_stat & OPB_STAT_ERRACK) &&
547
((fsi_addr & ~0x2ffu) == mfsi->reg_base)) {
548
mfsi_log(PR_ERR, mfsi, " Error appears to be out of bounds reg %08x\n",
550
found_root_cause = true;
552
/* Else check for other OPB errors */
553
else if (opb_stat & OPB_STAT_ERR_OPB) {
554
mfsi_log(PR_ERR, mfsi, " Error appears to be an OPB error\n");
555
found_root_cause = true;
558
/* Root cause not found, dig into FSI logic */
559
if (!found_root_cause) {
560
rc = mfsi_analyse_fsi_error(mfsi);
562
/* If that failed too, reset the PIB2OPB again */
563
mfsi_reset_pib2opb(mfsi);
567
/* Cleanup MFSI master */
568
mfsi_master_cleanup(mfsi, port);
570
return OPAL_HARDWARE;
573
int64_t mfsi_read(uint32_t chip, uint32_t unit, uint32_t port,
574
uint32_t fsi_addr, uint32_t *data)
576
struct mfsi *mfsi = mfsi_get(chip, unit);
579
int64_t rc = OPAL_SUCCESS;
581
if (!mfsi || port > 7)
582
return OPAL_PARAMETER;
586
/* Calculate port address */
587
port_addr = mfsi->ports_base + port * MFSI_OPB_PORT_STRIDE;
588
port_addr += fsi_addr;
590
/* Perform OPB access */
591
opb_stat = mfsi_opb_read(mfsi, port_addr, data);
593
rc = mfsi_handle_error(mfsi, port, opb_stat, port_addr);
600
int64_t mfsi_write(uint32_t chip, uint32_t unit, uint32_t port,
601
uint32_t fsi_addr, uint32_t data)
603
struct mfsi *mfsi = mfsi_get(chip, unit);
606
int64_t rc = OPAL_SUCCESS;
608
if (!mfsi || port > 7)
609
return OPAL_PARAMETER;
613
/* Calculate port address */
614
port_addr = mfsi->ports_base + port * MFSI_OPB_PORT_STRIDE;
615
port_addr += fsi_addr;
617
/* Perform OPB access */
618
opb_stat = mfsi_opb_write(mfsi, port_addr, data);
620
rc = mfsi_handle_error(mfsi, port, opb_stat, port_addr);
627
static void mfsi_add(struct proc_chip *chip, struct mfsi *mfsi, uint32_t unit)
629
mfsi->chip_id = chip->id;
632
/* We hard code everything for now */
635
mfsi->xscom_base = PIB2OPB_MFSI0_ADDR;
636
mfsi->ports_base = cMFSI_OPB_PORTS_BASE;
637
mfsi->reg_base = cMFSI_OPB_REG_BASE;
638
mfsi->err_bits = OPB_STAT_ERR_BASE | OPB_STAT_ERR_CMFSI;
641
mfsi->xscom_base = PIB2OPB_MFSI1_ADDR;
642
mfsi->ports_base = cMFSI_OPB_PORTS_BASE;
643
mfsi->reg_base = cMFSI_OPB_REG_BASE;
644
mfsi->err_bits = OPB_STAT_ERR_BASE | OPB_STAT_ERR_CMFSI;
647
mfsi->xscom_base = PIB2OPB_MFSI0_ADDR;
648
mfsi->ports_base = hMFSI_OPB_PORTS_BASE;
649
mfsi->reg_base = hMFSI_OPB_REG_BASE;
650
mfsi->err_bits = OPB_STAT_ERR_BASE | OPB_STAT_ERR_HMFSI;
657
/* Hardware Bug HW222712 on Murano DD1.0 causes the
658
* any_error bit to be un-clearable so we just
659
* have to ignore it. Additionally, HostBoot applies
660
* this to Venice too, though the comment there claims
661
* this is a Simics workaround.
663
* The doc says that bit can be safely ignored, so let's
664
* just not bother and always take it out.
667
/* 16: cMFSI any-master-error */
668
/* 24: hMFSI any-master-error */
669
mfsi->err_bits &= 0xFFFF7F7F;
671
mfsi_log(PR_INFO, mfsi, "Initialized\n");
676
struct proc_chip *chip;
678
for_each_chip(chip) {
679
chip->fsi_masters = zalloc(sizeof(struct mfsi) * 3);
680
mfsi_add(chip, &chip->fsi_masters[MFSI_cMFSI0], MFSI_cMFSI0);
681
mfsi_add(chip, &chip->fsi_masters[MFSI_hMFSI0], MFSI_hMFSI0);
682
mfsi_add(chip, &chip->fsi_masters[MFSI_cMFSI1], MFSI_cMFSI1);