156
157
if (IB_WC_SUCCESS != wc->status) {
157
dprintk("RPC: %s: %s WC status %X, connection lost\n",
158
__func__, (wc->opcode & IB_WC_RECV) ? "recv" : "send",
158
dprintk("RPC: %s: WC opcode %d status %X, connection lost\n",
159
__func__, wc->opcode, wc->status);
160
160
rep->rr_len = ~0U;
161
rpcrdma_schedule_tasklet(rep);
161
if (wc->opcode != IB_WC_FAST_REG_MR && wc->opcode != IB_WC_LOCAL_INV)
162
rpcrdma_schedule_tasklet(rep);
165
166
switch (wc->opcode) {
167
case IB_WC_FAST_REG_MR:
168
frmr = (struct rpcrdma_mw *)(unsigned long)wc->wr_id;
169
frmr->r.frmr.state = FRMR_IS_VALID;
171
case IB_WC_LOCAL_INV:
172
frmr = (struct rpcrdma_mw *)(unsigned long)wc->wr_id;
173
frmr->r.frmr.state = FRMR_IS_INVALID;
167
176
rep->rr_len = wc->byte_len;
168
177
ib_dma_sync_single_for_cpu(
1450
1459
seg->mr_dma = ib_dma_map_single(ia->ri_id->device,
1451
1460
seg->mr_offset,
1452
1461
seg->mr_dmalen, seg->mr_dir);
1462
if (ib_dma_mapping_error(ia->ri_id->device, seg->mr_dma)) {
1463
dprintk("RPC: %s: mr_dma %llx mr_offset %p mr_dma_len %zu\n",
1465
(unsigned long long)seg->mr_dma,
1466
seg->mr_offset, seg->mr_dmalen);
1469
1484
struct rpcrdma_xprt *r_xprt)
1471
1486
struct rpcrdma_mr_seg *seg1 = seg;
1472
struct ib_send_wr frmr_wr, *bad_wr;
1487
struct ib_send_wr invalidate_wr, frmr_wr, *bad_wr, *post_wr;
1474
1490
int len, pageoff;
1494
1511
dprintk("RPC: %s: Using frmr %p to map %d segments\n",
1495
1512
__func__, seg1->mr_chunk.rl_mw, i);
1514
if (unlikely(seg1->mr_chunk.rl_mw->r.frmr.state == FRMR_IS_VALID)) {
1515
dprintk("RPC: %s: frmr %x left valid, posting invalidate.\n",
1517
seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey);
1518
/* Invalidate before using. */
1519
memset(&invalidate_wr, 0, sizeof invalidate_wr);
1520
invalidate_wr.wr_id = (unsigned long)(void *)seg1->mr_chunk.rl_mw;
1521
invalidate_wr.next = &frmr_wr;
1522
invalidate_wr.opcode = IB_WR_LOCAL_INV;
1523
invalidate_wr.send_flags = IB_SEND_SIGNALED;
1524
invalidate_wr.ex.invalidate_rkey =
1525
seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1526
DECR_CQCOUNT(&r_xprt->rx_ep);
1527
post_wr = &invalidate_wr;
1497
1531
/* Bump the key */
1498
1532
key = (u8)(seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey & 0x000000FF);
1499
1533
ib_update_fast_reg_key(seg1->mr_chunk.rl_mw->r.frmr.fr_mr, ++key);
1501
1535
/* Prepare FRMR WR */
1502
1536
memset(&frmr_wr, 0, sizeof frmr_wr);
1537
frmr_wr.wr_id = (unsigned long)(void *)seg1->mr_chunk.rl_mw;
1503
1538
frmr_wr.opcode = IB_WR_FAST_REG_MR;
1504
frmr_wr.send_flags = 0; /* unsignaled */
1539
frmr_wr.send_flags = IB_SEND_SIGNALED;
1505
1540
frmr_wr.wr.fast_reg.iova_start = seg1->mr_dma;
1506
1541
frmr_wr.wr.fast_reg.page_list = seg1->mr_chunk.rl_mw->r.frmr.fr_pgl;
1507
1542
frmr_wr.wr.fast_reg.page_list_len = i;
1508
1543
frmr_wr.wr.fast_reg.page_shift = PAGE_SHIFT;
1509
1544
frmr_wr.wr.fast_reg.length = i << PAGE_SHIFT;
1545
BUG_ON(frmr_wr.wr.fast_reg.length < len);
1510
1546
frmr_wr.wr.fast_reg.access_flags = (writing ?
1511
1547
IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE :
1512
1548
IB_ACCESS_REMOTE_READ);
1513
1549
frmr_wr.wr.fast_reg.rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1514
1550
DECR_CQCOUNT(&r_xprt->rx_ep);
1516
rc = ib_post_send(ia->ri_id->qp, &frmr_wr, &bad_wr);
1552
rc = ib_post_send(ia->ri_id->qp, post_wr, &bad_wr);
1519
1555
dprintk("RPC: %s: failed ib_post_send for register,"
1542
1578
rpcrdma_unmap_one(ia, seg++);
1544
1580
memset(&invalidate_wr, 0, sizeof invalidate_wr);
1581
invalidate_wr.wr_id = (unsigned long)(void *)seg1->mr_chunk.rl_mw;
1545
1582
invalidate_wr.opcode = IB_WR_LOCAL_INV;
1546
invalidate_wr.send_flags = 0; /* unsignaled */
1583
invalidate_wr.send_flags = IB_SEND_SIGNALED;
1547
1584
invalidate_wr.ex.invalidate_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1548
1585
DECR_CQCOUNT(&r_xprt->rx_ep);