mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2024-11-22 04:38:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

xprtrdma: Remove temp allocation of rpcrdma_rep objects

Browse Source 
The original code was designed so that most calls to
rpcrdma_rep_create() would occur on the NUMA node that the device
preferred. There are a few cases where that's not possible, so
those reps are marked as temporary.

However, we have the device (and its preferred node) already in
rpcrdma_rep_create(), so let's use that to guarantee the memory
is allocated from the correct node.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
This commit is contained in:
Chuck Lever 2024-06-04 15:45:27 -04:00 committed by Anna Schumaker
parent 9d53378c2c
commit 0e13dd9ea8
3 changed files with 26 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
net/sunrpc/xprtrdma/rpc_rdma.c
View File

@ -1471,8 +1471,7 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep)
credits = 1; /* don't deadlock */ credits = 1; /* don't deadlock */
else if (credits > r_xprt->rx_ep->re_max_requests) else if (credits > r_xprt->rx_ep->re_max_requests)
credits = r_xprt->rx_ep->re_max_requests; credits = r_xprt->rx_ep->re_max_requests;
rpcrdma_post_recvs(r_xprt, credits + (buf->rb_bc_srv_max_requests << 1), rpcrdma_post_recvs(r_xprt, credits + (buf->rb_bc_srv_max_requests << 1));
false);
if (buf->rb_credits != credits) if (buf->rb_credits != credits)
rpcrdma_update_cwnd(r_xprt, credits); rpcrdma_update_cwnd(r_xprt, credits);

57
net/sunrpc/xprtrdma/verbs.c
View File

@ -69,13 +69,15 @@ static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,
struct rpcrdma_sendctx *sc); struct rpcrdma_sendctx *sc);
static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt); static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt);
static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt); static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt);
static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep);
static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt); static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt);
static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt); static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt); static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);
static void rpcrdma_ep_get(struct rpcrdma_ep *ep); static void rpcrdma_ep_get(struct rpcrdma_ep *ep);
static int rpcrdma_ep_put(struct rpcrdma_ep *ep); static int rpcrdma_ep_put(struct rpcrdma_ep *ep);
static struct rpcrdma_regbuf * static struct rpcrdma_regbuf *
rpcrdma_regbuf_alloc_node(size_t size, enum dma_data_direction direction,
int node);
static struct rpcrdma_regbuf *
rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction); rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction);
static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb); static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb);
static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb); static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb);
@ -510,7 +512,7 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)
* outstanding Receives. * outstanding Receives.
*/ */
rpcrdma_ep_get(ep); rpcrdma_ep_get(ep);
rpcrdma_post_recvs(r_xprt, 1, true); rpcrdma_post_recvs(r_xprt, 1);
rc = rdma_connect(ep->re_id, &ep->re_remote_cma); rc = rdma_connect(ep->re_id, &ep->re_remote_cma);
if (rc) if (rc)
@ -943,18 +945,20 @@ static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt)
} }
static noinline static noinline
struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt)
bool temp)
{ {
struct rpcrdma_buffer *buf = &r_xprt->rx_buf; struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
struct rpcrdma_ep *ep = r_xprt->rx_ep;
struct ib_device *device = ep->re_id->device;
struct rpcrdma_rep *rep; struct rpcrdma_rep *rep;
rep = kzalloc(sizeof(*rep), XPRTRDMA_GFP_FLAGS); rep = kzalloc(sizeof(*rep), XPRTRDMA_GFP_FLAGS);
if (rep == NULL) if (rep == NULL)
goto out; goto out;
rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep->re_inline_recv, rep->rr_rdmabuf = rpcrdma_regbuf_alloc_node(ep->re_inline_recv,
DMA_FROM_DEVICE); DMA_FROM_DEVICE,
ibdev_to_node(device));
if (!rep->rr_rdmabuf) if (!rep->rr_rdmabuf)
goto out_free; goto out_free;
@ -969,7 +973,6 @@ struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt,
rep->rr_recv_wr.wr_cqe = &rep->rr_cqe; rep->rr_recv_wr.wr_cqe = &rep->rr_cqe;
rep->rr_recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov; rep->rr_recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov;
rep->rr_recv_wr.num_sge = 1; rep->rr_recv_wr.num_sge = 1;
rep->rr_temp = temp;
spin_lock(&buf->rb_lock); spin_lock(&buf->rb_lock);
list_add(&rep->rr_all, &buf->rb_all_reps); list_add(&rep->rr_all, &buf->rb_all_reps);
@ -988,17 +991,6 @@ static void rpcrdma_rep_free(struct rpcrdma_rep *rep)
kfree(rep); kfree(rep);
} }
static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep)
{
struct rpcrdma_buffer *buf = &rep->rr_rxprt->rx_buf;
spin_lock(&buf->rb_lock);
list_del(&rep->rr_all);
spin_unlock(&buf->rb_lock);
rpcrdma_rep_free(rep);
}
static struct rpcrdma_rep *rpcrdma_rep_get_locked(struct rpcrdma_buffer *buf) static struct rpcrdma_rep *rpcrdma_rep_get_locked(struct rpcrdma_buffer *buf)
{ {
struct llist_node *node; struct llist_node *node;
@ -1030,10 +1022,8 @@ static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt)
struct rpcrdma_buffer *buf = &r_xprt->rx_buf; struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
struct rpcrdma_rep *rep; struct rpcrdma_rep *rep;
list_for_each_entry(rep, &buf->rb_all_reps, rr_all) { list_for_each_entry(rep, &buf->rb_all_reps, rr_all)
rpcrdma_regbuf_dma_unmap(rep->rr_rdmabuf); rpcrdma_regbuf_dma_unmap(rep->rr_rdmabuf);
rep->rr_temp = true; /* Mark this rep for destruction */
}
} }
static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf) static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf)
@ -1250,14 +1240,15 @@ void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req)
* or Replies they may be registered externally via frwr_map. * or Replies they may be registered externally via frwr_map.
*/ */
static struct rpcrdma_regbuf * static struct rpcrdma_regbuf *
rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction) rpcrdma_regbuf_alloc_node(size_t size, enum dma_data_direction direction,
int node)
{ {
struct rpcrdma_regbuf *rb; struct rpcrdma_regbuf *rb;
rb = kmalloc(sizeof(*rb), XPRTRDMA_GFP_FLAGS); rb = kmalloc_node(sizeof(*rb), XPRTRDMA_GFP_FLAGS, node);
if (!rb) if (!rb)
return NULL; return NULL;
rb->rg_data = kmalloc(size, XPRTRDMA_GFP_FLAGS); rb->rg_data = kmalloc_node(size, XPRTRDMA_GFP_FLAGS, node);
if (!rb->rg_data) { if (!rb->rg_data) {
kfree(rb); kfree(rb);
return NULL; return NULL;
@ -1269,6 +1260,12 @@ rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction)
return rb; return rb;
} }
static struct rpcrdma_regbuf *
rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction)
{
return rpcrdma_regbuf_alloc_node(size, direction, NUMA_NO_NODE);
}
/** /**
* rpcrdma_regbuf_realloc - re-allocate a SEND/RECV buffer * rpcrdma_regbuf_realloc - re-allocate a SEND/RECV buffer
* @rb: regbuf to reallocate * @rb: regbuf to reallocate
@ -1346,10 +1343,9 @@ static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb)
* rpcrdma_post_recvs - Refill the Receive Queue * rpcrdma_post_recvs - Refill the Receive Queue
* @r_xprt: controlling transport instance * @r_xprt: controlling transport instance
* @needed: current credit grant * @needed: current credit grant
* @temp: mark Receive buffers to be deleted after one use
* *
*/ */
void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp) void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed)
{ {
struct rpcrdma_buffer *buf = &r_xprt->rx_buf; struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
struct rpcrdma_ep *ep = r_xprt->rx_ep; struct rpcrdma_ep *ep = r_xprt->rx_ep;
@ -1363,8 +1359,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp)
if (likely(ep->re_receive_count > needed)) if (likely(ep->re_receive_count > needed))
goto out; goto out;
needed -= ep->re_receive_count; needed -= ep->re_receive_count;
if (!temp) needed += RPCRDMA_MAX_RECV_BATCH;
needed += RPCRDMA_MAX_RECV_BATCH;
if (atomic_inc_return(&ep->re_receiving) > 1) if (atomic_inc_return(&ep->re_receiving) > 1)
goto out; goto out;
@ -1373,12 +1368,8 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp)
wr = NULL; wr = NULL;
while (needed) { while (needed) {
rep = rpcrdma_rep_get_locked(buf); rep = rpcrdma_rep_get_locked(buf);
if (rep && rep->rr_temp) {
rpcrdma_rep_destroy(rep);
continue;
}
if (!rep) if (!rep)
rep = rpcrdma_rep_create(r_xprt, temp); rep = rpcrdma_rep_create(r_xprt);
if (!rep) if (!rep)
break; break;
if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf)) { if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf)) {

3
net/sunrpc/xprtrdma/xprt_rdma.h
View File

@ -200,7 +200,6 @@ struct rpcrdma_rep {
__be32 rr_proc; __be32 rr_proc;
int rr_wc_flags; int rr_wc_flags;
u32 rr_inv_rkey; u32 rr_inv_rkey;
bool rr_temp;
struct rpcrdma_regbuf *rr_rdmabuf; struct rpcrdma_regbuf *rr_rdmabuf;
struct rpcrdma_xprt *rr_rxprt; struct rpcrdma_xprt *rr_rxprt;
struct rpc_rqst *rr_rqst; struct rpc_rqst *rr_rqst;
@ -468,7 +467,7 @@ void rpcrdma_flush_disconnect(struct rpcrdma_xprt *r_xprt, struct ib_wc *wc);
int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt); int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt);
void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt); void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt);
void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp); void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed);
/* /*
* Buffer calls - xprtrdma/verbs.c * Buffer calls - xprtrdma/verbs.c