From: Tom Wu <tomwu@nvidia.com>
Subject: [PATCH] BACKPORT: net/sunrpc/xprtrdma/svc_rdma_rw.c

Change-Id: Ib0d75d5fdce7807273d8480425dec98efaa5ffb4
Signed-off-by: Tom Wu <tomwu@nvidia.com>
---
 net/sunrpc/xprtrdma/svc_rdma_rw.c | 571 +++++++++++++++++++++++++++++-
 1 file changed, 570 insertions(+), 1 deletion(-)

--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -12,7 +12,9 @@
 #include <linux/sunrpc/svc_rdma.h>
 
 #include "xprt_rdma.h"
+#ifdef HAVE_TRACE_RPCRDMA_H
 #include <trace/events/rpcrdma.h>
+#endif
 
 static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc);
 static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc);
@@ -75,23 +77,34 @@ svc_rdma_get_rw_ctxt(struct svcxprt_rdma
 	}
 
 	ctxt->rw_sg_table.sgl = ctxt->rw_first_sgl;
+#ifdef HAVE_SG_ALLOC_TABLE_CHAINED_NENTS_FIRST_CHUNK_PARAM
 	if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges,
 				   ctxt->rw_sg_table.sgl,
 				   first_sgl_nents))
+#else
+	if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges,
+				   ctxt->rw_sg_table.sgl))
+#endif
 		goto out_free;
 	return ctxt;
 
 out_free:
 	kfree(ctxt);
 out_noctx:
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_rwctx_empty(rdma, sges);
+#endif
 	return NULL;
 }
 
 static void __svc_rdma_put_rw_ctxt(struct svc_rdma_rw_ctxt *ctxt,
 				   struct llist_head *list)
 {
+#ifdef HAVE_SG_ALLOC_TABLE_CHAINED_NENTS_FIRST_CHUNK_PARAM
 	sg_free_table_chained(&ctxt->rw_sg_table, ctxt->rw_first_sgl_nents);
+#else
+	sg_free_table_chained(&ctxt->rw_sg_table, true);
+#endif
 	llist_add(&ctxt->rw_node, list);
 }
 
@@ -139,8 +152,10 @@ static int svc_rdma_rw_ctx_init(struct s
 			       ctxt->rw_sg_table.sgl, ctxt->rw_nents,
 			       0, offset, handle, direction);
 	if (unlikely(ret < 0)) {
+#ifdef HAVE_TRACE_RPCRDMA_H
 		trace_svcrdma_dma_map_rw_err(rdma, offset, handle,
 					     ctxt->rw_nents, ret);
+#endif
 		svc_rdma_put_rw_ctxt(rdma, ctxt);
 	}
 	return ret;
@@ -158,6 +173,9 @@ void svc_rdma_cc_init(struct svcxprt_rdm
 
 	if (unlikely(!cid->ci_completion_id))
 		svc_rdma_send_cid_init(rdma, cid);
+#ifndef HAVE_SVC_RDMA_PCL
+	cc->cc_rdma = rdma;
+#endif
 
 	INIT_LIST_HEAD(&cc->cc_rwctxts);
 	cc->cc_sqecount = 0;
@@ -177,7 +195,9 @@ void svc_rdma_cc_release(struct svcxprt_
 	struct svc_rdma_rw_ctxt *ctxt;
 	LLIST_HEAD(free);
 
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_cc_release(&cc->cc_cid, cc->cc_sqecount);
+#endif
 
 	first = last = NULL;
 	while ((ctxt = svc_rdma_next_ctxt(&cc->cc_rwctxts)) != NULL) {
@@ -198,8 +218,12 @@ void svc_rdma_cc_release(struct svcxprt_
 }
 
 static struct svc_rdma_write_info *
+#ifdef HAVE_SVC_RDMA_PCL
 svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma,
 			  const struct svc_rdma_chunk *chunk)
+#else
+svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma, __be32 *chunk)
+#endif
 {
 	struct svc_rdma_write_info *info;
 
@@ -209,12 +233,20 @@ svc_rdma_write_info_alloc(struct svcxprt
 		return info;
 
 	info->wi_rdma = rdma;
+#ifdef HAVE_SVC_RDMA_PCL
 	info->wi_chunk = chunk;
+#else
+	info->wi_nsegs = be32_to_cpup(++chunk);
+	info->wi_segs = ++chunk;
+	info->wi_seg_off = 0;
+	info->wi_seg_no = 0;
+#endif
 	svc_rdma_cc_init(rdma, &info->wi_cc);
 	info->wi_cc.cc_cqe.done = svc_rdma_write_done;
 	return info;
 }
 
+#ifdef HAVE_SVC_RDMA_PCL
 static void svc_rdma_write_info_free_async(struct work_struct *work)
 {
 	struct svc_rdma_write_info *info;
@@ -229,7 +261,15 @@ static void svc_rdma_write_info_free(str
 	INIT_WORK(&info->wi_work, svc_rdma_write_info_free_async);
 	queue_work(svcrdma_wq, &info->wi_work);
 }
+#else
+static void svc_rdma_write_info_free(struct svc_rdma_write_info *info)
+{
+	svc_rdma_cc_release(info->wi_cc.cc_rdma, &info->wi_cc, DMA_TO_DEVICE);
+	kfree(info);
+}
+#endif
 
+#ifdef HAVE_SVC_RDMA_PCL
 /**
  * svc_rdma_write_chunk_release - Release Write chunk I/O resources
  * @rdma: controlling transport
@@ -266,7 +306,9 @@ void svc_rdma_reply_chunk_release(struct
 		return;
 	svc_rdma_cc_release(rdma, cc, DMA_TO_DEVICE);
 }
+#endif
 
+#ifdef HAVE_SVC_RDMA_PCL
 /**
  * svc_rdma_reply_done - Reply chunk Write completion handler
  * @cq: controlling Completion Queue
@@ -276,10 +318,11 @@ void svc_rdma_reply_chunk_release(struct
  */
 static void svc_rdma_reply_done(struct ib_cq *cq, struct ib_wc *wc)
 {
+	struct svcxprt_rdma *rdma = cq->cq_context;
+#ifdef HAVE_TRACE_RPCRDMA_H
 	struct ib_cqe *cqe = wc->wr_cqe;
 	struct svc_rdma_chunk_ctxt *cc =
 			container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe);
-	struct svcxprt_rdma *rdma = cq->cq_context;
 
 	switch (wc->status) {
 	case IB_WC_SUCCESS:
@@ -291,10 +334,17 @@ static void svc_rdma_reply_done(struct i
 	default:
 		trace_svcrdma_wc_reply_err(wc, &cc->cc_cid);
 	}
+#endif
 
+#ifdef HAVE_SVC_XPRT_DEFERRED_CLOSE
 	svc_xprt_deferred_close(&rdma->sc_xprt);
+#else
+	set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags);
+#endif
 }
+#endif
 
+#ifdef HAVE_SVC_RDMA_PCL
 /**
  * svc_rdma_write_done - Write chunk completion
  * @cq: controlling Completion Queue
@@ -305,6 +355,7 @@ static void svc_rdma_reply_done(struct i
 static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
 {
 	struct svcxprt_rdma *rdma = cq->cq_context;
+#ifdef HAVE_TRACE_RPCRDMA_H
 	struct ib_cqe *cqe = wc->wr_cqe;
 	struct svc_rdma_chunk_ctxt *cc =
 			container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe);
@@ -319,20 +370,101 @@ static void svc_rdma_write_done(struct i
 	default:
 		trace_svcrdma_wc_write_err(wc, &cc->cc_cid);
 	}
+#endif
 
 	/* The RDMA Write has flushed, so the client won't get
 	 * some of the outgoing RPC message. Signal the loss
 	 * to the client by closing the connection.
 	 */
+#ifdef HAVE_SVC_XPRT_DEFERRED_CLOSE
 	svc_xprt_deferred_close(&rdma->sc_xprt);
+#else
+	set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags);
+#endif
+}
+#else
+/**
+ * svc_rdma_write_done - Write chunk completion
+ * @cq: controlling Completion Queue
+ * @wc: Work Completion
+ *
+ * Pages under I/O are freed by a subsequent Send completion.
+ */
+static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+struct ib_cqe *cqe = wc->wr_cqe;
+struct svc_rdma_chunk_ctxt *cc =
+		container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe);
+struct svcxprt_rdma *rdma = cc->cc_rdma;
+struct svc_rdma_write_info *info =
+		container_of(cc, struct svc_rdma_write_info, wi_cc);
+
+#ifdef HAVE_TRACE_RPCRDMA_H
+	switch (wc->status) {
+	case IB_WC_SUCCESS:
+		trace_svcrdma_wc_write(&cc->cc_cid);
+		break;
+	case IB_WC_WR_FLUSH_ERR:
+		trace_svcrdma_wc_write_flush(wc, &cc->cc_cid);
+		break;
+	default:
+		trace_svcrdma_wc_write_err(wc, &cc->cc_cid);
+	}
+#endif
+
+	svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount);
+
+	if (unlikely(wc->status != IB_WC_SUCCESS))
+#ifdef HAVE_SVC_XPRT_DEFERRED_CLOSE
+		svc_xprt_deferred_close(&rdma->sc_xprt);
+#else
+		set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags);
+#endif
+
+	svc_rdma_write_info_free(info);
+}
+#endif
+
+#ifndef HAVE_SVC_RDMA_PCL
+struct svc_rdma_read_info {
+	unsigned int			ri_position;
+	unsigned int			ri_chunklen;
+	struct svc_rdma_recv_ctxt	*ri_readctxt;
+	unsigned int			ri_pageno;
+	unsigned int			ri_pageoff;
+
+	struct svc_rdma_chunk_ctxt	ri_cc;
+};
+
+static struct svc_rdma_read_info *
+svc_rdma_read_info_alloc(struct svcxprt_rdma *rdma)
+{
+	struct svc_rdma_read_info *info;
+
+	info = kmalloc_node(sizeof(*info), GFP_KERNEL,
+			    ibdev_to_node(rdma->sc_cm_id->device));
+	if (!info)
+		return info;
+
+	svc_rdma_cc_init(rdma, &info->ri_cc);
+	info->ri_cc.cc_cqe.done = svc_rdma_wc_read_done;
+	return info;
 }
 
+static void svc_rdma_read_info_free(struct svc_rdma_read_info *info)
+{
+	svc_rdma_cc_release(info->ri_cc.cc_rdma, &info->ri_cc, DMA_FROM_DEVICE);
+	kfree(info);
+}
+#endif
+
 /**
  * svc_rdma_wc_read_done - Handle completion of an RDMA Read ctx
  * @cq: controlling Completion Queue
  * @wc: Work Completion
  *
  */
+#ifdef HAVE_SVC_RDMA_PCL
 static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc)
 {
 	struct svcxprt_rdma *rdma = cq->cq_context;
@@ -346,8 +478,10 @@ static void svc_rdma_wc_read_done(struct
 	ctxt = container_of(cc, struct svc_rdma_recv_ctxt, rc_cc);
 	switch (wc->status) {
 	case IB_WC_SUCCESS:
+#ifdef HAVE_TRACE_RPCRDMA_H
 		trace_svcrdma_wc_read(wc, &cc->cc_cid, ctxt->rc_readbytes,
 				      cc->cc_posttime);
+#endif
 
 		spin_lock(&rdma->sc_rq_dto_lock);
 		list_add_tail(&ctxt->rc_list, &rdma->sc_read_complete_q);
@@ -357,10 +491,15 @@ static void svc_rdma_wc_read_done(struct
 		svc_xprt_enqueue(&rdma->sc_xprt);
 		return;
 	case IB_WC_WR_FLUSH_ERR:
+#ifdef HAVE_TRACE_RPCRDMA_H
 		trace_svcrdma_wc_read_flush(wc, &cc->cc_cid);
+#endif
 		break;
 	default:
+#ifdef HAVE_TRACE_RPCRDMA_H
 		trace_svcrdma_wc_read_err(wc, &cc->cc_cid);
+#endif
+		break;
 	}
 
 	/* The RDMA Read has flushed, so the incoming RPC message
@@ -369,8 +508,54 @@ static void svc_rdma_wc_read_done(struct
 	 */
 	svc_rdma_cc_release(rdma, cc, DMA_FROM_DEVICE);
 	svc_rdma_recv_ctxt_put(rdma, ctxt);
+#ifdef HAVE_SVC_XPRT_DEFERRED_CLOSE
 	svc_xprt_deferred_close(&rdma->sc_xprt);
+#else
+	set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags);
+#endif
+}
+#else
+static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct ib_cqe *cqe = wc->wr_cqe;
+	struct svc_rdma_chunk_ctxt *cc =
+			container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe);
+	struct svcxprt_rdma *rdma = cc->cc_rdma;
+	struct svc_rdma_read_info *info =
+			container_of(cc, struct svc_rdma_read_info, ri_cc);
+
+#ifdef HAVE_TRACE_RPCRDMA_H
+	switch (wc->status) {
+	case IB_WC_SUCCESS:
+		trace_svcrdma_wc_read(wc, &cc->cc_cid, info->ri_chunklen,
+				      cc->cc_posttime);
+		break;
+	case IB_WC_WR_FLUSH_ERR:
+		trace_svcrdma_wc_read_flush(wc, &cc->cc_cid);
+		break;
+	default:
+		trace_svcrdma_wc_read_err(wc, &cc->cc_cid);
+	}
+#endif
+
+	svc_rdma_wake_send_waiters(cc->cc_rdma, cc->cc_sqecount);
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags);
+		svc_rdma_recv_ctxt_put(rdma, info->ri_readctxt);
+	} else {
+		spin_lock(&rdma->sc_rq_dto_lock);
+		list_add_tail(&info->ri_readctxt->rc_list,
+			      &rdma->sc_read_complete_q);
+		/* Note the unlock pairs with the smp_rmb in svc_xprt_ready: */
+		set_bit(XPT_DATA, &rdma->sc_xprt.xpt_flags);
+		spin_unlock(&rdma->sc_rq_dto_lock);
+
+		svc_xprt_enqueue(&rdma->sc_xprt);
+	}
+
+	svc_rdma_read_info_free(info);
 }
+#endif
 
 /*
  * Assumptions:
@@ -414,15 +599,25 @@ static int svc_rdma_post_chunk_ctxt(stru
 		}
 
 		percpu_counter_inc(&svcrdma_stat_sq_starve);
+#ifdef HAVE_TRACE_RPCRDMA_H
 		trace_svcrdma_sq_full(rdma, &cc->cc_cid);
+#endif
 		atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail);
 		wait_event(rdma->sc_send_wait,
 			   atomic_read(&rdma->sc_sq_avail) > cc->cc_sqecount);
+#ifdef HAVE_TRACE_RPCRDMA_H
 		trace_svcrdma_sq_retry(rdma, &cc->cc_cid);
+#endif
 	} while (1);
 
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_sq_post_err(rdma, &cc->cc_cid, ret);
+#endif
+#ifdef HAVE_SVC_XPRT_DEFERRED_CLOSE
 	svc_xprt_deferred_close(&rdma->sc_xprt);
+#else
+	set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags);
+#endif
 
 	/* If even one was posted, there will be a completion. */
 	if (bad_wr != first_wr)
@@ -491,11 +686,17 @@ svc_rdma_build_writes(struct svc_rdma_wr
 		      unsigned int remaining)
 {
 	struct svc_rdma_chunk_ctxt *cc = &info->wi_cc;
+#ifdef HAVE_SVC_RDMA_PCL
 	struct svcxprt_rdma *rdma = info->wi_rdma;
 	const struct svc_rdma_segment *seg;
+#else
+	struct svcxprt_rdma *rdma = cc->cc_rdma;
+	__be32 *seg;
+#endif
 	struct svc_rdma_rw_ctxt *ctxt;
 	int ret;
 
+#ifdef HAVE_SVC_RDMA_PCL
 	do {
 		unsigned int write_len;
 		u64 offset;
@@ -505,6 +706,21 @@ svc_rdma_build_writes(struct svc_rdma_wr
 
 		seg = &info->wi_chunk->ch_segments[info->wi_seg_no];
 		write_len = min(remaining, seg->rs_length - info->wi_seg_off);
+#else
+	seg = info->wi_segs + info->wi_seg_no * rpcrdma_segment_maxsz;
+	do {
+		unsigned int write_len;
+		u32 handle, length;
+		u64 offset;
+
+		if (info->wi_seg_no >= info->wi_nsegs)
+			goto out_overflow;
+
+		xdr_decode_rdma_segment(seg, &handle, &length, &offset);
+		offset += info->wi_seg_off;
+
+		write_len = min(remaining, length - info->wi_seg_off);
+#endif
 		if (!write_len)
 			goto out_overflow;
 		ctxt = svc_rdma_get_rw_ctxt(rdma,
@@ -513,8 +729,12 @@ svc_rdma_build_writes(struct svc_rdma_wr
 			return -ENOMEM;
 
 		constructor(info, write_len, ctxt);
+#ifdef HAVE_SVC_RDMA_PCL
 		offset = seg->rs_offset + info->wi_seg_off;
 		ret = svc_rdma_rw_ctx_init(rdma, ctxt, offset, seg->rs_handle,
+#else
+		ret = svc_rdma_rw_ctx_init(rdma, ctxt, offset, handle,
+#endif
 					   DMA_TO_DEVICE);
 		if (ret < 0)
 			return -EIO;
@@ -522,7 +742,12 @@ svc_rdma_build_writes(struct svc_rdma_wr
 
 		list_add(&ctxt->rw_list, &cc->cc_rwctxts);
 		cc->cc_sqecount += ret;
+#ifdef HAVE_SVC_RDMA_PCL
 		if (write_len == seg->rs_length - info->wi_seg_off) {
+#else
+		if (write_len == length - info->wi_seg_off) {
+			seg += 4;
+#endif
 			info->wi_seg_no++;
 			info->wi_seg_off = 0;
 		} else {
@@ -534,8 +759,14 @@ svc_rdma_build_writes(struct svc_rdma_wr
 	return 0;
 
 out_overflow:
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_small_wrch_err(&cc->cc_cid, remaining, info->wi_seg_no,
+#ifdef HAVE_SVC_RDMA_PCL
 				     info->wi_chunk->ch_segcount);
+#else
+				     info->wi_nsegs);
+#endif
+#endif
 	return -E2BIG;
 }
 
@@ -582,6 +813,7 @@ static int svc_rdma_pages_write(struct s
 				     length);
 }
 
+#ifdef HAVE_SVC_RDMA_PCL
 /**
  * svc_rdma_xb_write - Construct RDMA Writes to write an xdr_buf
  * @xdr: xdr_buf to write
@@ -666,7 +898,9 @@ static int svc_rdma_prepare_write_chunk(
 	sctxt->sc_sqecount += cc->cc_sqecount;
 	list_add(&info->wi_list, &sctxt->sc_write_info_list);
 
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount);
+#endif
 	return 0;
 
 out_err:
@@ -754,10 +988,99 @@ int svc_rdma_prepare_reply_chunk(struct
 	sctxt->sc_wr_chain = first_wr;
 	sctxt->sc_sqecount += cc->cc_sqecount;
 
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_post_reply_chunk(&cc->cc_cid, cc->cc_sqecount);
+#endif
 	return xdr->len;
 }
+#else
+int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, __be32 *wr_ch,
+			      struct xdr_buf *xdr,
+			      unsigned int offset, unsigned long length)
+{
+	struct svc_rdma_write_info *info;
+	struct svc_rdma_chunk_ctxt *cc;
+	int ret;
+
+	if (!length)
+		return 0;
+
+	info = svc_rdma_write_info_alloc(rdma, wr_ch);
+	if (!info)
+		return -ENOMEM;
+	cc = &info->wi_cc;
+
+	ret = svc_rdma_pages_write(info, xdr, offset, length);
+	if (ret < 0)
+		goto out_err;
+
+#ifdef HAVE_TRACE_RPCRDMA_H
+	trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount);
+#endif
+	ret = svc_rdma_post_chunk_ctxt(cc->cc_rdma, cc);
+	if (ret < 0)
+		goto out_err;
+
+	return length;
+
+out_err:
+	svc_rdma_write_info_free(info);
+	return ret;
+}
+
+int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
+			      const struct svc_rdma_recv_ctxt *rctxt,
+			      struct xdr_buf *xdr)
+{
+	struct svc_rdma_write_info *info;
+	struct svc_rdma_chunk_ctxt *cc;
+	int consumed, ret;
+
+	info = svc_rdma_write_info_alloc(rdma, rctxt->rc_reply_chunk);
+	if (!info)
+		return -ENOMEM;
+	cc = &info->wi_cc;
+
+	ret = svc_rdma_iov_write(info, &xdr->head[0]);
+	if (ret < 0)
+		goto out_err;
+
+	consumed = xdr->head[0].iov_len;
+
+	/* Send the page list in the Reply chunk only if the
+	 * client did not provide Write chunks.
+	 */
+	if (!rctxt->rc_write_list && xdr->page_len) {
+		ret = svc_rdma_pages_write(info, xdr, xdr->head[0].iov_len,
+					   xdr->page_len);
+		if (ret < 0)
+			goto out_err;
+		consumed += xdr->page_len;
+	}
+
+	if (xdr->tail[0].iov_len) {
+		ret = svc_rdma_iov_write(info, &xdr->tail[0]);
+		if (ret < 0)
+			goto out_err;
+		consumed += xdr->tail[0].iov_len;
+	}
+
+#ifdef HAVE_TRACE_RPCRDMA_H
+	trace_svcrdma_post_reply_chunk(&cc->cc_cid, cc->cc_sqecount);
+#endif
+	ret = svc_rdma_post_chunk_ctxt(cc->cc_rdma, cc);
+	if (ret < 0)
+		goto out_err;
+
+	return consumed;
+
+out_err:
+	svc_rdma_write_info_free(info);
+	return ret;
+}
+#endif
 
+#ifdef HAVE_SVC_RDMA_PCL
 /**
  * svc_rdma_build_read_segment - Build RDMA Read WQEs to pull one RDMA segment
  * @rqstp: RPC transaction context
@@ -822,10 +1145,87 @@ static int svc_rdma_build_read_segment(s
 	return 0;
 
 out_overrun:
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_page_overrun_err(&cc->cc_cid, head->rc_curpage);
+#endif
+	return -EINVAL;
+}
+#else
+/**
+ * svc_rdma_build_read_segment - Build RDMA Read WQEs to pull one RDMA segment
+ * @info: context for ongoing I/O
+ * @segment: co-ordinates of remote memory to be read
+ *
+ * Returns:
+ *   %0: the Read WR chain was constructed successfully
+ *   %-EINVAL: there were not enough rq_pages to finish
+ *   %-ENOMEM: allocating a local resources failed
+ *   %-EIO: a DMA mapping error occurred
+ */
+static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info,
+				       struct svc_rqst *rqstp,
+				       u32 rkey, u32 len, u64 offset)
+{
+	struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
+	struct svc_rdma_chunk_ctxt *cc = &info->ri_cc;
+	unsigned int sge_no, seg_len;
+	struct svc_rdma_rw_ctxt *ctxt;
+	struct scatterlist *sg;
+	int ret;
+
+	sge_no = PAGE_ALIGN(info->ri_pageoff + len) >> PAGE_SHIFT;
+	ctxt = svc_rdma_get_rw_ctxt(cc->cc_rdma, sge_no);
+	if (!ctxt)
+		return -ENOMEM;
+	ctxt->rw_nents = sge_no;
+
+	sg = ctxt->rw_sg_table.sgl;
+	for (sge_no = 0; sge_no < ctxt->rw_nents; sge_no++) {
+		seg_len = min_t(unsigned int, len,
+				PAGE_SIZE - info->ri_pageoff);
+
+		head->rc_arg.pages[info->ri_pageno] =
+			rqstp->rq_pages[info->ri_pageno];
+
+		if (!info->ri_pageoff)
+			head->rc_page_count++;
+
+		sg_set_page(sg, rqstp->rq_pages[info->ri_pageno],
+			    seg_len, info->ri_pageoff);
+		sg = sg_next(sg);
+
+		info->ri_pageoff += seg_len;
+		if (info->ri_pageoff == PAGE_SIZE) {
+			info->ri_pageno++;
+			info->ri_pageoff = 0;
+		}
+		len -= seg_len;
+
+		/* Safety check */
+		if (len &&
+		    &rqstp->rq_pages[info->ri_pageno + 1] > rqstp->rq_page_end)
+			goto out_overrun;
+	}
+
+	ret = svc_rdma_rw_ctx_init(cc->cc_rdma, ctxt, offset, rkey,
+				   DMA_FROM_DEVICE);
+	if (ret < 0)
+		return -EIO;
+	percpu_counter_inc(&svcrdma_stat_read);
+
+	list_add(&ctxt->rw_list, &cc->cc_rwctxts);
+	cc->cc_sqecount += ret;
+	return 0;
+
+out_overrun:
+#ifdef HAVE_TRACE_RPCRDMA_H
+	trace_svcrdma_page_overrun_err(&cc->cc_cid, info->ri_pageno);
+#endif
 	return -EINVAL;
 }
+#endif
 
+#ifdef HAVE_SVC_RDMA_PCL
 /**
  * svc_rdma_build_read_chunk - Build RDMA Read WQEs to pull one RDMA chunk
  * @rqstp: RPC transaction context
@@ -854,7 +1254,35 @@ static int svc_rdma_build_read_chunk(str
 	}
 	return ret;
 }
+#else
+/* Walk the segments in the Read chunk starting at @p and construct
+ * RDMA Read operations to pull the chunk to the server.
+ */
+static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp,
+				     struct svc_rdma_read_info *info,
+				     __be32 *p)
+{
+	int ret;
+
+	ret = -EINVAL;
+	info->ri_chunklen = 0;
+	while (*p++ != xdr_zero && be32_to_cpup(p++) == info->ri_position) {
+		u32 handle, length;
+		u64 offset;
+
+		p = xdr_decode_rdma_segment(p, &handle, &length, &offset);
+		ret = svc_rdma_build_read_segment(info, rqstp, handle, length,
+						  offset);
+		if (ret < 0)
+			break;
+
+		info->ri_chunklen += length;
+	}
+	return ret;
+}
+#endif
 
+#ifdef HAVE_SVC_RDMA_PCL
 /**
  * svc_rdma_copy_inline_range - Copy part of the inline content into pages
  * @rqstp: RPC transaction context
@@ -1178,7 +1606,148 @@ int svc_rdma_process_read_list(struct sv
 	if (ret < 0)
 		return ret;
 
+#ifdef HAVE_TRACE_RPCRDMA_H
 	trace_svcrdma_post_read_chunk(&cc->cc_cid, cc->cc_sqecount);
+#endif
 	ret = svc_rdma_post_chunk_ctxt(rdma, cc);
 	return ret < 0 ? ret : 1;
 }
+#else
+static int svc_rdma_build_normal_read_chunk(struct svc_rqst *rqstp,
+					    struct svc_rdma_read_info *info,
+					    __be32 *p)
+{
+	struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
+	int ret;
+
+	ret = svc_rdma_build_read_chunk(rqstp, info, p);
+	if (ret < 0)
+		goto out;
+
+	/* Split the Receive buffer between the head and tail
+	 * buffers at Read chunk's position. XDR roundup of the
+	 * chunk is not included in either the pagelist or in
+	 * the tail.
+	 */
+	head->rc_hdr_count = 0;
+	head->rc_arg.tail[0].iov_base =
+		head->rc_arg.head[0].iov_base + info->ri_position;
+	head->rc_arg.tail[0].iov_len =
+		head->rc_arg.head[0].iov_len - info->ri_position;
+	head->rc_arg.head[0].iov_len = info->ri_position;
+
+	/* Read chunk may need XDR roundup (see RFC 8166, s. 3.4.5.2).
+	 *
+	 * If the client already rounded up the chunk length, the
+	 * length does not change. Otherwise, the length of the page
+	 * list is increased to include XDR round-up.
+	 *
+	 * Currently these chunks always start at page offset 0,
+	 * thus the rounded-up length never crosses a page boundary.
+	 */
+	info->ri_chunklen = XDR_QUADLEN(info->ri_chunklen) << 2;
+
+	head->rc_arg.page_len = info->ri_chunklen;
+	head->rc_arg.len += info->ri_chunklen;
+	head->rc_arg.buflen += info->ri_chunklen;
+
+out:
+	return ret;
+}
+
+static int svc_rdma_build_pz_read_chunk(struct svc_rqst *rqstp,
+					struct svc_rdma_read_info *info,
+					__be32 *p)
+{
+	struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
+	int ret;
+
+	ret = svc_rdma_build_read_chunk(rqstp, info, p);
+	if (ret < 0)
+		goto out;
+
+	head->rc_arg.len += info->ri_chunklen;
+	head->rc_arg.buflen += info->ri_chunklen;
+
+	head->rc_hdr_count = 1;
+	head->rc_arg.head[0].iov_base = page_address(head->rc_pages[0]);
+	head->rc_arg.head[0].iov_len = min_t(size_t, PAGE_SIZE,
+					     info->ri_chunklen);
+
+	head->rc_arg.page_len = info->ri_chunklen -
+				head->rc_arg.head[0].iov_len;
+
+out:
+	return ret;
+}
+
+/* Pages under I/O have been copied to head->rc_pages. Ensure they
+ * are not released by svc_xprt_release() until the I/O is complete.
+ *
+ * This has to be done after all Read WRs are constructed to properly
+ * handle a page that is part of I/O on behalf of two different RDMA
+ * segments.
+ *
+ * Do this only if I/O has been posted. Otherwise, we do indeed want
+ * svc_xprt_release() to clean things up properly.
+ */
+static void svc_rdma_save_io_pages(struct svc_rqst *rqstp,
+				   const unsigned int start,
+				   const unsigned int num_pages)
+{
+	unsigned int i;
+
+	for (i = start; i < num_pages + start; i++)
+		rqstp->rq_pages[i] = NULL;
+}
+
+int svc_rdma_recv_read_chunk(struct svcxprt_rdma *rdma, struct svc_rqst *rqstp,
+			     struct svc_rdma_recv_ctxt *head, __be32 *p)
+{
+	struct svc_rdma_read_info *info;
+	struct svc_rdma_chunk_ctxt *cc;
+	int ret;
+
+	/* The request (with page list) is constructed in
+	 * head->rc_arg. Pages involved with RDMA Read I/O are
+	 * transferred there.
+	 */
+	head->rc_arg.head[0] = rqstp->rq_arg.head[0];
+	head->rc_arg.tail[0] = rqstp->rq_arg.tail[0];
+	head->rc_arg.pages = head->rc_pages;
+	head->rc_arg.page_base = 0;
+	head->rc_arg.page_len = 0;
+	head->rc_arg.len = rqstp->rq_arg.len;
+	head->rc_arg.buflen = rqstp->rq_arg.buflen;
+
+	info = svc_rdma_read_info_alloc(rdma);
+	if (!info)
+		return -ENOMEM;
+	cc = &info->ri_cc;
+	info->ri_readctxt = head;
+	info->ri_pageno = 0;
+	info->ri_pageoff = 0;
+
+	info->ri_position = be32_to_cpup(p + 1);
+	if (info->ri_position)
+		ret = svc_rdma_build_normal_read_chunk(rqstp, info, p);
+	else
+		ret = svc_rdma_build_pz_read_chunk(rqstp, info, p);
+	if (ret < 0)
+		goto out_err;
+
+#ifdef HAVE_TRACE_RPCRDMA_H
+	trace_svcrdma_post_read_chunk(&cc->cc_cid, cc->cc_sqecount);
+#endif
+	ret = svc_rdma_post_chunk_ctxt(cc->cc_rdma, cc);
+	if (ret < 0)
+		goto out_err;
+
+	svc_rdma_save_io_pages(rqstp, 0, head->rc_page_count);
+	return 1;
+
+out_err:
+	svc_rdma_read_info_free(info);
+	return ret;
+}
+#endif
