From: Israel Rukshin <israelr@mellanox.com>
Subject: [PATCH] BACKPORT: drivers/nvme/host/pci.c

Change-Id: I126cff9b8793bd2de2ee0cce424f41fcebe7fb78
---
 drivers/nvme/host/pci.c | 554 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 543 insertions(+), 11 deletions(-)

--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -5,16 +5,20 @@
  */
 
 #include <linux/acpi.h>
+#ifdef HAVE_PCI_ENABLE_PCIE_ERROR_REPORTING
+#include <linux/aer.h>
+#endif
 #include <linux/async.h>
 #include <linux/blkdev.h>
 #include <linux/blk-mq.h>
 #include <linux/blk-mq-pci.h>
+#ifdef HAVE_BLK_INTEGRITY_H
 #include <linux/blk-integrity.h>
+#endif
 #include <linux/dmi.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
-#include <linux/kstrtox.h>
 #include <linux/memremap.h>
 #include <linux/mm.h>
 #include <linux/module.h>
@@ -27,8 +31,14 @@
 #include <linux/types.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
 #include <linux/io-64-nonatomic-hi-lo.h>
+#ifdef HAVE_LINUX_SED_OPAL_H
 #include <linux/sed-opal.h>
+#endif
+#include <linux/sizes.h>
 #include <linux/pci-p2pdma.h>
+#ifdef CONFIG_NVME_POLL
+#include <linux/delay.h>
+#endif
 
 #include "trace.h"
 #include "nvme.h"
@@ -98,6 +108,7 @@ static const struct kernel_param_ops io_
 	.get = param_get_uint,
 };
 
+#ifdef HAVE_BLK_MQ_HCTX_TYPE
 static unsigned int write_queues;
 module_param_cb(write_queues, &io_queue_count_ops, &write_queues, 0644);
 MODULE_PARM_DESC(write_queues,
@@ -107,6 +118,14 @@ MODULE_PARM_DESC(write_queues,
 static unsigned int poll_queues;
 module_param_cb(poll_queues, &io_queue_count_ops, &poll_queues, 0644);
 MODULE_PARM_DESC(poll_queues, "Number of queues to use for polled IO.");
+#else
+static int write_queues = 0;
+MODULE_PARM_DESC(write_queues,
+	"Number of queues to use for writes [deprecated]");
+
+static int poll_queues = 0;
+MODULE_PARM_DESC(poll_queues, "Number of queues to use for polled IO [deprecated]");
+#endif
 
 static bool noacpi;
 module_param(noacpi, bool, 0444);
@@ -147,11 +166,14 @@ struct nvme_dev {
 	struct dma_pool *prp_small_pool;
 	unsigned online_queues;
 	unsigned max_qid;
+#ifdef HAVE_BLK_MQ_HCTX_TYPE
 	unsigned io_queues[HCTX_MAX_TYPES];
+#endif
 	unsigned int num_vecs;
 	u32 q_depth;
 	int io_sqes;
 	u32 db_stride;
+	struct msix_entry *entry;
 	void __iomem *bar;
 	unsigned long bar_mapped_size;
 	struct mutex shutdown_lock;
@@ -186,8 +208,19 @@ struct nvme_dev {
 
 static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
 {
+#ifdef HAVE_PARAM_SET_UINT_MINMAX
 	return param_set_uint_minmax(val, kp, NVME_PCI_MIN_QUEUE_SIZE,
 			NVME_PCI_MAX_QUEUE_SIZE);
+#else
+	int ret;
+	u32 n;
+
+	ret = kstrtou32(val, 10, &n);
+	if (ret != 0 || n < 2)
+		return -EINVAL;
+
+	return param_set_uint(val, kp);
+#endif
 }
 
 static int num_p2p_queues_set(const char *val, const struct kernel_param *kp)
@@ -249,7 +282,7 @@ struct nvme_queue {
 	__le32 *dbbuf_sq_ei;
 	__le32 *dbbuf_cq_ei;
 	struct completion delete_done;
- 
+
 	/* p2p */
  	bool p2p;
 	struct nvme_peer_resource resource;
@@ -269,13 +302,27 @@ union nvme_descriptor {
 struct nvme_iod {
 	struct nvme_request req;
 	struct nvme_command cmd;
+#ifndef HAVE_REQUEST_MQ_HCTX
+	struct nvme_queue *nvmeq;
+#endif
 	bool aborted;
 	s8 nr_allocations;	/* PRP list pool allocations. 0 means small
 				   pool in use */
-	unsigned int dma_len;	/* length of single DMA segment mapping */
+#ifndef HAVE_DMA_PCI_P2PDMA_SUPPORTED
+	int nents;		/* Used in scatterlist */
+#endif
 	dma_addr_t first_dma;
+#if defined(HAVE_BLKDEV_DMA_MAP_BVEC) && defined(HAVE_BLKDEV_REQ_BVEC)
+	unsigned int dma_len;	/* length of single DMA segment mapping */
 	dma_addr_t meta_dma;
+#else
+	struct scatterlist meta_sg;
+#endif
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	struct sg_table sgt;
+#else
+	struct scatterlist *sg;
+#endif
 	union nvme_descriptor list[NVME_MAX_NR_ALLOCATIONS];
 };
 
@@ -289,11 +336,19 @@ static void nvme_peer_init_resource(stru
 
 	if (mask & NVME_PEER_SQT_DBR)
 		/* Calculation from NVMe 1.2.1 SPEC */
+#ifndef CONFIG_PPC
 		nvmeq->resource.sqt_dbr_addr = pci_bus_address(pdev, 0) + (0x1000 + ((2 * (qid)) * (4 << NVME_CAP_STRIDE(dev->ctrl.cap))));
+#else
+		nvmeq->resource.sqt_dbr_addr = 0x800000000000000 | (pci_resource_start(pdev, 0) + (0x1000 + ((2 * (qid)) * (4 << NVME_CAP_STRIDE(dev->ctrl.cap)))));
+#endif
 
 	if (mask & NVME_PEER_CQH_DBR)
 		/* Calculation from NVMe 1.2.1 SPEC */
+#ifndef CONFIG_PPC
 		nvmeq->resource.cqh_dbr_addr = pci_bus_address(pdev, 0) + (0x1000 + ((2 * (qid) + 1) * (4 << NVME_CAP_STRIDE(dev->ctrl.cap))));
+#else
+		nvmeq->resource.cqh_dbr_addr = 0x800000000000000 | (pci_resource_start(pdev, 0) + (0x1000 + ((2 * (qid) + 1) * (4 << NVME_CAP_STRIDE(dev->ctrl.cap)))));
+#endif
 
 	if (mask & NVME_PEER_SQ_PAS)
 		nvmeq->resource.sq_dma_addr = nvmeq->sq_dma_addr;
@@ -578,6 +633,14 @@ static int nvme_pci_init_request(struct
 		unsigned int numa_node)
 {
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+#ifndef HAVE_REQUEST_MQ_HCTX
+	struct nvme_dev *dev = to_nvme_dev(set->driver_data);
+	int queue_idx = (set == &dev->tagset) ? hctx_idx + 1 : 0;
+	struct nvme_queue *nvmeq = &dev->queues[queue_idx];
+
+	BUG_ON(!nvmeq);
+	iod->nvmeq = nvmeq;
+#endif
 
 	nvme_req(req)->ctrl = set->driver_data;
 	nvme_req(req)->cmd = &iod->cmd;
@@ -593,7 +656,12 @@ static int queue_irq_offset(struct nvme_
 	return 0;
 }
 
+#ifdef HAVE_BLK_MQ_HCTX_TYPE
+#ifdef HAVE_BLK_MQ_OPS_MAP_QUEUES_RETURN_INT
+static int nvme_pci_map_queues(struct blk_mq_tag_set *set)
+#else
 static void nvme_pci_map_queues(struct blk_mq_tag_set *set)
+#endif
 {
 	struct nvme_dev *dev = to_nvme_dev(set->driver_data);
 	int i, qoff, offset;
@@ -620,7 +688,24 @@ static void nvme_pci_map_queues(struct b
 		qoff += map->nr_queues;
 		offset += map->nr_queues;
 	}
+
+#ifdef HAVE_BLK_MQ_OPS_MAP_QUEUES_RETURN_INT
+	return 0;
+#endif
 }
+#else
+static int nvme_pci_map_queues(struct blk_mq_tag_set *set)
+{
+	struct nvme_dev *dev = to_nvme_dev(set->driver_data);
+	int offset = queue_irq_offset(dev);
+
+#ifdef HAVE_BLK_MQ_QUEUE_MAP
+	return blk_mq_pci_map_queues(&set->map[0], to_pci_dev(dev->dev), offset);
+#else
+	return blk_mq_pci_map_queues(set, to_pci_dev(dev->dev), offset);
+#endif
+}
+#endif /* HAVE_BLK_MQ_TAG_SET_NR_MAPS */
 
 /*
  * Write sq tail if we are asked to, or if the next command would wrap.
@@ -642,6 +727,9 @@ static inline void nvme_write_sq_db(stru
 	nvmeq->last_sq_tail = nvmeq->sq_tail;
 }
 
+#ifndef absolute_pointer
+#define absolute_pointer(val)  RELOC_HIDE((void *)(val), 0)
+#endif
 static inline void nvme_sq_copy_cmd(struct nvme_queue *nvmeq,
 				    struct nvme_command *cmd)
 {
@@ -651,6 +739,7 @@ static inline void nvme_sq_copy_cmd(stru
 		nvmeq->sq_tail = 0;
 }
 
+#ifdef HAVE_BLK_MQ_OPS_COMMIT_RQS
 static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx)
 {
 	struct nvme_queue *nvmeq = hctx->driver_data;
@@ -660,18 +749,27 @@ static void nvme_commit_rqs(struct blk_m
 		nvme_write_sq_db(nvmeq, true);
 	spin_unlock(&nvmeq->sq_lock);
 }
+#endif
 
 static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req,
 				     int nseg)
 {
+#ifdef HAVE_REQUEST_MQ_HCTX
 	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
+#else
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+#endif
 	unsigned int avg_seg_size;
 
 	avg_seg_size = DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg);
 
 	if (!nvme_ctrl_sgl_supported(&dev->ctrl))
 		return false;
+#ifdef HAVE_REQUEST_MQ_HCTX
 	if (!nvmeq->qid)
+#else
+	if (!iod->nvmeq->qid)
+#endif
 		return false;
 	if (!sgl_threshold || avg_seg_size < sgl_threshold)
 		return false;
@@ -698,16 +796,37 @@ static void nvme_free_prps(struct nvme_d
 #include "nvfs-dma.h"
 #endif
 
+#ifndef HAVE_DMA_PCI_P2PDMA_SUPPORTED
+static void nvme_unmap_sg(struct nvme_dev *dev, struct request *req)
+{
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+
+#ifdef CONFIG_NVFS
+	if (nvme_nvfs_unmap_data(dev, req))
+		return;
+#endif
+
+	if (is_pci_p2pdma_page(sg_page(iod->sg)))
+		pci_p2pdma_unmap_sg(dev->dev, iod->sg, iod->nents,
+				    rq_dma_dir(req));
+	else
+		dma_unmap_sg(dev->dev, iod->sg, iod->nents, rq_dma_dir(req));
+}
+#endif
+
 static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
 {
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 
+#if defined(HAVE_BLKDEV_DMA_MAP_BVEC) && defined(HAVE_BLKDEV_REQ_BVEC)
 	if (iod->dma_len) {
 		dma_unmap_page(dev->dev, iod->first_dma, iod->dma_len,
 			       rq_dma_dir(req));
 		return;
 	}
+#endif
 
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	WARN_ON_ONCE(!iod->sgt.nents);
 
 #ifdef CONFIG_NVFS
@@ -716,6 +835,11 @@ static void nvme_unmap_data(struct nvme_
 #else
 	dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0);
 #endif
+#else
+	WARN_ON_ONCE(!iod->nents);
+
+	nvme_unmap_sg(dev, req);
+#endif
 
 	if (iod->nr_allocations == 0)
 		dma_pool_free(dev->prp_small_pool, iod->list[0].sg_list,
@@ -725,7 +849,11 @@ static void nvme_unmap_data(struct nvme_
 			      iod->first_dma);
 	else
 		nvme_free_prps(dev, req);
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	mempool_free(iod->sgt.sgl, dev->iod_mempool);
+#else
+	mempool_free(iod->sg, dev->iod_mempool);
+#endif
 }
 
 static void nvme_print_sgl(struct scatterlist *sgl, int nents)
@@ -748,7 +876,11 @@ static blk_status_t nvme_pci_setup_prps(
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 	struct dma_pool *pool;
 	int length = blk_rq_payload_bytes(req);
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	struct scatterlist *sg = iod->sgt.sgl;
+#else
+	struct scatterlist *sg = iod->sg;
+#endif
 	int dma_len = sg_dma_len(sg);
 	u64 dma_addr = sg_dma_address(sg);
 	int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1);
@@ -819,16 +951,28 @@ static blk_status_t nvme_pci_setup_prps(
 		dma_len = sg_dma_len(sg);
 	}
 done:
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sgt.sgl));
+#else
+	cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
+#endif
 	cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);
 	return BLK_STS_OK;
 free_prps:
 	nvme_free_prps(dev, req);
 	return BLK_STS_RESOURCE;
 bad_sgl:
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	WARN(DO_ONCE(nvme_print_sgl, iod->sgt.sgl, iod->sgt.nents),
+#else
+	WARN(DO_ONCE(nvme_print_sgl, iod->sg, iod->nents),
+#endif
 			"Invalid SGL for payload:%d nents:%d\n",
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 			blk_rq_payload_bytes(req), iod->sgt.nents);
+#else
+			blk_rq_payload_bytes(req), iod->nents);
+#endif
 	return BLK_STS_IOERR;
 }
 
@@ -849,13 +993,21 @@ static void nvme_pci_sgl_set_seg(struct
 }
 
 static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 		struct request *req, struct nvme_rw_command *cmd)
+#else
+		struct request *req, struct nvme_rw_command *cmd, int entries)
+#endif
 {
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 	struct dma_pool *pool;
 	struct nvme_sgl_desc *sg_list;
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	struct scatterlist *sg = iod->sgt.sgl;
 	unsigned int entries = iod->sgt.nents;
+#else
+	struct scatterlist *sg = iod->sg;
+#endif
 	dma_addr_t sgl_dma;
 	int i = 0;
 
@@ -893,6 +1045,7 @@ static blk_status_t nvme_pci_setup_sgls(
 	return BLK_STS_OK;
 }
 
+#if defined(HAVE_BLKDEV_DMA_MAP_BVEC) && defined(HAVE_BLKDEV_REQ_BVEC)
 static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev,
 		struct request *req, struct nvme_rw_command *cmnd,
 		struct bio_vec *bv)
@@ -931,13 +1084,18 @@ static blk_status_t nvme_setup_sgl_simpl
 	cmnd->dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4;
 	return BLK_STS_OK;
 }
+#endif
 
 static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
 		struct nvme_command *cmnd)
 {
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 	blk_status_t ret = BLK_STS_RESOURCE;
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	int rc;
+#else
+	int nr_mapped;
+#endif
 
 #ifdef CONFIG_NVFS
 	bool is_nvfs_io = false;
@@ -946,8 +1104,11 @@ static blk_status_t nvme_map_data(struct
 		return ret;
 #endif
 
+#if defined(HAVE_BLKDEV_DMA_MAP_BVEC) && defined(HAVE_BLKDEV_REQ_BVEC)
 	if (blk_rq_nr_phys_segments(req) == 1) {
+#ifdef HAVE_REQUEST_MQ_HCTX
 		struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
+#endif
 		struct bio_vec bv = req_bvec(req);
 
 		if (!is_pci_p2pdma_page(bv.bv_page)) {
@@ -956,7 +1117,11 @@ static blk_status_t nvme_map_data(struct
 				return nvme_setup_prp_simple(dev, req,
 							     &cmnd->rw, &bv);
 
+#ifdef HAVE_REQUEST_MQ_HCTX
 			if (nvmeq->qid && sgl_threshold &&
+#else
+			if (iod->nvmeq->qid && sgl_threshold &&
+#endif
 			    nvme_ctrl_sgl_supported(&dev->ctrl))
 				return nvme_setup_sgl_simple(dev, req,
 							     &cmnd->rw, &bv);
@@ -964,6 +1129,8 @@ static blk_status_t nvme_map_data(struct
 	}
 
 	iod->dma_len = 0;
+#endif
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	iod->sgt.sgl = mempool_alloc(dev->iod_mempool, GFP_ATOMIC);
 	if (!iod->sgt.sgl)
 		return BLK_STS_RESOURCE;
@@ -979,19 +1146,53 @@ static blk_status_t nvme_map_data(struct
 			ret = BLK_STS_TARGET;
 		goto out_free_sg;
 	}
+#else
+	iod->sg = mempool_alloc(dev->iod_mempool, GFP_ATOMIC);
+	if (!iod->sg)
+		return BLK_STS_RESOURCE;
+	sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
+	iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
+	if (!iod->nents)
+		goto out_free_sg;
 
+	if (is_pci_p2pdma_page(sg_page(iod->sg)))
+#ifdef HAVE_PCI_P2PDMA_MAP_SG_ATTRS
+		nr_mapped = pci_p2pdma_map_sg_attrs(dev->dev, iod->sg,
+				iod->nents, rq_dma_dir(req), DMA_ATTR_NO_WARN);
+#else
+		nr_mapped = pci_p2pdma_map_sg(dev->dev, iod->sg, iod->nents,
+				rq_dma_dir(req));
+#endif
+	else
+		nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents,
+					     rq_dma_dir(req), DMA_ATTR_NO_WARN);
+	if (!nr_mapped)
+		goto out_free_sg;
+#endif
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	if (nvme_pci_use_sgls(dev, req, iod->sgt.nents))
 		ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw);
+#else
+	if (nvme_pci_use_sgls(dev, req, nr_mapped))
+		ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw, nr_mapped);
+#endif
 	else
 		ret = nvme_pci_setup_prps(dev, req, &cmnd->rw);
 	if (ret != BLK_STS_OK)
 		goto out_unmap_sg;
 	return BLK_STS_OK;
 
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 out_unmap_sg:
 	dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0);
 out_free_sg:
 	mempool_free(iod->sgt.sgl, dev->iod_mempool);
+#else
+out_unmap_sg:
+	nvme_unmap_sg(dev, req);
+out_free_sg:
+	mempool_free(iod->sg, dev->iod_mempool);
+#endif
 	return ret;
 }
 
@@ -999,12 +1200,32 @@ static blk_status_t nvme_map_metadata(st
 		struct nvme_command *cmnd)
 {
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+#ifdef HAVE_RQ_INTEGRITY_RETURN_BIO_VEC
+	struct bio_vec bv = rq_integrity_vec(req);
+#endif
 
+#if defined(HAVE_BLKDEV_DMA_MAP_BVEC) && defined(HAVE_BLKDEV_REQ_BVEC)
+#ifdef HAVE_RQ_INTEGRITY_RETURN_BIO_VEC
+	iod->meta_dma = dma_map_bvec(dev->dev, &bv, rq_dma_dir(req), 0);
+#else
 	iod->meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req),
 			rq_dma_dir(req), 0);
+#endif
 	if (dma_mapping_error(dev->dev, iod->meta_dma))
 		return BLK_STS_IOERR;
 	cmnd->rw.metadata = cpu_to_le64(iod->meta_dma);
+#else
+	if (blk_rq_count_integrity_sg(req->q, req->bio) != 1)
+		return BLK_STS_IOERR;
+
+	sg_init_table(&iod->meta_sg, 1);
+	if (blk_rq_map_integrity_sg(req->q, req->bio, &iod->meta_sg) != 1)
+		return BLK_STS_IOERR;
+
+	if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, rq_dma_dir(req)))
+		return BLK_STS_IOERR;
+	cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg));
+#endif
 	return BLK_STS_OK;
 }
 
@@ -1015,7 +1236,11 @@ static blk_status_t nvme_prep_rq(struct
 
 	iod->aborted = false;
 	iod->nr_allocations = -1;
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	iod->sgt.nents = 0;
+#else
+	iod->nents = 0;
+#endif
 
 	ret = nvme_setup_cmd(req->q->queuedata, req);
 	if (ret)
@@ -1074,6 +1299,7 @@ static blk_status_t nvme_queue_rq(struct
 	return BLK_STS_OK;
 }
 
+#ifdef HAVE_BLK_MQ_OPS_QUEUE_RQS
 static void nvme_submit_cmds(struct nvme_queue *nvmeq, struct request **rqlist)
 {
 	spin_lock(&nvmeq->sq_lock);
@@ -1098,6 +1324,9 @@ static bool nvme_prep_rq_batch(struct nv
 	if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true)))
 		return false;
 
+#ifdef HAVE_RQF_MQ_INFLIGHT
+	req->mq_hctx->tags->rqs[req->tag] = req;
+#endif
 	return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK;
 }
 
@@ -1130,17 +1359,33 @@ static void nvme_queue_rqs(struct reques
 
 	*rqlist = requeue_list;
 }
+#endif /* HAVE_BLK_MQ_OPS_QUEUE_RQS */
 
 static __always_inline void nvme_pci_unmap_rq(struct request *req)
 {
+#ifdef HAVE_REQUEST_MQ_HCTX
 	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 	struct nvme_dev *dev = nvmeq->dev;
 
 	if (blk_integrity_rq(req)) {
 	        struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+#else
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	struct nvme_dev *dev = iod->nvmeq->dev;
+
+	if (blk_integrity_rq(req)) {
+#endif
+#if defined(HAVE_BLKDEV_DMA_MAP_BVEC) && defined(HAVE_BLKDEV_REQ_BVEC)
 
 		dma_unmap_page(dev->dev, iod->meta_dma,
+#ifdef HAVE_RQ_INTEGRITY_RETURN_BIO_VEC
+			       rq_integrity_vec(req).bv_len, rq_dma_dir(req));
+#else
 			       rq_integrity_vec(req)->bv_len, rq_dma_dir(req));
+#endif
+#else
+		dma_unmap_sg(dev->dev, &iod->meta_sg, 1, rq_data_dir(req));
+#endif
 	}
 
 	if (blk_rq_nr_phys_segments(req))
@@ -1153,10 +1398,12 @@ static void nvme_pci_complete_rq(struct
 	nvme_complete_rq(req);
 }
 
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 static void nvme_pci_complete_batch(struct io_comp_batch *iob)
 {
 	nvme_complete_batch(iob, nvme_pci_unmap_rq);
 }
+#endif
 
 /* We read the CQE phase first to check if the rest of the entry is valid */
 static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
@@ -1182,8 +1429,12 @@ static inline struct blk_mq_tags *nvme_q
 	return nvmeq->dev->tagset.tags[nvmeq->qid - 1];
 }
 
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 static inline void nvme_handle_cqe(struct nvme_queue *nvmeq,
 				   struct io_comp_batch *iob, u16 idx)
+#else
+static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx)
+#endif
 {
 	struct nvme_completion *cqe = &nvmeq->cqes[idx];
 	__u16 command_id = READ_ONCE(cqe->command_id);
@@ -1210,9 +1461,13 @@ static inline void nvme_handle_cqe(struc
 	}
 
 	trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail);
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 	if (!nvme_try_complete_req(req, cqe->status, cqe->result) &&
 	    !blk_mq_add_to_batch(req, iob, nvme_req(req)->status,
 					nvme_pci_complete_batch))
+#else
+	if (!nvme_try_complete_req(req, cqe->status, cqe->result))
+#endif
 		nvme_pci_complete_rq(req);
 }
 
@@ -1228,8 +1483,12 @@ static inline void nvme_update_cq_head(s
 	}
 }
 
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 static inline int nvme_poll_cq(struct nvme_queue *nvmeq,
 			       struct io_comp_batch *iob)
+#else
+static inline int nvme_poll_cq(struct nvme_queue *nvmeq)
+#endif
 {
 	int found = 0;
 
@@ -1240,7 +1499,11 @@ static inline int nvme_poll_cq(struct nv
 		 * the cqe requires a full read memory barrier
 		 */
 		dma_rmb();
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 		nvme_handle_cqe(nvmeq, iob, nvmeq->cq_head);
+#else
+		nvme_handle_cqe(nvmeq, nvmeq->cq_head);
+#endif
 		nvme_update_cq_head(nvmeq);
 	}
 
@@ -1252,11 +1515,19 @@ static inline int nvme_poll_cq(struct nv
 static irqreturn_t nvme_irq(int irq, void *data)
 {
 	struct nvme_queue *nvmeq = data;
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 	DEFINE_IO_COMP_BATCH(iob);
+#endif
 
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 	if (nvme_poll_cq(nvmeq, &iob)) {
+#else
+	if (nvme_poll_cq(nvmeq)) {
+#endif
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 		if (!rq_list_empty(iob.req_list))
 			nvme_pci_complete_batch(&iob);
+#endif
 		return IRQ_HANDLED;
 	}
 	return IRQ_NONE;
@@ -1285,11 +1556,23 @@ static void nvme_poll_irqdisable(struct
 	WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags));
 
 	disable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 	nvme_poll_cq(nvmeq, NULL);
+#else
+	nvme_poll_cq(nvmeq);
+#endif
 	enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
 }
 
+#ifdef HAVE_BLK_MQ_OPS_POLL_1_ARG
+static int nvme_poll(struct blk_mq_hw_ctx *hctx)
+#else
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 static int nvme_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
+#else
+static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
+#endif
+#endif
 {
 	struct nvme_queue *nvmeq = hctx->driver_data;
 	bool found;
@@ -1297,9 +1580,29 @@ static int nvme_poll(struct blk_mq_hw_ct
 	if (!nvme_cqe_pending(nvmeq))
 		return 0;
 
-	spin_lock(&nvmeq->cq_poll_lock);
-	found = nvme_poll_cq(nvmeq, iob);
-	spin_unlock(&nvmeq->cq_poll_lock);
+	/*
+	 * For a poll queue we need to protect against the polling thread
+	 * using the CQ lock. For normal interrupt driven threads we have
+	 * to disable the interrupt to avoid racing with it.
+	 * Note: The polling of non-polled queue is not allowed in new kernels.
+	 */
+	if (test_bit(NVMEQ_POLLED, &nvmeq->flags)) {
+		spin_lock(&nvmeq->cq_poll_lock);
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
+		found = nvme_poll_cq(nvmeq, iob);
+#else
+		found = nvme_poll_cq(nvmeq);
+#endif
+		spin_unlock(&nvmeq->cq_poll_lock);
+	} else {
+		disable_irq(pci_irq_vector(to_pci_dev(nvmeq->dev->dev), nvmeq->cq_vector));
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
+		found = nvme_poll_cq(nvmeq, iob);
+#else
+		found = nvme_poll_cq(nvmeq);
+#endif
+		enable_irq(pci_irq_vector(to_pci_dev(nvmeq->dev->dev), nvmeq->cq_vector));
+	}
 
 	return found;
 }
@@ -1391,15 +1694,26 @@ static int adapter_delete_sq(struct nvme
 	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
 }
 
+#ifdef HAVE_RQ_END_IO_RET
 static enum rq_end_io_ret abort_endio(struct request *req, blk_status_t error)
+#else
+static void abort_endio(struct request *req, blk_status_t error)
+#endif
 {
+#ifdef HAVE_REQUEST_MQ_HCTX
 	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
+#else
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	struct nvme_queue *nvmeq = iod->nvmeq;
+#endif
 
 	dev_warn(nvmeq->dev->ctrl.device,
 		 "Abort status: 0x%x", nvme_req(req)->status);
 	atomic_inc(&nvmeq->dev->ctrl.abort_limit);
 	blk_mq_free_request(req);
+#ifdef HAVE_RQ_END_IO_RET
 	return RQ_END_IO_NONE;
+#endif
 }
 
 static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
@@ -1453,10 +1767,18 @@ static void nvme_warn_reset(struct nvme_
 		 "Try \"nvme_core.default_ps_max_latency_us=0 pcie_aspm=off\" and report a bug\n");
 }
 
+#ifdef HAVE_BLK_MQ_OPS_TIMEOUT_1_PARAM
 static enum blk_eh_timer_return nvme_timeout(struct request *req)
+#else
+static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
+#endif
 {
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+#ifdef HAVE_REQUEST_MQ_HCTX
 	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
+#else
+	struct nvme_queue *nvmeq = iod->nvmeq;
+#endif
 	struct nvme_dev *dev = nvmeq->dev;
 	struct request *abort_req;
 	struct nvme_command cmd = { };
@@ -1484,12 +1806,35 @@ static enum blk_eh_timer_return nvme_tim
 	/*
 	 * Did we miss an interrupt?
 	 */
+#ifdef HAVE_REQUEST_MQ_HCTX
 	if (test_bit(NVMEQ_POLLED, &nvmeq->flags))
+#ifdef HAVE_BLK_MQ_OPS_POLL_1_ARG
+		nvme_poll(req->mq_hctx);
+#else
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 		nvme_poll(req->mq_hctx, NULL);
+#else
+		nvme_poll(req->mq_hctx, req->tag);
+#endif
+#endif /* HAVE_BLK_MQ_OPS_POLL_1_ARG */
+#else
+	if (test_bit(NVMEQ_POLLED, &nvmeq->flags)) {
+		if (nvme_cqe_pending(nvmeq)) {
+			spin_lock(&nvmeq->cq_poll_lock);
+			nvme_poll_cq(nvmeq);
+			spin_unlock(&nvmeq->cq_poll_lock);
+		}
+	}
+#endif
 	else
 		nvme_poll_irqdisable(nvmeq);
 
+#ifdef HAVE_BLK_MQ_RQ_STATE
 	if (blk_mq_rq_state(req) != MQ_RQ_IN_FLIGHT) {
+#else
+	if (READ_ONCE(req->state) != MQ_RQ_IN_FLIGHT) {
+#endif
+
 		dev_warn(dev->ctrl.device,
 			 "I/O tag %d (%04x) QID %d timeout, completion polled\n",
 			 req->tag, nvme_cid(req), nvmeq->qid);
@@ -1544,12 +1889,19 @@ static enum blk_eh_timer_return nvme_tim
 	cmd.abort.cid = nvme_cid(req);
 	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
 
+#ifdef HAVE_BLK_OP_STR
 	dev_warn(nvmeq->dev->ctrl.device,
 		 "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, aborting req_op:%s(%u) size:%u\n",
 		 req->tag, nvme_cid(req), opcode, nvme_get_opcode_str(opcode),
 		 nvmeq->qid, blk_op_str(req_op(req)), req_op(req),
 		 blk_rq_bytes(req));
-
+#else
+	        dev_warn(nvmeq->dev->ctrl.device,
+			 "I/O %d (%s) QID %d timeout, aborting\n",
+			 req->tag,
+			 nvme_get_opcode_str(nvme_req(req)->cmd->common.opcode),
+			 nvmeq->qid);
+#endif
 	abort_req = blk_mq_alloc_request(dev->ctrl.admin_q, nvme_req_op(&cmd),
 					 BLK_MQ_REQ_NOWAIT);
 	if (IS_ERR(abort_req)) {
@@ -1558,9 +1910,21 @@ static enum blk_eh_timer_return nvme_tim
 	}
 	nvme_init_request(abort_req, &cmd);
 
-	abort_req->end_io = abort_endio;
 	abort_req->end_io_data = NULL;
+#ifdef HAVE_BLK_EXECUTE_RQ_NOWAIT_2_PARAM
+	abort_req->end_io = abort_endio;
 	blk_execute_rq_nowait(abort_req, false);
+#else
+#ifdef HAVE_BLK_EXECUTE_RQ_NOWAIT_5_PARAM
+	blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
+#else
+#ifdef HAVE_BLK_EXECUTE_RQ_NOWAIT_3_PARAM
+	blk_execute_rq_nowait(abort_req, false, abort_endio);
+#else
+	blk_execute_rq_nowait(NULL, abort_req, 0, abort_endio);
+#endif
+#endif
+#endif
 
 	/*
 	 * The aborted req will be completed on receiving the abort req.
@@ -1656,7 +2020,11 @@ static void nvme_reap_pending_cqes(struc
 		if (dev->queues[i].p2p)
 			continue;
 		spin_lock(&dev->queues[i].cq_poll_lock);
+#ifdef HAVE_BLK_MQ_OPS_POLL_2_ARG
 		nvme_poll_cq(&dev->queues[i], NULL);
+#else
+		nvme_poll_cq(&dev->queues[i]);
+#endif
 		spin_unlock(&dev->queues[i].cq_poll_lock);
 	}
 }
@@ -1868,9 +2236,13 @@ static const struct blk_mq_ops nvme_mq_a
 
 static const struct blk_mq_ops nvme_mq_ops = {
 	.queue_rq	= nvme_queue_rq,
+#ifdef HAVE_BLK_MQ_OPS_QUEUE_RQS
 	.queue_rqs	= nvme_queue_rqs,
+#endif
 	.complete	= nvme_pci_complete_rq,
+#ifdef HAVE_BLK_MQ_OPS_COMMIT_RQS
 	.commit_rqs	= nvme_commit_rqs,
+#endif
 	.init_hctx	= nvme_init_hctx,
 	.init_request	= nvme_pci_init_request,
 	.map_queues	= nvme_pci_map_queues,
@@ -1988,12 +2360,16 @@ static int nvme_create_io_queues(struct
 	}
 
 	max = min(dev->max_qid, dev->ctrl.queue_count - 1);
+#ifdef HAVE_BLK_MQ_HCTX_TYPE
 	if (max != 1 && dev->io_queues[HCTX_TYPE_POLL]) {
 		rw_queues = dev->io_queues[HCTX_TYPE_DEFAULT] +
 				dev->io_queues[HCTX_TYPE_READ];
 	} else {
 		rw_queues = max;
 	}
+#else
+	rw_queues = max;
+#endif
 
 	for (i = dev->online_queues; i <= max; i++) {
 		bool polled = i > rw_queues && !dev->queues[i].p2p;
@@ -2385,6 +2761,7 @@ static void nvme_update_attrs(struct nvm
 	sysfs_update_group(&dev->ctrl.device->kobj, &nvme_pci_dev_attrs_group);
 }
 
+#ifdef HAVE_IRQ_AFFINITY_PRIV
 /*
  * nirqs is the number of interrupts available for write and read
  * queues. The core already reserved an interrupt for the admin queue.
@@ -2422,14 +2799,17 @@ static void nvme_calc_irq_sets(struct ir
 	affd->set_size[HCTX_TYPE_READ] = nr_read_queues;
 	affd->nr_sets = nr_read_queues ? 2 : 1;
 }
+#endif
 
 static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues)
 {
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
 	struct irq_affinity affd = {
 		.pre_vectors	= 1,
+#ifdef HAVE_IRQ_AFFINITY_PRIV
 		.calc_sets	= nvme_calc_irq_sets,
 		.priv		= dev,
+#endif
 	};
 	unsigned int irq_queues, poll_queues;
 	unsigned int flags = PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY;
@@ -2439,6 +2819,7 @@ static int nvme_setup_irqs(struct nvme_d
 	 * left over for non-polled I/O.
 	 */
 	poll_queues = min(dev->nr_poll_queues, nr_io_queues - 1);
+#ifdef HAVE_BLK_MQ_HCTX_TYPE
 	dev->io_queues[HCTX_TYPE_POLL] = poll_queues;
 
 	/*
@@ -2447,6 +2828,7 @@ static int nvme_setup_irqs(struct nvme_d
 	 */
 	dev->io_queues[HCTX_TYPE_DEFAULT] = 1;
 	dev->io_queues[HCTX_TYPE_READ] = 0;
+#endif
 
 	/*
 	 * We need interrupts for the admin queue and each non-polled I/O queue,
@@ -2563,7 +2945,11 @@ static int nvme_setup_io_queues(struct n
 
 	dev->num_vecs = result;
 	result = max(result - 1 + dev->num_p2p_queues, 1u);
+#ifdef HAVE_BLK_MQ_HCTX_TYPE
 	dev->max_qid = result + dev->io_queues[HCTX_TYPE_POLL];
+#else
+	dev->max_qid = result;
+#endif
 
 	/*
 	 * Should investigate if there's a performance win from allocating
@@ -2590,35 +2976,51 @@ static int nvme_setup_io_queues(struct n
 		nvme_suspend_io_queues(dev);
 		goto retry;
 	}
+#ifdef HAVE_BLK_MQ_HCTX_TYPE
 	dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n",
 					dev->io_queues[HCTX_TYPE_DEFAULT],
 					dev->io_queues[HCTX_TYPE_READ],
 					dev->io_queues[HCTX_TYPE_POLL]);
+#endif
 	return 0;
 out_unlock:
 	mutex_unlock(&dev->shutdown_lock);
 	return result;
 }
 
+#ifdef HAVE_RQ_END_IO_RET
 static enum rq_end_io_ret nvme_del_queue_end(struct request *req,
 					     blk_status_t error)
+#else
+static void nvme_del_queue_end(struct request *req, blk_status_t error)
+#endif
 {
 	struct nvme_queue *nvmeq = req->end_io_data;
 
 	blk_mq_free_request(req);
 	complete(&nvmeq->delete_done);
+#ifdef HAVE_RQ_END_IO_RET
 	return RQ_END_IO_NONE;
+#endif
 }
 
+#ifdef HAVE_RQ_END_IO_RET
 static enum rq_end_io_ret nvme_del_cq_end(struct request *req,
 					  blk_status_t error)
+#else
+static void nvme_del_cq_end(struct request *req, blk_status_t error)
+#endif
 {
 	struct nvme_queue *nvmeq = req->end_io_data;
 
 	if (error)
 		set_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags);
 
+#ifdef HAVE_RQ_END_IO_RET
 	return nvme_del_queue_end(req, error);
+#else
+	nvme_del_queue_end(req, error);
+#endif
 }
 
 static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
@@ -2635,14 +3037,28 @@ static int nvme_delete_queue(struct nvme
 		return PTR_ERR(req);
 	nvme_init_request(req, &cmd);
 
+	req->end_io_data = nvmeq;
+
+	init_completion(&nvmeq->delete_done);
+#ifdef HAVE_BLK_EXECUTE_RQ_NOWAIT_2_PARAM
 	if (opcode == nvme_admin_delete_cq)
 		req->end_io = nvme_del_cq_end;
 	else
 		req->end_io = nvme_del_queue_end;
-	req->end_io_data = nvmeq;
-
-	init_completion(&nvmeq->delete_done);
 	blk_execute_rq_nowait(req, false);
+#else
+#ifdef HAVE_BLK_EXECUTE_RQ_NOWAIT_5_PARAM
+	blk_execute_rq_nowait(q, NULL, req, false,
+#else
+#ifdef HAVE_BLK_EXECUTE_RQ_NOWAIT_3_PARAM
+	blk_execute_rq_nowait(req, false,
+#else
+	blk_execute_rq_nowait(NULL, req, false,
+#endif
+#endif
+			opcode == nvme_admin_delete_cq ?
+				nvme_del_cq_end : nvme_del_queue_end);
+#endif
 	return 0;
 }
 
@@ -2682,10 +3098,12 @@ static void nvme_delete_io_queues(struct
 
 static unsigned int nvme_pci_nr_maps(struct nvme_dev *dev)
 {
+#ifdef HAVE_BLK_MQ_HCTX_TYPE
 	if (dev->io_queues[HCTX_TYPE_POLL])
 		return 3;
 	if (dev->io_queues[HCTX_TYPE_READ])
 		return 2;
+#endif
 	return 1;
 }
 
@@ -2771,6 +3189,9 @@ static int nvme_pci_enable(struct nvme_d
 
 	nvme_map_cmb(dev);
 
+#ifdef HAVE_PCI_ENABLE_PCIE_ERROR_REPORTING
+ 	pci_enable_pcie_error_reporting(pdev);
+#endif
 	pci_save_state(pdev);
 
 	result = nvme_pci_configure_admin_queue(dev);
@@ -2835,8 +3256,15 @@ static void nvme_dev_disable(struct nvme
 	nvme_suspend_io_queues(dev);
 	nvme_suspend_queue(dev, 0);
 	pci_free_irq_vectors(pdev);
+#ifdef HAVE_PCI_ENABLE_PCIE_ERROR_REPORTING
+	if (pci_is_enabled(pdev)) {
+		pci_disable_pcie_error_reporting(pdev);
+		pci_disable_device(pdev);
+	}
+#else
 	if (pci_is_enabled(pdev))
 		pci_disable_device(pdev);
+#endif
 	nvme_reap_pending_cqes(dev);
 
 	nvme_cancel_tagset(&dev->ctrl);
@@ -2912,8 +3340,17 @@ static void nvme_pci_free_ctrl(struct nv
 {
 	struct nvme_dev *dev = to_nvme_dev(ctrl);
 
+#ifdef CONFIG_NVME_POLL
+	dev->ctrl.run_poll_work = false;
+	cancel_delayed_work_sync(&ctrl->poll_work);
+#endif
+
 	nvme_free_tagset(dev);
 	put_device(dev->dev);
+#ifndef HAVE_BLK_MQ_WAIT_QUIESCE_DONE_TAGSET
+	if (dev->ctrl.admin_q)
+		blk_put_queue(dev->ctrl.admin_q);
+#endif
 	kfree(dev->queues);
 	kfree(dev);
 }
@@ -2962,6 +3399,7 @@ static void nvme_reset_work(struct work_
 	if (result)
 		goto out;
 
+
 	nvme_dbbuf_dma_alloc(dev);
 
 	result = nvme_setup_host_mem(dev);
@@ -3022,6 +3460,29 @@ static void nvme_reset_work(struct work_
 	nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
 }
 
+#ifdef CONFIG_NVME_POLL
+static void nvme_poll_work(struct work_struct *work)
+{
+	struct nvme_dev *dev =
+		container_of(to_delayed_work(work), struct nvme_dev, ctrl.poll_work);
+	int i;
+
+	for (i = 0; i < dev->ctrl.queue_count; i++) {
+		struct nvme_queue *nvmeq = &dev->queues[i];
+
+		if (test_bit(NVMEQ_POLLED, &nvmeq->flags))
+			continue;
+
+		if (test_bit(NVMEQ_ENABLED, &nvmeq->flags))
+			nvme_poll_irqdisable(nvmeq);
+	}
+
+	usleep_range(1000, 2000);
+	if (dev->ctrl.run_poll_work)
+		queue_delayed_work(nvme_wq, &dev->ctrl.poll_work, 0);
+}
+#endif
+
 static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
 {
 	*val = readl(to_nvme_dev(ctrl)->bar + off);
@@ -3061,17 +3522,24 @@ static void nvme_pci_print_device_info(s
 		subsys->firmware_rev);
 }
 
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 static bool nvme_pci_supports_pci_p2pdma(struct nvme_ctrl *ctrl)
 {
 	struct nvme_dev *dev = to_nvme_dev(ctrl);
 
 	return dma_pci_p2pdma_supported(dev->dev);
 }
+#endif
 
 static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
 	.name			= "pcie",
 	.module			= THIS_MODULE,
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	.flags			= NVME_F_METADATA_SUPPORTED,
+#else
+	.flags			= NVME_F_METADATA_SUPPORTED|
+					  NVME_F_PCI_P2PDMA,
+#endif
 	.dev_attr_groups	= nvme_pci_dev_attr_groups,
 	.reg_read32		= nvme_pci_reg_read32,
 	.reg_write32		= nvme_pci_reg_write32,
@@ -3080,7 +3548,9 @@ static const struct nvme_ctrl_ops nvme_p
 	.submit_async_event	= nvme_pci_submit_async_event,
 	.get_address		= nvme_pci_get_address,
 	.print_device_info	= nvme_pci_print_device_info,
+#ifdef HAVE_DMA_PCI_P2PDMA_SUPPORTED
 	.supports_pci_p2pdma	= nvme_pci_supports_pci_p2pdma,
+#endif
 };
 
 struct pci_dev *nvme_find_pdev_from_bdev(struct block_device *bdev)
@@ -3187,6 +3657,13 @@ static unsigned long check_vendor_combin
 	return 0;
 }
 
+#ifndef HAVE_ACPI_STORAGE_D3
+static inline bool acpi_storage_d3(struct device *dev)
+{
+	return false;
+}
+#endif
+
 static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev,
 		const struct pci_device_id *id)
 {
@@ -3198,7 +3675,11 @@ static struct nvme_dev *nvme_pci_alloc_d
 	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
 	if (!dev)
 		return ERR_PTR(-ENOMEM);
+
 	INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
+#ifdef CONFIG_NVME_POLL
+	INIT_DELAYED_WORK(&dev->ctrl.poll_work, nvme_poll_work);
+#endif
 	mutex_init(&dev->shutdown_lock);
 
 	dev->nr_write_queues = write_queues;
@@ -3233,15 +3714,24 @@ static struct nvme_dev *nvme_pci_alloc_d
 		dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
 	else
 		dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+#ifdef HAVE_DMA_SET_MIN_ALIGN_MASK
 	dma_set_min_align_mask(&pdev->dev, NVME_CTRL_PAGE_SIZE - 1);
+#endif
 	dma_set_max_seg_size(&pdev->dev, 0xffffffff);
 
 	/*
 	 * Limit the max command size to prevent iod->sg allocations going
 	 * over a single page.
 	 */
+#ifdef HAVE_DMA_OPT_MAPPING_SIZE
 	dev->ctrl.max_hw_sectors = min_t(u32,
 		NVME_MAX_KB_SZ << 1, dma_opt_mapping_size(&pdev->dev) >> 9);
+#elif defined HAVE_DMA_MAX_MAPPING_SIZE
+	dev->ctrl.max_hw_sectors = min_t(u32,
+		NVME_MAX_KB_SZ << 1, dma_max_mapping_size(&pdev->dev) >> 9);
+#else
+	dev->ctrl.max_hw_sectors = NVME_MAX_KB_SZ << 1;
+#endif
 	dev->ctrl.max_segments = NVME_MAX_SEGS;
 
 	/*
@@ -3249,6 +3739,13 @@ static struct nvme_dev *nvme_pci_alloc_d
 	 * a single integrity segment for the separate metadata pointer.
 	 */
 	dev->ctrl.max_integrity_segments = 1;
+
+#ifdef CONFIG_NVME_POLL
+	dev->ctrl.run_poll_work = true;
+	queue_delayed_work(nvme_wq, &dev->ctrl.poll_work,
+			   DIV_ROUND_UP(HZ, 1000));
+#endif
+
 	return dev;
 
 out_put_device:
@@ -3639,8 +4136,10 @@ static const struct pci_device_id nvme_i
 		.driver_data = NVME_QUIRK_IDENTIFY_CNS |
 				NVME_QUIRK_DISABLE_WRITE_ZEROES |
 				NVME_QUIRK_BOGUS_NID, },
+#ifdef HAVE_PCI_VENDOR_ID_REDHAT
 	{ PCI_VDEVICE(REDHAT, 0x0010),	/* Qemu emulated controller */
 		.driver_data = NVME_QUIRK_BOGUS_NID, },
+#endif
 	{ PCI_DEVICE(0x126f, 0x2262),	/* Silicon Motion generic */
 		.driver_data = NVME_QUIRK_NO_DEEPEST_PS |
 				NVME_QUIRK_BOGUS_NID, },
@@ -3788,6 +4287,26 @@ static const struct pci_device_id nvme_i
 };
 MODULE_DEVICE_TABLE(pci, nvme_id_table);
 
+#ifndef PCI_SRIOV_CONFIGURE_SIMPLE
+static int nvme_pci_sriov_configure(struct pci_dev *pdev, int numvfs)
+{
+	int ret = 0;
+
+	if (numvfs == 0) {
+		if (pci_vfs_assigned(pdev)) {
+			dev_warn(&pdev->dev,
+				 "Cannot disable SR-IOV VFs while assigned\n");
+			return -EPERM;
+		}
+		pci_disable_sriov(pdev);
+		return 0;
+	}
+
+	ret = pci_enable_sriov(pdev, numvfs);
+	return ret ? ret : numvfs;
+}
+#endif
+
 static struct pci_driver nvme_driver = {
 	.name		= "nvme",
 	.id_table	= nvme_id_table,
@@ -3800,7 +4319,11 @@ static struct pci_driver nvme_driver = {
 		.pm		= &nvme_dev_pm_ops,
 #endif
 	},
+#ifdef PCI_SRIOV_CONFIGURE_SIMPLE
 	.sriov_configure = pci_sriov_configure_simple,
+#else
+	.sriov_configure = nvme_pci_sriov_configure,
+#endif
 	.err_handler	= &nvme_err_handler,
 };
 
@@ -3831,10 +4354,16 @@ EXPORT_SYMBOL_GPL(nvme_pdev_admin_passth
 
 static int __init nvme_init(void)
 {
+#ifdef CONFIG_NVME_POLL
+	pr_info("Loading nvme MOFED driver\n");
+#endif
+
 	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
+#ifdef HAVE_IRQ_AFFINITY_PRIV
 	BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2);
+#endif
 	BUILD_BUG_ON(NVME_MAX_SEGS > SGES_PER_PAGE);
 	BUILD_BUG_ON(sizeof(struct scatterlist) * NVME_MAX_SEGS > PAGE_SIZE);
 	BUILD_BUG_ON(nvme_pci_npages_prp() > NVME_MAX_NR_ALLOCATIONS);
@@ -3850,6 +4379,9 @@ static void __exit nvme_exit(void)
 
 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
 MODULE_LICENSE("GPL");
+#ifdef RETPOLINE_MLNX
+MODULE_INFO(retpoline, "Y");
+#endif
 MODULE_VERSION("1.0");
 MODULE_DESCRIPTION("NVMe host PCIe transport driver");
 module_init(nvme_init);
