From: Valentine Fatiev <valentinef@nvidia.com>
Subject: [PATCH] BACKPORT: drivers/infiniband/core/umem_odp.c

Change-Id: I49156b49f684f3b2ab7ab31f68d0cd8655acec45
---
 drivers/infiniband/core/umem_odp.c | 990 ++++++++++++++++++++++++++++-
 1 file changed, 989 insertions(+), 1 deletion(-)

--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -40,36 +40,441 @@
 #include <linux/vmalloc.h>
 #include <linux/hugetlb.h>
 #include <linux/interval_tree.h>
+#if defined(HAVE_HMM_RANGE_FAULT_SUPPORT)
 #include <linux/hmm.h>
+#endif
 #include <linux/pagemap.h>
 
 #include <rdma/ib_umem_odp.h>
 
 #include "uverbs.h"
 
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp,
 				   const struct mmu_interval_notifier_ops *ops)
+#else
+static void ib_umem_notifier_start_account(struct ib_umem_odp *umem_odp)
+#endif
+{
+#ifndef HAVE_MMU_INTERVAL_NOTIFIER
+	mutex_lock(&umem_odp->umem_mutex);
+	if (umem_odp->notifiers_count++ == 0)
+		/*
+		 * Initialize the completion object for waiting on
+		 * notifiers. Since notifier_count is zero, no one should be
+		 * waiting right now.
+		 */
+		reinit_completion(&umem_odp->notifier_completion);
+	mutex_unlock(&umem_odp->umem_mutex);
+}
+
+static void ib_umem_notifier_end_account(struct ib_umem_odp *umem_odp)
+{
+	mutex_lock(&umem_odp->umem_mutex);
+	/*
+	 * This sequence increase will notify the QP page fault that the page
+	 * that is going to be mapped in the spte could have been freed.
+	 */
+	++umem_odp->notifiers_seq;
+	if (--umem_odp->notifiers_count == 0)
+		complete_all(&umem_odp->notifier_completion);
+	mutex_unlock(&umem_odp->umem_mutex);
+}
+
+static void ib_umem_notifier_release(struct mmu_notifier *mn,
+				     struct mm_struct *mm)
+{
+	struct ib_ucontext_per_mm *per_mm =
+		container_of(mn, struct ib_ucontext_per_mm, mn);
+	struct rb_node *node;
+
+	down_read(&per_mm->umem_rwsem);
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	if (!per_mm->mn.users)
+#else
+	if (!per_mm->active)
+#endif
+		goto out;
+
+	for (node = rb_first_cached(&per_mm->umem_tree); node;
+	     node = rb_next(node)) {
+		struct ib_umem_odp *umem_odp =
+			rb_entry(node, struct ib_umem_odp, interval_tree.rb);
+
+		/*
+		 * Increase the number of notifiers running, to prevent any
+		 * further fault handling on this MR.
+		 */
+		ib_umem_notifier_start_account(umem_odp);
+		complete_all(&umem_odp->notifier_completion);
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+		umem_odp->umem.ibdev->ops.invalidate_range(
+#else
+		umem_odp->umem.context->device->ops.invalidate_range(
+#endif
+			umem_odp, ib_umem_start(umem_odp),
+			ib_umem_end(umem_odp));
+	}
+
+out:
+	up_read(&per_mm->umem_rwsem);
+}
+
+static int invalidate_range_start_trampoline(struct ib_umem_odp *item,
+					     u64 start, u64 end, void *cookie)
+{
+	ib_umem_notifier_start_account(item);
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	item->umem.ibdev->ops.invalidate_range(item, start, end);
+#else
+	item->umem.context->device->ops.invalidate_range(item, start, end);
+#endif
+	return 0;
+}
+
+#ifdef HAVE_MMU_NOTIFIER_RANGE_STRUCT
+static int ib_umem_notifier_invalidate_range_start(struct mmu_notifier *mn,
+				const struct mmu_notifier_range *range)
+#else
+#ifdef HAVE_UMEM_NOTIFIER_PARAM_BLOCKABLE
+ static int ib_umem_notifier_invalidate_range_start(struct mmu_notifier *mn,
+                                                   struct mm_struct *mm,
+                                                   unsigned long start,
+                                                   unsigned long end,
+                                                   bool blockable)
+#else
+static void ib_umem_notifier_invalidate_range_start(struct mmu_notifier *mn,
+                                                   struct mm_struct *mm,
+                                                   unsigned long start,
+                                                   unsigned long end)
+#endif
+#endif /*HAVE_MMU_NOTIFIER_RANGE_STRUCT*/
+{
+	struct ib_ucontext_per_mm *per_mm =
+		container_of(mn, struct ib_ucontext_per_mm, mn);
+
+#if defined(HAVE_UMEM_NOTIFIER_PARAM_BLOCKABLE) || defined(HAVE_MMU_NOTIFIER_RANGE_STRUCT)
+	int rc;
+#ifdef HAVE_MMU_NOTIFIER_RANGE_BLOCKABLE
+	if (mmu_notifier_range_blockable(range))
+#else
+#ifdef HAVE_MMU_NOTIFIER_RANGE_STRUCT
+	if (range->blockable)
+#else
+        if (blockable)
+#endif
+#endif
+		down_read(&per_mm->umem_rwsem);
+	else if (!down_read_trylock(&per_mm->umem_rwsem))
+		return -EAGAIN;
+
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	if (!per_mm->mn.users) {
+#else
+	if (!per_mm->active) {
+#endif
+		up_read(&per_mm->umem_rwsem);
+		/*
+		 * At this point users is permanently zero and visible to this
+		 * CPU without a lock, that fact is relied on to skip the unlock
+		 * in range_end.
+		 */
+		return 0;
+	}
+
+#ifdef HAVE_MMU_NOTIFIER_RANGE_BLOCKABLE
+	rc = rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, range->start,
+					   range->end,
+					   invalidate_range_start_trampoline,
+					   mmu_notifier_range_blockable(range),
+					   NULL);
+	if (rc)
+		up_read(&per_mm->umem_rwsem);
+#else
+#ifdef HAVE_MMU_NOTIFIER_RANGE_STRUCT
+	rc = rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, range->start,
+					    range->end,
+					    invalidate_range_start_trampoline,
+					    range->blockable, NULL);
+#else
+	rc = rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, start, end,
+					    invalidate_range_start_trampoline,
+				             blockable, NULL);
+#endif
+#endif //HAVE_MMU_NOTIFIER_RANGE_BLOCKABLE
+	return rc;
+#else /*defined(HAVE_UMEM_NOTIFIER_PARAM_BLOCKABLE) || defined(HAVE_MMU_NOTIFIER_RANGE_STRUCT)*/ 
+
+	//ib_ucontext_notifier_start_account(context);
+	down_read(&per_mm->umem_rwsem);
+
+	if (!per_mm->active) {
+		up_read(&per_mm->umem_rwsem);
+		/*
+		 * At this point active is permanently set and visible to this
+		 * CPU without a lock, that fact is relied on to skip the unlock
+		 * in range_end.
+		 */
+		return;
+	}
+	rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, start, end,
+			invalidate_range_start_trampoline,
+			NULL);
+
+#endif
+}
+
+static int invalidate_range_end_trampoline(struct ib_umem_odp *item, u64 start,
+					   u64 end, void *cookie)
 {
+	ib_umem_notifier_end_account(item);
+	return 0;
+}
+
+static void ib_umem_notifier_invalidate_range_end(struct mmu_notifier *mn,
+#ifdef HAVE_MMU_NOTIFIER_RANGE_STRUCT
+				const struct mmu_notifier_range *range)
+#else
+ 						  struct mm_struct *mm,
+ 						  unsigned long start,
+ 						  unsigned long end)
+#endif
+{
+	struct ib_ucontext_per_mm *per_mm =
+		container_of(mn, struct ib_ucontext_per_mm, mn);
+
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	if (unlikely(!per_mm->mn.users))
+#else
+	if (unlikely(!per_mm->active))
+#endif
+		return;
+
+	rbt_ib_umem_for_each_in_range(&per_mm->umem_tree,
+#ifdef HAVE_MMU_NOTIFIER_RANGE_STRUCT
+					range->start,
+					range->end,
+#else 
+					 start,
+					 end,
+#endif/*HAVE_MMU_NOTIFIER_RANGE_STRUCT*/
+#if defined(HAVE_UMEM_NOTIFIER_PARAM_BLOCKABLE) || defined(HAVE_MMU_NOTIFIER_RANGE_STRUCT)
+				      invalidate_range_end_trampoline, true, NULL);
+#else
+				      invalidate_range_end_trampoline, NULL);
+#endif/* defined(HAVE_UMEM_NOTIFIER_PARAM_BLOCKABLE) || defined(HAVE_MMU_NOTIFIER_RANGE_STRUCT) */
+	up_read(&per_mm->umem_rwsem);
+}
+
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+static struct mmu_notifier *ib_umem_alloc_notifier(struct mm_struct *mm)
+#else
+static const struct mmu_notifier_ops ib_umem_notifiers = {
+	.release                    = ib_umem_notifier_release,
+	.invalidate_range_start     = ib_umem_notifier_invalidate_range_start,
+	.invalidate_range_end       = ib_umem_notifier_invalidate_range_end,
+};
+
+static void remove_umem_from_per_mm(struct ib_umem_odp *umem_odp)
+{
+	struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
+
+	down_write(&per_mm->umem_rwsem);
+	interval_tree_remove(&umem_odp->interval_tree, &per_mm->umem_tree);
+	complete_all(&umem_odp->notifier_completion);
+	up_write(&per_mm->umem_rwsem);
+}
+
+static struct ib_ucontext_per_mm *alloc_per_mm(struct ib_ucontext *ctx,
+					       struct mm_struct *mm)
+#endif
+{
+	struct ib_ucontext_per_mm *per_mm;
+#ifndef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	int ret;
+#endif
+	per_mm = kzalloc(sizeof(*per_mm), GFP_KERNEL);
+	if (!per_mm)
+		return ERR_PTR(-ENOMEM);
+
+#ifndef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	per_mm->context = ctx;
+	per_mm->mm = mm;
+#endif
+        per_mm->umem_tree = RB_ROOT_CACHED;
+	init_rwsem(&per_mm->umem_rwsem);
+
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	WARN_ON(mm != current->mm);
+#else
+	per_mm->active = true;
+#endif
+	rcu_read_lock();
+	per_mm->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
+	rcu_read_unlock();
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	return &per_mm->mn;
+#else
+	WARN_ON(mm != current->mm);
+
+	per_mm->mn.ops = &ib_umem_notifiers;
+	ret = mmu_notifier_register(&per_mm->mn, per_mm->mm);
+	if (ret) {
+		dev_err(&ctx->device->dev,
+			"Failed to register mmu_notifier %d\n", ret);
+		goto out_pid;
+	}
+
+	list_add(&per_mm->ucontext_list, &ctx->per_mm_list);
+	return per_mm;
+
+out_pid:
+	put_pid(per_mm->tgid);
+	kfree(per_mm);
+	return ERR_PTR(ret);
+}
+
+static struct ib_ucontext_per_mm *get_per_mm(struct ib_umem_odp *umem_odp)
+{
+	struct ib_ucontext *ctx = umem_odp->umem.context;
+	struct ib_ucontext_per_mm *per_mm;
+
+	lockdep_assert_held(&ctx->per_mm_list_lock);
+
+	/*
+	 * Generally speaking we expect only one or two per_mm in this list,
+	 * so no reason to optimize this search today.
+	 */
+	list_for_each_entry(per_mm, &ctx->per_mm_list, ucontext_list) {
+		if (per_mm->mm == umem_odp->umem.owning_mm)
+			return per_mm;
+	}
+
+	return alloc_per_mm(ctx, umem_odp->umem.owning_mm);
+#ifdef HAVE_MMU_NOTIFIER_CALL_SRCU
+}
+
+static void free_per_mm(struct rcu_head *rcu)
+{
+	kfree(container_of(rcu, struct ib_ucontext_per_mm, rcu));
+#endif
+#endif
+}
+
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+static void ib_umem_free_notifier(struct mmu_notifier *mn)
+#else
+static void put_per_mm(struct ib_umem_odp *umem_odp)
+#endif
+{
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	struct ib_ucontext_per_mm *per_mm =
+		container_of(mn, struct ib_ucontext_per_mm, mn);
+#else
+	struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
+	struct ib_ucontext *ctx = umem_odp->umem.context;
+	bool need_free;
+
+	mutex_lock(&ctx->per_mm_list_lock);
+	umem_odp->per_mm = NULL;
+	per_mm->odp_mrs_count--;
+	need_free = per_mm->odp_mrs_count == 0;
+	if (need_free)
+		list_del(&per_mm->ucontext_list);
+	mutex_unlock(&ctx->per_mm_list_lock);
+
+	if (!need_free)
+		return;
+
+	down_write(&per_mm->umem_rwsem);
+	per_mm->active = false;
+	up_write(&per_mm->umem_rwsem);
+#endif
+	WARN_ON(!RB_EMPTY_ROOT(&per_mm->umem_tree.rb_root));
+#ifndef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+#ifdef HAVE_MMU_NOTIFIER_UNREGISTER_NO_RELEASE
+	mmu_notifier_unregister_no_release(&per_mm->mn, per_mm->mm);
+#else
+	mmu_notifier_unregister(&per_mm->mn, per_mm->mm);
+#endif
+#endif
+
+	put_pid(per_mm->tgid);
+#if defined(HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER) || !defined(HAVE_MMU_NOTIFIER_CALL_SRCU)
+	kfree(per_mm);
+#else
+	mmu_notifier_call_srcu(&per_mm->rcu, free_per_mm);
+#endif
+}
+
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+static const struct mmu_notifier_ops ib_umem_notifiers = {
+	.release                    = ib_umem_notifier_release,
+	.invalidate_range_start     = ib_umem_notifier_invalidate_range_start,
+	.invalidate_range_end       = ib_umem_notifier_invalidate_range_end,
+	.alloc_notifier		    = ib_umem_alloc_notifier,
+	.free_notifier		    = ib_umem_free_notifier,
+};
+
+static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp)
+#else
+static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp,
+				   struct ib_ucontext_per_mm *per_mm)
+#endif
+{
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	struct ib_ucontext_per_mm *per_mm;
+	struct mmu_notifier *mn;
+#else
+	struct ib_ucontext *ctx = umem_odp->umem.context;
+#endif
+#endif
 	int ret;
 
 	umem_odp->umem.is_odp = 1;
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 	mutex_init(&umem_odp->umem_mutex);
+#endif
 
 	if (!umem_odp->is_implicit_odp) {
 		size_t page_size = 1UL << umem_odp->page_shift;
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 		unsigned long start;
 		unsigned long end;
+#if defined(HAVE_HMM_RANGE_FAULT_SUPPORT)
 		size_t ndmas, npfns;
+#else
+		size_t pages;
+#endif
+#else
+		size_t pages;
+#endif
 
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 		start = ALIGN_DOWN(umem_odp->umem.address, page_size);
+#else
+		umem_odp->interval_tree.start =
+			ALIGN_DOWN(umem_odp->umem.address, page_size);
+#endif
 		if (check_add_overflow(umem_odp->umem.address,
 				       (unsigned long)umem_odp->umem.length,
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 				       &end))
+#else
+				       &umem_odp->interval_tree.last))
+#endif
 			return -EOVERFLOW;
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 		end = ALIGN(end, page_size);
 		if (unlikely(end < page_size))
+#else
+		umem_odp->interval_tree.last =
+			ALIGN(umem_odp->interval_tree.last, page_size);
+		if (unlikely(umem_odp->interval_tree.last < page_size))
+#endif
 			return -EOVERFLOW;
 
+#if defined(HAVE_MMU_INTERVAL_NOTIFIER) && defined(HAVE_HMM_RANGE_FAULT_SUPPORT)
 		ndmas = (end - start) >> umem_odp->page_shift;
 		if (!ndmas)
 			return -EINVAL;
@@ -102,6 +507,99 @@ out_pfn_list:
 	kvfree(umem_odp->pfn_list);
 	return ret;
 }
+#else
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
+		pages = (end - start) >> umem_odp->page_shift;
+#else
+		pages = (umem_odp->interval_tree.last -
+			 umem_odp->interval_tree.start) >>
+			umem_odp->page_shift;
+#endif
+		if (!pages)
+			return -EINVAL;
+
+		/*
+		 * Note that the representation of the intervals in the
+		 * interval tree considers the ending point as contained in
+		 * the interval.
+		 */
+#ifndef HAVE_MMU_INTERVAL_NOTIFIER
+		umem_odp->interval_tree.last--;
+#endif
+
+		umem_odp->page_list = kvcalloc(
+			pages, sizeof(*umem_odp->page_list), GFP_KERNEL);
+		if (!umem_odp->page_list)
+			return -ENOMEM;
+
+		umem_odp->dma_list = kvcalloc(
+			pages, sizeof(*umem_odp->dma_list), GFP_KERNEL);
+		if (!umem_odp->dma_list) {
+			ret = -ENOMEM;
+			goto out_page_list;
+		}
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
+		ret = mmu_interval_notifier_insert(&umem_odp->notifier,
+						   umem_odp->umem.owning_mm,
+						   start, end - start, ops);
+		if (ret)
+			goto out_dma_list;
+#else
+	}
+
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	mn = mmu_notifier_get(&ib_umem_notifiers, umem_odp->umem.owning_mm);
+	if (IS_ERR(mn)) {
+		ret = PTR_ERR(mn);
+		goto out_dma_list;
+#else
+	mutex_lock(&ctx->per_mm_list_lock);
+	if (!per_mm) {
+		per_mm = get_per_mm(umem_odp);
+		if (IS_ERR(per_mm)) {
+			ret = PTR_ERR(per_mm);
+			goto out_unlock;
+		}
+#endif
+	}
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	umem_odp->per_mm = per_mm =
+		container_of(mn, struct ib_ucontext_per_mm, mn);
+#else
+	umem_odp->per_mm = per_mm;
+	per_mm->odp_mrs_count++;
+	mutex_unlock(&ctx->per_mm_list_lock);
+#endif
+#endif
+#ifndef HAVE_MMU_INTERVAL_NOTIFIER
+	mutex_init(&umem_odp->umem_mutex);
+	init_completion(&umem_odp->notifier_completion);
+
+	if (!umem_odp->is_implicit_odp) {
+		down_write(&per_mm->umem_rwsem);
+		interval_tree_insert(&umem_odp->interval_tree,
+				     &per_mm->umem_tree);
+		up_write(&per_mm->umem_rwsem);
+#endif
+	}
+#ifndef HAVE_MMU_INTERVAL_NOTIFIER
+	mmgrab(umem_odp->umem.owning_mm);
+#endif
+
+	return 0;
+
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+out_dma_list:
+#else
+out_unlock:
+	mutex_unlock(&ctx->per_mm_list_lock);
+#endif
+	kvfree(umem_odp->dma_list);
+out_page_list:
+	kvfree(umem_odp->page_list);
+	return ret;
+}
+#endif
 
 /**
  * ib_umem_odp_alloc_implicit - Allocate a parent implicit ODP umem
@@ -113,9 +611,18 @@ out_pfn_list:
  * @device: IB device to create UMEM
  * @access: ib_reg_mr access flags
  */
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 struct ib_umem_odp *ib_umem_odp_alloc_implicit(struct ib_device *device,
+#else
+struct ib_umem_odp *ib_umem_odp_alloc_implicit(struct ib_udata *udata,
+#endif
 					       int access)
 {
+#ifndef HAVE_MMU_INTERVAL_NOTIFIER
+	struct ib_ucontext *context =
+		container_of(udata, struct uverbs_attr_bundle, driver_udata)
+			->context;
+#endif
 	struct ib_umem *umem;
 	struct ib_umem_odp *umem_odp;
 	int ret;
@@ -123,20 +630,44 @@ struct ib_umem_odp *ib_umem_odp_alloc_im
 	if (access & IB_ACCESS_HUGETLB)
 		return ERR_PTR(-EINVAL);
 
+#ifndef HAVE_MMU_INTERVAL_NOTIFIER
+	if (!context)
+		return ERR_PTR(-EIO);
+	if (WARN_ON_ONCE(!context->device->ops.invalidate_range))
+		return ERR_PTR(-EINVAL);
+#endif
 	umem_odp = kzalloc(sizeof(*umem_odp), GFP_KERNEL);
 	if (!umem_odp)
 		return ERR_PTR(-ENOMEM);
 	umem = &umem_odp->umem;
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 	umem->ibdev = device;
+#else
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	umem->ibdev = context->device;
+#else
+	umem->context = context;
+#endif
+#endif
 	umem->writable = ib_access_writable(access);
 	umem->owning_mm = current->mm;
 	umem_odp->is_implicit_odp = 1;
 	umem_odp->page_shift = PAGE_SHIFT;
 
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 	umem_odp->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
 	ret = ib_init_umem_odp(umem_odp, NULL);
+#else
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	ret = ib_init_umem_odp(umem_odp);
+#else
+	ret = ib_init_umem_odp(umem_odp, NULL);
+#endif
+#endif
 	if (ret) {
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 		put_pid(umem_odp->tgid);
+#endif
 		kfree(umem_odp);
 		return ERR_PTR(ret);
 	}
@@ -154,10 +685,15 @@ EXPORT_SYMBOL(ib_umem_odp_alloc_implicit
  * @size: The length of the userspace VA
  * @ops: MMU interval ops, currently only @invalidate
  */
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 struct ib_umem_odp *
 ib_umem_odp_alloc_child(struct ib_umem_odp *root, unsigned long addr,
 			size_t size,
 			const struct mmu_interval_notifier_ops *ops)
+#else
+struct ib_umem_odp *ib_umem_odp_alloc_child(struct ib_umem_odp *root,
+					    unsigned long addr, size_t size)
+#endif
 {
 	/*
 	 * Caller must ensure that root cannot be freed during the call to
@@ -174,13 +710,19 @@ ib_umem_odp_alloc_child(struct ib_umem_o
 	if (!odp_data)
 		return ERR_PTR(-ENOMEM);
 	umem = &odp_data->umem;
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
 	umem->ibdev = root->umem.ibdev;
+#else
+	umem->context    = root->umem.context;
+#endif
 	umem->length     = size;
 	umem->address    = addr;
 	umem->writable   = root->umem.writable;
 	umem->owning_mm  = root->umem.owning_mm;
 	odp_data->page_shift = PAGE_SHIFT;
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 	odp_data->notifier.ops = ops;
+#endif
 
 	/*
 	 * A mmget must be held when registering a notifier, the owming_mm only
@@ -190,16 +732,25 @@ ib_umem_odp_alloc_child(struct ib_umem_o
 		ret = -EFAULT;
 		goto out_free;
 	}
-
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 	odp_data->tgid = get_pid(root->tgid);
 	ret = ib_init_umem_odp(odp_data, ops);
+#else
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	ret = ib_init_umem_odp(odp_data);
+#else
+	ret = ib_init_umem_odp(odp_data, root->per_mm);
+#endif
+#endif
 	if (ret)
 		goto out_tgid;
 	mmput(umem->owning_mm);
 	return odp_data;
 
 out_tgid:
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 	put_pid(odp_data->tgid);
+#endif
 	mmput(umem->owning_mm);
 out_free:
 	kfree(odp_data);
@@ -220,41 +771,95 @@ EXPORT_SYMBOL(ib_umem_odp_alloc_child);
  * pinning, instead, stores the mm for future page fault handling in
  * conjunction with MMU notifiers.
  */
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 struct ib_umem_odp *ib_umem_odp_get(struct ib_device *device,
 				    unsigned long addr, size_t size, int access,
 				    const struct mmu_interval_notifier_ops *ops)
+#else
+struct ib_umem_odp *ib_umem_odp_get(struct ib_udata *udata, unsigned long addr,
+				    size_t size, int access)
+#endif
 {
 	struct ib_umem_odp *umem_odp;
+#ifndef HAVE_MMU_INTERVAL_NOTIFIER
+	struct ib_ucontext *context;
+#endif
 	int ret;
 
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 	if (WARN_ON_ONCE(!(access & IB_ACCESS_ON_DEMAND)))
+#else
+	if (!udata)
+		return ERR_PTR(-EIO);
+
+	context = container_of(udata, struct uverbs_attr_bundle, driver_udata)
+			  ->context;
+	if (!context)
+		return ERR_PTR(-EIO);
+
+	if (WARN_ON_ONCE(!(access & IB_ACCESS_ON_DEMAND)) ||
+	    WARN_ON_ONCE(!context->device->ops.invalidate_range))
+#endif
 		return ERR_PTR(-EINVAL);
 
 	umem_odp = kzalloc(sizeof(struct ib_umem_odp), GFP_KERNEL);
 	if (!umem_odp)
 		return ERR_PTR(-ENOMEM);
 
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 	umem_odp->umem.ibdev = device;
+#else
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	umem_odp->umem.ibdev = context->device;
+#else
+	umem_odp->umem.context = context;
+#endif
+#endif
 	umem_odp->umem.length = size;
 	umem_odp->umem.address = addr;
 	umem_odp->umem.writable = ib_access_writable(access);
 	umem_odp->umem.owning_mm = current->mm;
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 	umem_odp->notifier.ops = ops;
+#endif
 
 	umem_odp->page_shift = PAGE_SHIFT;
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 #ifdef CONFIG_HUGETLB_PAGE
 	if (access & IB_ACCESS_HUGETLB)
 		umem_odp->page_shift = HPAGE_SHIFT;
 #endif
+#else
+	if (access & IB_ACCESS_HUGETLB) {
+		ret = -EINVAL;
+		goto err_free;
+	}
+#endif
 
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 	umem_odp->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
 	ret = ib_init_umem_odp(umem_odp, ops);
+#else
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	ret = ib_init_umem_odp(umem_odp);
+#else
+	ret = ib_init_umem_odp(umem_odp, NULL);
+#endif
+#endif
 	if (ret)
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 		goto err_put_pid;
+#else
+		goto err_free;
+#endif
 	return umem_odp;
 
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 err_put_pid:
 	put_pid(umem_odp->tgid);
+#else
+err_free:
+#endif
 	kfree(umem_odp);
 	return ERR_PTR(ret);
 }
@@ -262,6 +867,10 @@ EXPORT_SYMBOL(ib_umem_odp_get);
 
 void ib_umem_odp_release(struct ib_umem_odp *umem_odp)
 {
+#if !defined(HAVE_MMU_INTERVAL_NOTIFIER) && defined(HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER)
+	struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
+
+#endif
 	/*
 	 * Ensure that no more pages are mapped in the umem.
 	 *
@@ -273,11 +882,47 @@ void ib_umem_odp_release(struct ib_umem_
 		ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp),
 					    ib_umem_end(umem_odp));
 		mutex_unlock(&umem_odp->umem_mutex);
+#ifndef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+		remove_umem_from_per_mm(umem_odp);
+#endif
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 		mmu_interval_notifier_remove(&umem_odp->notifier);
+#endif
 		kvfree(umem_odp->dma_list);
+#if defined(HAVE_MMU_INTERVAL_NOTIFIER) && defined(HAVE_HMM_RANGE_FAULT_SUPPORT)
 		kvfree(umem_odp->pfn_list);
+#else
+		kvfree(umem_odp->page_list);
+#endif
 	}
+#ifndef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	put_per_mm(umem_odp);
+#endif
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
 	put_pid(umem_odp->tgid);
+#else
+
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	down_write(&per_mm->umem_rwsem);
+	if (!umem_odp->is_implicit_odp) {
+		interval_tree_remove(&umem_odp->interval_tree,
+				     &per_mm->umem_tree);
+		complete_all(&umem_odp->notifier_completion);
+	}
+	/*
+	 * NOTE! mmu_notifier_unregister() can happen between a start/end
+	 * callback, resulting in a missing end, and thus an unbalanced
+	 * lock. This doesn't really matter to us since we are about to kfree
+	 * the memory that holds the lock, however LOCKDEP doesn't like this.
+	 * Thus we call the mmu_notifier_put under the rwsem and test the
+	 * internal users count to reliably see if we are past this point.
+	 */
+	mmu_notifier_put(&per_mm->mn);
+	up_write(&per_mm->umem_rwsem);
+#endif
+
+	mmdrop(umem_odp->umem.owning_mm);
+#endif
 	kfree(umem_odp);
 }
 EXPORT_SYMBOL(ib_umem_odp_release);
@@ -293,6 +938,7 @@ EXPORT_SYMBOL(ib_umem_odp_release);
  * The function returns -EFAULT if the DMA mapping operation fails.
  *
  */
+#if defined(HAVE_MMU_INTERVAL_NOTIFIER) && defined(HAVE_HMM_RANGE_FAULT_SUPPORT)
 static int ib_umem_odp_map_dma_single_page(
 		struct ib_umem_odp *umem_odp,
 		unsigned int dma_index,
@@ -370,7 +1016,11 @@ int ib_umem_odp_map_dma_and_lock(struct
 	 * mmget_not_zero will fail in this case.
 	 */
 	owning_process = get_pid_task(umem_odp->tgid, PIDTYPE_PID);
+#ifdef HAVE_MMPUT_ASYNC_EXPORTED /* Forward port */
 	if (!owning_process || !mmget_not_zero(owning_mm)) {
+#else
+	if (!owning_process) {
+#endif
 		ret = -EINVAL;
 		goto out_put_task;
 	}
@@ -397,10 +1047,19 @@ retry:
 	mmap_read_lock(owning_mm);
 	ret = hmm_range_fault(&range);
 	mmap_read_unlock(owning_mm);
+#ifdef CONFIG_COMPAT_HMM_RANGE_FAULT_RETURNS_INT
 	if (unlikely(ret)) {
 		if (ret == -EBUSY && !time_after(jiffies, timeout))
+#else
+	if (unlikely(ret <= 0)) {
+		if ((ret == 0 || ret == -EBUSY) && !time_after(jiffies, timeout))
+#endif
 			goto retry;
+#ifdef HAVE_MMPUT_ASYNC_EXPORTED /* Forward port */
 		goto out_put_mm;
+#else
+		goto out_put_task;
+#endif
 	}
 
 	start_idx = (range.start - ib_umem_start(umem_odp)) >> page_shift;
@@ -459,23 +1118,306 @@ retry:
 	else
 		mutex_unlock(&umem_odp->umem_mutex);
 
+#ifdef HAVE_MMPUT_ASYNC_EXPORTED
 out_put_mm:
 	mmput_async(owning_mm);
+#endif
 out_put_task:
 	if (owning_process)
 		put_task_struct(owning_process);
 	return ret;
 }
 EXPORT_SYMBOL(ib_umem_odp_map_dma_and_lock);
+#else
+static int ib_umem_odp_map_dma_single_page(
+		struct ib_umem_odp *umem_odp,
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
+		unsigned int page_index,
+#else
+		int page_index,
+#endif
+		struct page *page,
+		u64 access_mask,
+		unsigned long current_seq)
+{
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
+	struct ib_device *dev = umem_odp->umem.ibdev;
+#else
+	struct ib_ucontext *context = umem_odp->umem.context;
+	struct ib_device *dev = context->device;
+#endif
+	dma_addr_t dma_addr;
+	int ret = 0;
+
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
+	if (mmu_interval_check_retry(&umem_odp->notifier, current_seq)) {
+#else
+	if (ib_umem_mmu_notifier_retry(umem_odp, current_seq)) {
+#endif
+		ret = -EAGAIN;
+		goto out;
+	}
+	if (!(umem_odp->dma_list[page_index])) {
+		dma_addr =
+			ib_dma_map_page(dev, page, 0, BIT(umem_odp->page_shift),
+					DMA_BIDIRECTIONAL);
+		if (ib_dma_mapping_error(dev, dma_addr)) {
+			ret = -EFAULT;
+			goto out;
+		}
+		umem_odp->dma_list[page_index] = dma_addr | access_mask;
+		umem_odp->page_list[page_index] = page;
+		umem_odp->npages++;
+	} else if (umem_odp->page_list[page_index] == page) {
+		umem_odp->dma_list[page_index] |= access_mask;
+	} else {
+		/*
+		 * This is a race here where we could have done:
+		 *
+		 *         CPU0                             CPU1
+		 *   get_user_pages()
+		 *                                       invalidate()
+		 *                                       page_fault()
+		 *   mutex_lock(umem_mutex)
+		 *    page from GUP != page in ODP
+		 *
+		 * It should be prevented by the retry test above as reading
+		 * the seq number should be reliable under the
+		 * umem_mutex. Thus something is really not working right if
+		 * things get here.
+		 */
+		WARN(true,
+		     "Got different pages in IB device and from get_user_pages. IB device page: %p, gup page: %p\n",
+		     umem_odp->page_list[page_index], page);
+		ret = -EAGAIN;
+	}
+
+out:
+#ifdef HAVE_PUT_USER_PAGES_DIRTY_LOCK_2_PARAMS
+	put_user_page(page);
+#else
+	put_page(page);
+#endif
+	return ret;
+}
+
+/**
+ * ib_umem_odp_map_dma_pages - Pin and DMA map userspace memory in an ODP MR.
+ *
+ * Pins the range of pages passed in the argument, and maps them to
+ * DMA addresses. The DMA addresses of the mapped pages is updated in
+ * umem_odp->dma_list.
+ *
+ * Returns the number of pages mapped in success, negative error code
+ * for failure.
+ * An -EAGAIN error code is returned when a concurrent mmu notifier prevents
+ * the function from completing its task.
+ * An -ENOENT error code indicates that userspace process is being terminated
+ * and mm was already destroyed.
+ * @umem_odp: the umem to map and pin
+ * @user_virt: the address from which we need to map.
+ * @bcnt: the minimal number of bytes to pin and map. The mapping might be
+ *        bigger due to alignment, and may also be smaller in case of an error
+ *        pinning or mapping a page. The actual pages mapped is returned in
+ *        the return value.
+ * @access_mask: bit mask of the requested access permissions for the given
+ *               range.
+ * @current_seq: the MMU notifiers sequance value for synchronization with
+ *               invalidations. the sequance number is read from
+ *               umem_odp->notifiers_seq before calling this function
+ */
+int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
+			      u64 bcnt, u64 access_mask,
+			      unsigned long current_seq)
+{
+	struct task_struct *owning_process  = NULL;
+	struct mm_struct *owning_mm = umem_odp->umem.owning_mm;
+	struct page       **local_page_list = NULL;
+	u64 page_mask, off;
+	int j, k, ret = 0, start_idx, npages = 0;
+	unsigned int flags = 0, page_shift;
+	phys_addr_t p = 0;
+
+	if (access_mask == 0)
+		return -EINVAL;
+
+	if (user_virt < ib_umem_start(umem_odp) ||
+	    user_virt + bcnt > ib_umem_end(umem_odp))
+		return -EFAULT;
+
+	local_page_list = (struct page **)__get_free_page(GFP_KERNEL);
+	if (!local_page_list)
+		return -ENOMEM;
+
+	page_shift = umem_odp->page_shift;
+	page_mask = ~(BIT(page_shift) - 1);
+	off = user_virt & (~page_mask);
+	user_virt = user_virt & page_mask;
+	bcnt += off; /* Charge for the first page offset as well. */
+
+	/*
+	 * owning_process is allowed to be NULL, this means somehow the mm is
+	 * existing beyond the lifetime of the originating process.. Presumably
+	 * mmget_not_zero will fail in this case.
+	 */
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
+	owning_process = get_pid_task(umem_odp->tgid, PIDTYPE_PID);
+#else
+	owning_process = get_pid_task(umem_odp->per_mm->tgid, PIDTYPE_PID);
+#endif
+	if (!owning_process || !mmget_not_zero(owning_mm)) {
+		ret = -EINVAL;
+		goto out_put_task;
+	}
+
+	if (access_mask & ODP_WRITE_ALLOWED_BIT)
+		flags |= FOLL_WRITE;
+
+	start_idx = (user_virt - ib_umem_start(umem_odp)) >> page_shift;
+	k = start_idx;
+
+	while (bcnt > 0) {
+		const size_t gup_num_pages = min_t(size_t,
+#ifdef HAVE_MMU_INTERVAL_NOTIFIER
+				ALIGN(bcnt, PAGE_SIZE) / PAGE_SIZE,
+#else
+				(bcnt + BIT(page_shift) - 1) >> page_shift,
+#endif
+				PAGE_SIZE / sizeof(struct page *));
+#ifdef HAVE_MMAP_READ_LOCK
+		mmap_read_lock(owning_mm);
+#else
+		down_read(&owning_mm->mmap_sem);
+#endif
+		/*
+		 * Note: this might result in redundent page getting. We can
+		 * avoid this by checking dma_list to be 0 before calling
+		 * get_user_pages. However, this make the code much more
+		 * complex (and doesn't gain us much performance in most use
+		 * cases).
+		 */
+#ifdef HAVE_GET_USER_PAGES_REMOTE_7_PARAMS_AND_SECOND_INT
+		npages = get_user_pages_remote(owning_mm,
+				user_virt, gup_num_pages,
+				flags, local_page_list, NULL, NULL);	
+#elif defined(HAVE_GET_USER_PAGES_REMOTE_8_PARAMS) || defined(HAVE_GET_USER_PAGES_REMOTE_8_PARAMS_W_LOCKED)
+		npages = get_user_pages_remote(owning_process, owning_mm,
+				user_virt, gup_num_pages,
+#ifdef HAVE_GET_USER_PAGES_GUP_FLAGS
+#ifdef HAVE_GET_USER_PAGES_REMOTE_8_PARAMS_W_LOCKED
+				flags, local_page_list, NULL, NULL);
+#else
+				flags, local_page_list, NULL);
+#endif
+#else
+				access_mask & ODP_WRITE_ALLOWED_BIT, 0,
+				local_page_list, NULL);
+#endif
+#else
+		npages = get_user_pages(owning_process, owning_mm,
+				user_virt, gup_num_pages,
+#ifdef HAVE_GET_USER_PAGES_7_PARAMS
+				flags, local_page_list, NULL);
+#else
+				access_mask & ODP_WRITE_ALLOWED_BIT,
+				0, local_page_list, NULL);
+#endif
+#endif
+
+#ifdef HAVE_MMAP_READ_LOCK
+				mmap_read_unlock(owning_mm);
+#else
+				up_read(&owning_mm->mmap_sem);
+#endif
+		if (npages < 0) {
+			if (npages != -EAGAIN)
+				pr_warn("fail to get %zu user pages with error %d\n", gup_num_pages, npages);
+			else
+				pr_debug("fail to get %zu user pages with error %d\n", gup_num_pages, npages);
+			break;
+		}
+
+		bcnt -= min_t(size_t, npages << PAGE_SHIFT, bcnt);
+		mutex_lock(&umem_odp->umem_mutex);
+		for (j = 0; j < npages; j++, user_virt += PAGE_SIZE) {
+			if (user_virt & ~page_mask) {
+				p += PAGE_SIZE;
+				if (page_to_phys(local_page_list[j]) != p) {
+					ret = -EFAULT;
+					break;
+				}
+#ifdef HAVE_PUT_USER_PAGES_DIRTY_LOCK_2_PARAMS
+				put_user_page(local_page_list[j]);
+#else
+				put_page(local_page_list[j]);
+#endif
+				continue;
+			}
+
+			ret = ib_umem_odp_map_dma_single_page(
+					umem_odp, k, local_page_list[j],
+					access_mask, current_seq);
+			if (ret < 0) {
+				if (ret != -EAGAIN)
+					pr_warn("ib_umem_odp_map_dma_single_page failed with error %d\n", ret);
+				else
+					pr_debug("ib_umem_odp_map_dma_single_page failed with error %d\n", ret);
+				break;
+			}
+
+			p = page_to_phys(local_page_list[j]);
+			k++;
+		}
+		mutex_unlock(&umem_odp->umem_mutex);
+
+		if (ret < 0) {
+			/*
+			 * Release pages, remembering that the first page
+			 * to hit an error was already released by
+			 * ib_umem_odp_map_dma_single_page().
+			 */
+			if (npages - (j + 1) > 0)
+				release_pages(&local_page_list[j+1],
+					      npages - (j + 1));
+			break;
+		}
+	}
+
+	if (ret >= 0) {
+		if (npages < 0 && k == start_idx)
+			ret = npages;
+		else
+			ret = k - start_idx;
+	}
+
+#ifdef HAVE_MMPUT_ASYNC_EXPORTED /* Forward port */
+	mmput_async(owning_mm);
+#else
+	mmput(owning_mm);
+#endif
+out_put_task:
+	if (owning_process)
+		put_task_struct(owning_process);
+	free_page((unsigned long)local_page_list);
+	return ret;
+}
+EXPORT_SYMBOL(ib_umem_odp_map_dma_pages);
+#endif
 
 void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt,
 				 u64 bound)
 {
+#if defined(HAVE_MMU_INTERVAL_NOTIFIER) && defined(HAVE_HMM_RANGE_FAULT_SUPPORT)
 	dma_addr_t dma_addr;
 	dma_addr_t dma;
+#endif
 	int idx;
 	u64 addr;
+#ifdef HAVE_MMU_NOTIFIER_OPS_HAS_FREE_NOTIFIER
 	struct ib_device *dev = umem_odp->umem.ibdev;
+#else
+	struct ib_device *dev = umem_odp->umem.context->device;
+#endif
 
 	lockdep_assert_held(&umem_odp->umem_mutex);
 
@@ -483,6 +1425,7 @@ void ib_umem_odp_unmap_dma_pages(struct
 	bound = min_t(u64, bound, ib_umem_end(umem_odp));
 	for (addr = virt; addr < bound; addr += BIT(umem_odp->page_shift)) {
 		idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift;
+#if defined(HAVE_MMU_INTERVAL_NOTIFIER) && defined(HAVE_HMM_RANGE_FAULT_SUPPORT)
 		dma = umem_odp->dma_list[idx];
 
 		/* The access flags guaranteed a valid DMA address in case was NULL */
@@ -491,6 +1434,13 @@ void ib_umem_odp_unmap_dma_pages(struct
 			struct page *page = hmm_pfn_to_page(umem_odp->pfn_list[pfn_idx]);
 
 			dma_addr = dma & ODP_DMA_ADDR_MASK;
+#else
+		if (umem_odp->page_list[idx]) {
+			struct page *page = umem_odp->page_list[idx];
+			dma_addr_t dma = umem_odp->dma_list[idx];
+			dma_addr_t dma_addr = dma & ODP_DMA_ADDR_MASK;
+
+#endif
 			ib_dma_unmap_page(dev, dma_addr,
 					  BIT(umem_odp->page_shift),
 					  DMA_BIDIRECTIONAL);
@@ -507,9 +1457,47 @@ void ib_umem_odp_unmap_dma_pages(struct
 				 */
 				set_page_dirty(head_page);
 			}
+#if ! (defined(HAVE_MMU_INTERVAL_NOTIFIER) && defined(HAVE_HMM_RANGE_FAULT_SUPPORT))
+			umem_odp->page_list[idx] = NULL;
+#endif
 			umem_odp->dma_list[idx] = 0;
 			umem_odp->npages--;
 		}
 	}
 }
 EXPORT_SYMBOL(ib_umem_odp_unmap_dma_pages);
+
+#ifndef HAVE_MMU_INTERVAL_NOTIFIER
+/* @last is not a part of the interval. See comment for function
+ * node_last.
+ */
+int rbt_ib_umem_for_each_in_range(struct rb_root_cached *root,
+				  u64 start, u64 last,
+				  umem_call_back cb,
+#if defined(HAVE_UMEM_NOTIFIER_PARAM_BLOCKABLE) || defined(HAVE_MMU_NOTIFIER_RANGE_STRUCT)
+				  bool blockable,
+#endif
+				  void *cookie)
+{
+	int ret_val = 0;
+	struct interval_tree_node *node, *next;
+	struct ib_umem_odp *umem;
+
+	if (unlikely(start == last))
+		return ret_val;
+
+	for (node = interval_tree_iter_first(root, start, last - 1);
+			node; node = next) {
+		/* TODO move the blockable decision up to the callback */
+#if defined(HAVE_UMEM_NOTIFIER_PARAM_BLOCKABLE) || defined(HAVE_MMU_NOTIFIER_RANGE_STRUCT)
+		if (!blockable)
+			return -EAGAIN;
+#endif
+		next = interval_tree_iter_next(node, start, last - 1);
+		umem = container_of(node, struct ib_umem_odp, interval_tree);
+		ret_val = cb(umem, start, last, cookie) || ret_val;
+	}
+
+	return ret_val;
+}
+#endif
