provider/irdma: Implement CoCo buffer APIs

CMakeLists.txt

@@ -88,7 +88,7 @@ set(PACKAGE_VERSION "64.0")
 # When this is changed the values in these files need changing too:
 #   debian/control
 #   debian/libibverbs1.symbols
-set(IBVERBS_PABI_VERSION "59")
+set(IBVERBS_PABI_VERSION "64")
 set(IBVERBS_PROVIDER_SUFFIX "-rdmav${IBVERBS_PABI_VERSION}.so")
 
 #-------------------------
@@ -499,6 +499,8 @@ if (NOT HAVE_SPARSE)
   RDMA_DoFixup("${HAVE_STDATOMIC}" "stdatomic.h")
 endif()
 
+CHECK_INCLUDE_FILE("linux/dma-heap.h" HAVE_LINUX_DMA_HEAP_H)
+
 RDMA_Check_SSE(HAVE_TARGET_SSE)
 RDMA_Check_LS64(HAVE_LS64)
 

buildlib/config.h.in

@@ -52,6 +52,8 @@
 
 #cmakedefine HAVE_WORKING_IF_H 1
 
+#cmakedefine HAVE_LINUX_DMA_HEAP_H 1
+
 #cmakedefine HAVE_SOCKADDR_ARG_AS_UNION 1
 
 // Operating mode for symbol versions

debian/control

@@ -151,7 +151,7 @@ Section: libs
 Pre-Depends: ${misc:Pre-Depends}
 Depends: adduser, ${misc:Depends}, ${shlibs:Depends}
 Recommends: ibverbs-providers
-Breaks: ibverbs-providers (<< 59~)
+Breaks: ibverbs-providers (<< 64~)
 Description: Library for direct userspace use of RDMA (InfiniBand/iWARP)
  libibverbs is a library that allows userspace processes to use RDMA
  "verbs" as described in the InfiniBand Architecture Specification and

debian/libibverbs1.symbols

@@ -14,13 +14,15 @@ libibverbs.so.1 libibverbs1 #MINVER#
  IBVERBS_1.14@IBVERBS_1.14 36
  IBVERBS_1.15@IBVERBS_1.15 59
  IBVERBS_1.16@IBVERBS_1.16 62
- (symver)IBVERBS_PRIVATE_59 59
+ IBVERBS_1.17@IBVERBS_1.17 64
+ (symver)IBVERBS_PRIVATE_64 64
  _ibv_query_gid_ex@IBVERBS_1.11 32
  _ibv_query_gid_table@IBVERBS_1.11 32
  ibv_ack_async_event@IBVERBS_1.0 1.1.6
  ibv_ack_async_event@IBVERBS_1.1 1.1.6
  ibv_ack_cq_events@IBVERBS_1.0 1.1.6
  ibv_ack_cq_events@IBVERBS_1.1 1.1.6
+ ibv_alloc_buf@IBVERBS_1.17 64
  ibv_alloc_dmah@IBVERBS_1.15 59
  ibv_alloc_pd@IBVERBS_1.0 1.1.6
  ibv_alloc_pd@IBVERBS_1.1 1.1.6
@@ -63,6 +65,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_dontfork_range@IBVERBS_1.1 1.1.6
  ibv_event_type_str@IBVERBS_1.1 1.1.6
  ibv_fork_init@IBVERBS_1.1 1.1.6
+ ibv_free_buf@IBVERBS_1.17 64
  ibv_free_device_list@IBVERBS_1.0 1.1.6
  ibv_free_device_list@IBVERBS_1.1 1.1.6
  ibv_get_async_event@IBVERBS_1.0 1.1.6
@@ -111,6 +114,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_rate_to_mbps@IBVERBS_1.1 1.1.8
  ibv_rate_to_mult@IBVERBS_1.0 1.1.6
  ibv_read_sysfs_file@IBVERBS_1.0 1.1.6
+ ibv_reg_buf_mr@IBVERBS_1.17 64
  ibv_reg_dmabuf_mr@IBVERBS_1.12 34
  ibv_reg_mr@IBVERBS_1.0 1.1.6
  ibv_reg_mr@IBVERBS_1.1 1.1.6

kernel-headers/rdma/ib_user_ioctl_cmds.h

@@ -117,6 +117,7 @@ enum uverbs_attrs_create_cq_cmd_attr_ids {
 	UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH,
 	UVERBS_ATTR_CREATE_CQ_BUFFER_FD,
 	UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET,
+	UVERBS_ATTR_CREATE_CQ_BUF_UMEM,
 };
 
 enum uverbs_attrs_destroy_cq_cmd_attr_ids {
@@ -158,6 +159,9 @@ enum uverbs_attrs_create_qp_cmd_attr_ids {
 	UVERBS_ATTR_CREATE_QP_EVENT_FD,
 	UVERBS_ATTR_CREATE_QP_RESP_CAP,
 	UVERBS_ATTR_CREATE_QP_RESP_QP_NUM,
+	UVERBS_ATTR_CREATE_QP_BUF_UMEM,
+	UVERBS_ATTR_CREATE_QP_RQ_BUF_UMEM,
+	UVERBS_ATTR_CREATE_QP_SQ_BUF_UMEM,
 };
 
 enum uverbs_attrs_destroy_qp_cmd_attr_ids {

kernel-headers/rdma/ib_user_ioctl_verbs.h

@@ -273,4 +273,31 @@ struct ib_uverbs_gid_entry {
 	__u32 netdev_ifindex; /* It is 0 if there is no netdev associated with it */
 };
 
+enum ib_uverbs_buffer_type {
+	IB_UVERBS_BUFFER_TYPE_DMABUF,
+	IB_UVERBS_BUFFER_TYPE_VA,
+};
+
+/*
+ * Describes a single buffer backed by dma-buf or user virtual address.
+ * Used as the payload of a per-attribute UVERBS_ATTR_UMEM-typed attribute.
+ *
+ * @type: buffer type from enum ib_uverbs_buffer_type
+ * @fd: dma-buf file descriptor (valid for IB_UVERBS_BUFFER_TYPE_DMABUF)
+ * @flags: required flags; the kernel rejects the call with -EINVAL if any
+ *         bit is not understood. No bits are defined yet.
+ * @optional_flags: advisory flags; bits the kernel does not understand are
+ *                  silently ignored. No bits are defined yet.
+ * @addr: offset within dma-buf, or user virtual address for VA
+ * @length: buffer length in bytes
+ */
+struct ib_uverbs_buffer_desc {
+	__u32 type;
+	__s32 fd;
+	__u32 flags;
+	__u32 optional_flags;
+	__aligned_u64 addr;
+	__aligned_u64 length;
+};
+
 #endif

kernel-headers/rdma/ib_user_verbs.h

@@ -1368,6 +1368,8 @@ enum ib_uverbs_device_cap_flags {
 	IB_UVERBS_DEVICE_FLUSH_PERSISTENT = 1ULL << 39,
 	/* Atomic write attributes */
 	IB_UVERBS_DEVICE_ATOMIC_WRITE = 1ULL << 40,
+	/* CoCo guest with DMA bounce buffering required */
+	IB_UVERBS_DEVICE_CC_DMA_BOUNCE = 1ULL << 41,
 };
 
 enum ib_uverbs_raw_packet_caps {

kernel-headers/rdma/irdma-abi.h

@@ -73,11 +73,19 @@ struct irdma_resize_cq_req {
 struct irdma_create_cq_req {
 	__aligned_u64 user_cq_buf;
 	__aligned_u64 user_shadow_area;
+	__s32 cq_dmabuf_fd;
+	__s32 shadow_dmabuf_fd;
+	__u8 is_cq_dmabuf;
+	__u8 is_shadow_dmabuf;
+	__u8 rsvd[6];
 };
 
 struct irdma_create_srq_req {
 	__aligned_u64 user_srq_buf;
 	__aligned_u64 user_shadow_area;
+	__s32 srq_dmabuf_fd;
+	__u8 is_srq_dmabuf;
+	__u8 rsvd[3];
 };
 
 struct irdma_create_srq_resp {
@@ -88,6 +96,9 @@ struct irdma_create_srq_resp {
 struct irdma_create_qp_req {
 	__aligned_u64 user_wqe_bufs;
 	__aligned_u64 user_compl_ctx;
+	__s32 sq_dmabuf_fd;
+	__u8 is_sq_dmabuf;
+	__u8 rsvd[3];
 };
 
 struct irdma_mem_reg_req {

kernel-headers/rdma/mlx5_user_ioctl_cmds.h

@@ -274,6 +274,11 @@ enum mlx5_ib_device_query_context_attrs {
 
 enum mlx5_ib_create_cq_attrs {
 	MLX5_IB_ATTR_CREATE_CQ_UAR_INDEX = UVERBS_ID_DRIVER_NS_WITH_UHW,
+	MLX5_IB_ATTR_CREATE_CQ_DBR_BUF_UMEM,
+};
+
+enum mlx5_ib_create_qp_attrs {
+	MLX5_IB_ATTR_CREATE_QP_DBR_BUF_UMEM = UVERBS_ID_DRIVER_NS_WITH_UHW,
 };
 
 enum mlx5_ib_reg_dmabuf_mr_attrs {

libibverbs/CMakeLists.txt

@@ -11,6 +11,7 @@ publish_headers(infiniband
 publish_internal_headers(infiniband
   cmd_ioctl.h
   cmd_write.h
+  dmabuf_heap.h
   driver.h
   kern-abi.h
   marshall.h
@@ -21,7 +22,7 @@ configure_file("libibverbs.map.in"
 
 rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map"
   # See Documentation/versioning.md
-  1 1.16.${PACKAGE_VERSION}
+  1 1.17.${PACKAGE_VERSION}
   all_providers.c
   cmd.c
   cmd_ah.c
@@ -31,6 +32,7 @@ rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map"
   cmd_dm.c
   cmd_dmabuf.c
   cmd_dmah.c
+  dmabuf_heap.c
   cmd_fallback.c
   cmd_flow.c
   cmd_flow_action.c

libibverbs/cmd_mr.c

@@ -160,8 +160,8 @@ int ibv_cmd_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset, size_t length,
 int ibv_cmd_reg_mr_ex(struct ibv_pd *pd, struct verbs_mr *vmr,
 		      struct ibv_mr_init_attr *mr_init_attr)
 {
-	DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_MR,
-			       UVERBS_METHOD_REG_MR, 11);
+	DECLARE_FBCMD_BUFFER(cmdb, UVERBS_OBJECT_MR,
+			     UVERBS_METHOD_REG_MR, 11, NULL);
 	bool fd_based = (mr_init_attr->comp_mask & IBV_REG_MR_MASK_FD);
 	struct ib_uverbs_attr *handle;
 	uint64_t length = mr_init_attr->length;
@@ -222,9 +222,43 @@ int ibv_cmd_reg_mr_ex(struct ibv_pd *pd, struct verbs_mr *vmr,
 		fill_attr_in_obj(cmdb, UVERBS_ATTR_REG_MR_DMA_HANDLE,
 				 verbs_get_dmah(mr_init_attr->dmah)->handle);
 
-	ret = execute_ioctl(pd->context, cmdb);
-	if (ret)
-		return errno;
+	switch (execute_ioctl_fallback(pd->context, reg_mr_ex, cmdb, &ret)) {
+	case SUCCESS:
+		break;
+
+	case TRY_WRITE:
+	case TRY_WRITE_EX: {
+		/*
+		 * The kernel has no REG_MR ioctl method.  Only an address-based
+		 * request maps to the legacy write-ABI REG_MR command; a
+		 * dma-buf (fd) or DMA-handle request has no write-ABI
+		 * equivalent (and such a kernel could not service it anyway).
+		 * This mirrors ibv_cmd_reg_mr()'s write path so an ordinary
+		 * buffer registered via ibv_reg_mr_ex() -- e.g. through
+		 * ibv_reg_buf_mr() -- keeps working on a write-ABI-only kernel.
+		 */
+		struct ib_uverbs_reg_mr_resp wresp;
+		struct ibv_reg_mr req;
+		uint64_t hca_va;
+
+		if (fd_based ||
+		    (mr_init_attr->comp_mask & IBV_REG_MR_MASK_DMAH)) {
+			errno = EOPNOTSUPP;
+			return EOPNOTSUPP;
+		}
+
+		hca_va = (mr_init_attr->comp_mask & IBV_REG_MR_MASK_IOVA) ?
+				 mr_init_attr->iova :
+				 (uintptr_t)mr_init_attr->addr;
+
+		return ibv_cmd_reg_mr(pd, mr_init_attr->addr, length, hca_va,
+				      mr_init_attr->access, vmr, &req,
+				      sizeof(req), &wresp, sizeof(wresp));
+	}
+
+	default:
+		return ret;
+	}
 
 	vmr->ibv_mr.handle = read_attr_obj(UVERBS_ATTR_REG_MR_HANDLE,
 					   handle);
@@ -235,6 +269,7 @@ int ibv_cmd_reg_mr_ex(struct ibv_pd *pd, struct verbs_mr *vmr,
 		vmr->mr_type = IBV_MR_TYPE_DMABUF_MR;
 	else
 		vmr->mr_type = IBV_MR_TYPE_MR;
+	vmr->access = mr_init_attr->access;
 
 	return 0;
 }

libibverbs/cmd_qp.c

@@ -425,11 +425,12 @@ int ibv_cmd_create_qp_ex2(struct ibv_context *context,
 			  struct ibv_create_qp_ex *cmd,
 			  size_t cmd_size,
 			  struct ib_uverbs_ex_create_qp_resp *resp,
-			  size_t resp_size)
+			  size_t resp_size,
+			  struct ibv_command_buffer *driver)
 {
-	DECLARE_CMD_BUFFER_COMPAT(cmdb, UVERBS_OBJECT_QP,
-				  UVERBS_METHOD_QP_CREATE, cmd, cmd_size, resp,
-				  resp_size);
+	DECLARE_CMD_BUFFER_LINK_COMPAT(cmdb, UVERBS_OBJECT_QP,
+				       UVERBS_METHOD_QP_CREATE,
+				       driver, cmd, cmd_size, resp, resp_size);
 
 	if (!check_comp_mask(attr_ex->comp_mask,
 			     IBV_QP_INIT_ATTR_PD |