efa: Add option to create CQ with external memory

Extend the EFA direct verbs interface to enable creation of CQs on top
of pre-allocated memory buffers. The memory can be passed by supplying
a dmabuf fd and offset.

Reviewed-by: Daniel Kranzdorf <[email protected]>
Reviewed-by: Yonatan Nachum <[email protected]>
Signed-off-by: Michael Margolin <[email protected]>

Author: mrgolin

kernel-headers/rdma/efa-abi.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */
 /*
- * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef EFA_ABI_USER_H
@@ -56,14 +56,18 @@ struct efa_ibv_alloc_pd_resp {
 enum {
 	EFA_CREATE_CQ_WITH_COMPLETION_CHANNEL = 1 << 0,
 	EFA_CREATE_CQ_WITH_SGID               = 1 << 1,
+	EFA_CREATE_CQ_WITH_EXT_MEM_DMABUF     = 1 << 2,
 };
 
 struct efa_ibv_create_cq {
 	__u32 comp_mask;
 	__u32 cq_entry_size;
 	__u16 num_sub_cqs;
 	__u8 flags;
-	__u8 reserved_58[5];
+	__u8 reserved_58[1];
+	__u32 ext_mem_fd;
+	__aligned_u64 ext_mem_offset;
+	__aligned_u64 ext_mem_length;
 };
 
 enum {
@@ -131,6 +135,7 @@ enum {
 	EFA_QUERY_DEVICE_CAPS_DATA_POLLING_128 = 1 << 4,
 	EFA_QUERY_DEVICE_CAPS_RDMA_WRITE = 1 << 5,
 	EFA_QUERY_DEVICE_CAPS_UNSOLICITED_WRITE_RECV = 1 << 6,
+	EFA_QUERY_DEVICE_CAPS_CQ_WITH_EXT_MEM_DMABUF = 1 << 7,
 };
 
 struct efa_ibv_ex_query_device_resp {

providers/efa/efadv.h

@@ -22,6 +22,7 @@ enum {
 	EFADV_DEVICE_ATTR_CAPS_CQ_WITH_SGID = 1 << 2,
 	EFADV_DEVICE_ATTR_CAPS_RDMA_WRITE = 1 << 3,
 	EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV = 1 << 4,
+	EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF = 1 << 5,
 };
 
 struct efadv_device_attr {
@@ -98,9 +99,20 @@ enum {
 	EFADV_WC_EX_WITH_IS_UNSOLICITED = 1 << 1,
 };
 
+enum {
+	EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF = 1 << 0,
+};
+
 struct efadv_cq_init_attr {
 	uint64_t comp_mask;
 	uint64_t wc_flags;
+	uint64_t flags;
+	struct {
+		uint64_t offset;
+		uint64_t length;
+		uint32_t fd;
+		uint8_t reserved[4];
+	} ext_mem_dmabuf;
 };
 
 struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,

providers/efa/man/efadv_create_cq.3.md

@@ -47,6 +47,13 @@ Compatibility is handled using the comp_mask and inlen fields.
 struct efadv_cq_init_attr {
 	uint64_t comp_mask;
 	uint64_t wc_flags;
+	uint64_t flags;
+	struct {
+		uint64_t offset;
+		uint64_t length;
+		uint32_t fd;
+		uint8_t reserved[4];
+	} ext_mem_dmabuf;
 };
 ```
 
@@ -65,6 +72,25 @@ struct efadv_cq_init_attr {
 	EFADV_WC_EX_WITH_IS_UNSOLICITED:
 		request for an option to check whether a receive WC is unsolicited.
 
+*flags*
+:       A bitwise OR of the various values described below.
+
+	EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF:
+		create CQ with external memory provided via dmabuf.
+
+*ext_mem_dmabuf*
+:       Structure containing information about external memory when using
+	EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF flag.
+
+	fd:
+		file descriptor of the dmabuf.
+
+	offset:
+		offset within the dmabuf.
+
+	length:
+		length of the memory region to use.
+
 
 # Completion iterator functions
 

providers/efa/man/efadv_query_device.3.md

@@ -85,6 +85,10 @@ struct efadv_device_attr {
 		requests in order to receive RDMA write with immediate and a WC generated for such
 		receive will be marked as unsolicited.
 
+	EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF:
+		Indicates that creating CQs with external memory buffers by passing dmabuf is
+		supported.
+
 *max_rdma_size*
 :	Maximum RDMA transfer size in bytes.
 

providers/efa/verbs.c

@@ -175,6 +175,9 @@ int efadv_query_device(struct ibv_context *ibvctx,
 
 		if (EFA_DEV_CAP(ctx, UNSOLICITED_WRITE_RECV))
 			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_UNSOLICITED_WRITE_RECV;
+
+		if (EFA_DEV_CAP(ctx, CQ_WITH_EXT_MEM_DMABUF))
+			attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_CQ_WITH_EXT_MEM_DMABUF;
 	}
 
 	if (vext_field_avail(typeof(*attr), max_rdma_size, inlen)) {
@@ -873,9 +876,9 @@ static void efa_cq_fill_pfns(struct efa_cq *cq,
 	if (attr->wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS)
 		ibvcqx->read_dlid_path_bits = efa_wc_read_dlid_path_bits;
 
-	if (efa_attr && (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID))
+	if (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID)
 		cq->dv_cq.wc_read_sgid = efa_wc_read_sgid;
-	if (efa_attr && (efa_attr->wc_flags & EFADV_WC_EX_WITH_IS_UNSOLICITED))
+	if (efa_attr->wc_flags & EFADV_WC_EX_WITH_IS_UNSOLICITED)
 		cq->dv_cq.wc_is_unsolicited = efa_wc_is_unsolicited;
 }
 
@@ -925,12 +928,20 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
 	if (!cq)
 		return NULL;
 
-	if (efa_attr && (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID))
+	if (efa_attr->wc_flags & EFADV_WC_EX_WITH_SGID)
 		cmd.flags |= EFA_CREATE_CQ_WITH_SGID;
 
 	num_sub_cqs = ctx->sub_cqs_per_cq;
 	cmd.num_sub_cqs = num_sub_cqs;
 	cmd.cq_entry_size = cqe_size;
+
+	if (efa_attr->flags & EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF) {
+		cmd.ext_mem_fd = efa_attr->ext_mem_dmabuf.fd;
+		cmd.ext_mem_offset = efa_attr->ext_mem_dmabuf.offset;
+		cmd.ext_mem_length = efa_attr->ext_mem_dmabuf.length;
+		cmd.flags |= EFA_CREATE_CQ_WITH_EXT_MEM_DMABUF;
+	}
+
 	if (attr->channel)
 		cmd.flags |= EFA_CREATE_CQ_WITH_COMPLETION_CHANNEL;
 
@@ -950,17 +961,18 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
 	cq->cqe_size = cqe_size;
 	cq->dev = ibvctx->device;
 
-	cq->buf = mmap(NULL, cq->buf_size, PROT_READ, MAP_SHARED,
-		       ibvctx->cmd_fd, resp.q_mmap_key);
-	if (cq->buf == MAP_FAILED)
-		goto err_destroy_cq;
-
-	buf = cq->buf;
-	sub_buf_size = cq->cqe_size * sub_cq_size;
-	for (i = 0; i < num_sub_cqs; i++) {
-		efa_sub_cq_initialize(&cq->sub_cq_arr[i], buf, sub_cq_size,
-				      cq->cqe_size);
-		buf += sub_buf_size;
+	if (!(efa_attr->flags & EFADV_CQ_INIT_FLAGS_EXT_MEM_DMABUF)) {
+		cq->buf = mmap(NULL, cq->buf_size, PROT_READ, MAP_SHARED, ibvctx->cmd_fd,
+			       resp.q_mmap_key);
+		if (cq->buf == MAP_FAILED)
+			goto err_destroy_cq;
+
+		buf = cq->buf;
+		sub_buf_size = cq->cqe_size * sub_cq_size;
+		for (i = 0; i < num_sub_cqs; i++) {
+			efa_sub_cq_initialize(&cq->sub_cq_arr[i], buf, sub_cq_size, cq->cqe_size);
+			buf += sub_buf_size;
+		}
 	}
 
 	if (resp.comp_mask & EFA_CREATE_CQ_RESP_DB_OFF) {
@@ -991,29 +1003,33 @@ static struct ibv_cq_ex *create_cq(struct ibv_context *ibvctx,
 struct ibv_cq *efa_create_cq(struct ibv_context *ibvctx, int ncqe,
 			     struct ibv_comp_channel *channel, int vec)
 {
+	struct efadv_cq_init_attr efa_attr = {};
 	struct ibv_cq_init_attr_ex attr_ex = {
 		.cqe = ncqe,
 		.channel = channel,
 		.comp_vector = vec
 	};
 	struct ibv_cq_ex *ibvcqx;
 
-	ibvcqx = create_cq(ibvctx, &attr_ex, NULL);
+	ibvcqx = create_cq(ibvctx, &attr_ex, &efa_attr);
 
 	return ibvcqx ? ibv_cq_ex_to_cq(ibvcqx) : NULL;
 }
 
 struct ibv_cq_ex *efa_create_cq_ex(struct ibv_context *ibvctx,
 				   struct ibv_cq_init_attr_ex *attr_ex)
 {
-	return create_cq(ibvctx, attr_ex, NULL);
+	struct efadv_cq_init_attr efa_attr = {};
+
+	return create_cq(ibvctx, attr_ex, &efa_attr);
 }
 
 struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,
 				  struct ibv_cq_init_attr_ex *attr_ex,
 				  struct efadv_cq_init_attr *efa_attr,
 				  uint32_t inlen)
 {
+	struct efadv_cq_init_attr local_efa_attr = {};
 	uint64_t supp_wc_flags = 0;
 	struct efa_context *ctx;
 
@@ -1043,7 +1059,8 @@ struct ibv_cq_ex *efadv_create_cq(struct ibv_context *ibvctx,
 		return NULL;
 	}
 
-	return create_cq(ibvctx, attr_ex, efa_attr);
+	memcpy(&local_efa_attr, efa_attr, min_t(uint32_t, inlen, sizeof(local_efa_attr)));
+	return create_cq(ibvctx, attr_ex, &local_efa_attr);
 }
 
 int efadv_query_cq(struct ibv_cq *ibvcq, struct efadv_cq_attr *attr, uint32_t inlen)