verbs: Allow query only device support for QP data in order

Add a flag to query directly the RDMA device support for QP
data-in-order semantics without enforcing host CPU architecture
restrictions. It is particularly useful in scenarios where the GPU
performs data polling directly, with the application responsible for
ensuring the GPU side support for data-in-order semantics.

Reviewed-by: Michael Margolin <[email protected]>
Reviewed-by: Yonatan Nachum <[email protected]>
Signed-off-by: Daniel Kranzdorf <[email protected]>

Author: dkkranz

libibverbs/man/ibv_query_qp_data_in_order.3.md

@@ -45,6 +45,10 @@ This function describes ordering at the receiving side of the QP, not the sendin
 
 IBV_QUERY_QP_DATA_IN_ORDER_RETURN_CAPS - Query for supported capabilities and return a capabilities vector.
 
+IBV_QUERY_QP_DATA_IN_ORDER_DEVICE_ONLY - Allows querying only the RDMA device side support for data-in-order semantics
+    without enforcing host CPU architecture restrictions. It is the responsibility of the application
+    to ensure data-in-order semantics support for the host CPU or receiver device.
+
 Passing 0 is equivalent to using IBV_QUERY_QP_DATA_IN_ORDER_RETURN_CAPS and checking for IBV_QUERY_QP_DATA_IN_ORDER_WHOLE_MSG support.
 
 # RETURN VALUE

libibverbs/verbs.c

@@ -695,23 +695,31 @@ LATEST_SYMVER_FUNC(ibv_query_qp, 1_1, "IBVERBS_1.1",
 int ibv_query_qp_data_in_order(struct ibv_qp *qp, enum ibv_wr_opcode op,
 			       uint32_t flags)
 {
+	int result;
+
+	if (!check_comp_mask(flags,
+			     IBV_QUERY_QP_DATA_IN_ORDER_RETURN_CAPS |
+			     IBV_QUERY_QP_DATA_IN_ORDER_DEVICE_ONLY))
+		return 0;
+
 #if !defined(__i386__) && !defined(__x86_64__)
 	/* Currently this API is only supported for x86 architectures since most
 	 * non-x86 platforms are known to be OOO and need to do a per-platform study.
+	 * However, it is possible to override this restriction to allow querying
+	 * the device capability directly, regardless of the host CPU architecture.
 	 */
-	return 0;
-#else
-	int result;
-
-	if (!check_comp_mask(flags, IBV_QUERY_QP_DATA_IN_ORDER_RETURN_CAPS))
+	if (!(flags & IBV_QUERY_QP_DATA_IN_ORDER_DEVICE_ONLY))
 		return 0;
+#endif
 
 	result = get_ops(qp->context)->query_qp_data_in_order(qp, op, flags);
 	if (result & IBV_QUERY_QP_DATA_IN_ORDER_WHOLE_MSG)
 		result |= IBV_QUERY_QP_DATA_IN_ORDER_ALIGNED_128_BYTES;
 
-	return flags ? result : !!(result & IBV_QUERY_QP_DATA_IN_ORDER_WHOLE_MSG);
-#endif
+	if (flags & IBV_QUERY_QP_DATA_IN_ORDER_RETURN_CAPS)
+		return result;
+
+	return !!(result & IBV_QUERY_QP_DATA_IN_ORDER_WHOLE_MSG);
 }
 
 LATEST_SYMVER_FUNC(ibv_modify_qp, 1_1, "IBVERBS_1.1",

libibverbs/verbs.h

@@ -1042,6 +1042,7 @@ enum ibv_qp_attr_mask {
 
 enum ibv_query_qp_data_in_order_flags {
 	IBV_QUERY_QP_DATA_IN_ORDER_RETURN_CAPS = 1 << 0,
+	IBV_QUERY_QP_DATA_IN_ORDER_DEVICE_ONLY = 1 << 1,
 };
 
 enum ibv_query_qp_data_in_order_caps {