libvfio-user: include libvfio-user as a submodule with SPDK

Also add a CONFIG_VFIO_USER config flag, it's enabled by
default.

Change-Id: I18b44c024a264516a60f743d5c366a4c7f7c6785
Signed-off-by: Changpeng Liu <[email protected]>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5000
Tested-by: SPDK CI Jenkins <[email protected]>
Reviewed-by: Shuhei Matsumoto <[email protected]>
Reviewed-by: Jim Harris <[email protected]>

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "ocf"]
 	path = ocf
 	url = https://github.com/Open-CAS/ocf.git
+[submodule "libvfio-user"]
+	path = libvfio-user
+	url = https://github.com/nutanix/libvfio-user.git

CONFIG

@@ -117,6 +117,9 @@ CONFIG_VHOST=y
 # Build vhost initiator (Virtio) driver.
 CONFIG_VIRTIO=y
 
+# Build NVMf custom vfio-user target.
+CONFIG_VFIO_USER=y
+
 # Build with PMDK backends
 CONFIG_PMDK=n
 CONFIG_PMDK_DIR=

Makefile

@@ -45,6 +45,7 @@ DIRS-$(CONFIG_EXAMPLES) += examples
 DIRS-y += test
 DIRS-$(CONFIG_IPSEC_MB) += ipsecbuild
 DIRS-$(CONFIG_ISAL) += isalbuild
+DIRS-$(CONFIG_VFIO_USER) += vfiouserbuild
 
 .PHONY: all clean $(DIRS-y) include/spdk/config.h mk/config.mk \
 	cc_version cxx_version .libs_only_other .ldflags ldflags install \
@@ -78,6 +79,11 @@ LIB += isalbuild
 DPDK_DEPS += isalbuild
 endif
 
+ifeq ($(CONFIG_VFIO_USER),y)
+VFIOUSERBUILD = vfiouserbuild
+LIB += vfiouserbuild
+endif
+
 all: mk/cc.mk $(DIRS-y)
 clean: $(DIRS-y)
 	$(Q)rm -f include/spdk/config.h
@@ -97,7 +103,7 @@ ifneq ($(SKIP_DPDK_BUILD),1)
 dpdkbuild: $(DPDK_DEPS)
 endif
 
-lib: $(DPDKBUILD)
+lib: $(DPDKBUILD) $(VFIOUSERBUILD)
 module: lib
 shared_lib: module
 app: $(LIB)

configure

@@ -62,6 +62,8 @@ function usage()
 	echo "                           No path required."
 	echo " virtio                    Build vhost initiator and virtio-pci bdev modules."
 	echo "                           No path required."
+	echo " vfio-user                 Build NVMf custom vfio-user target."
+	echo "                           No path required."
 	echo " pmdk                      Build persistent memory bdev."
 	echo "                           example: /usr/share/pmdk"
 	echo " reduce                    Build vbdev compression module."
@@ -334,6 +336,12 @@ for i in "$@"; do
 		--without-virtio)
 			CONFIG[VIRTIO]=n
 			;;
+		--with-vfio-user)
+			CONFIG[VFIO_USER]=y
+			;;
+		--without-vfio-user)
+			CONFIG[VFIO_USER]=n
+			;;
 		--with-pmdk)
 			CONFIG[PMDK]=y
 			CONFIG[PMDK_DIR]=""
@@ -449,6 +457,53 @@ else
 fi
 BUILD_CMD+=(-I/usr/local/include -L/usr/local/lib)
 
+function set_os_id_version() {
+	if [[ -f /etc/os-release ]]; then
+		source /etc/os-release
+	fi
+
+	OSID=$ID
+	OSVERSION=$VERSION_ID
+
+	echo "OS-ID: $OSID | OS-Version: $OSVERSION"
+}
+
+if [[ "${CONFIG[VFIO_USER]}" = "y" ]]; then
+	if [[ $arch != x86_64* ]] || [[ $sys_name == "FreeBSD" ]]; then
+		echo "Non x86_64 and Linux platform, disable CONFIG_VFIO_USER"
+		CONFIG[VFIO_USER]="n"
+		break
+	fi
+
+	set_os_id_version
+
+	# disable tests on ubuntu16 due to lack of macro definition in pci_regs.h
+	if [[ $OSID == "ubuntu" ]] && [[ $OSVERSION == "16.04" ]]; then
+		echo "ubuntu16 OS, disable CONFIG_VFIO_USER"
+		CONFIG[VFIO_USER]="n"
+		break
+	fi
+
+	if ! hash cmake; then
+		echo "cmake not installed, disable CONFIG_VFIO_USER"
+		CONFIG[VFIO_USER]="n"
+		break
+	fi
+
+	if [[ ! -d /usr/include/json-c ]] && [[ ! -d /usr/local/include/json-c ]]; then
+		echo "json-c-devel not installed, disable CONFIG_VFIO_USER"
+		CONFIG[VFIO_USER]="n"
+		break
+	fi
+
+	if [[ ! -e /usr/include/cmocka.h ]] && [[ ! -e /usr/local/include/cmocka.h ]]; then
+		echo "libcmocka-devel not installed, disable CONFIG_VFIO_USER"
+		CONFIG[VFIO_USER]="n"
+		break
+	fi
+
+fi
+
 # IDXD uses Intel specific instructions.
 if [[ "${CONFIG[IDXD]}" = "y" ]]; then
 	if [ $(uname -s) == "FreeBSD" ]; then

mk/spdk.common.mk

@@ -166,6 +166,21 @@ SYS_LIBS += -L$(ISAL_DIR)/.libs -lisal
 COMMON_CFLAGS += -I$(ISAL_DIR)/..
 endif
 
+VFIO_USER_DIR=$(SPDK_ROOT_DIR)/libvfio-user
+ifeq ($(CONFIG_VFIO_USER), y)
+ifeq ($(CONFIG_DEBUG), y)
+VFIO_USER_BUILD_TYPE=dbg
+else
+VFIO_USER_BUILD_TYPE=release
+endif
+VFIO_USER_INSTALL_DIR=$(VFIO_USER_DIR)/build
+VFIO_USER_INCLUDE_DIR=$(VFIO_USER_INSTALL_DIR)/usr/local/include
+VFIO_USER_LIBRARY_DIR=$(VFIO_USER_DIR)/build/$(VFIO_USER_BUILD_TYPE)/lib
+CFLAGS += -I$(VFIO_USER_INCLUDE_DIR)
+LDFLAGS += -L$(VFIO_USER_LIBRARY_DIR)
+SYS_LIBS += -Wl,-Bstatic -lvfio-user -Wl,-Bdynamic -ljson-c
+endif
+
 #Attach only if FreeBSD and RDMA is specified with configure
 ifeq ($(OS),FreeBSD)
 ifeq ($(CONFIG_RDMA),y)

vfiouserbuild/Makefile

@@ -0,0 +1,56 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+.PHONY: all clean install uninstall
+
+#ifneq ($(Q),)
+#REDIRECT=> /dev/null
+#endif
+
+# Force-disable scan-build
+SUB_CC = $(patsubst %ccc-analyzer,$(DEFAULT_CC),$(CC))
+
+all:
+	$(Q)$(MAKE) -C $(VFIO_USER_DIR) BUILD_TYPE=$(VFIO_USER_BUILD_TYPE) DESTDIR=$(VFIO_USER_INSTALL_DIR) install CC=$(SUB_CC) $(REDIRECT)
+
+install: all
+
+uninstall:
+	@:
+
+clean:
+	$(Q)$(MAKE) -C $(VFIO_USER_DIR) clean $(REDIRECT)
+	$(Q)rm -rf $(VFIO_USER_DIR)/build