GA2 update: 2.01

Enhancements: 1. Improve KV Performance for iops and latency 2. Improve KV FFDC and error handling 3. Improve KV startup/shutdown performance for large persistant luns 4. Improve BLOCK Performance for polling mode 5. Improve the Block and KV FVT testcase coverage 6. Enable KV and Block tools to run in the 3 modes: memory, file, hardware. 7. Add doxygen for the KV and Block example programs 8. Fix Block IO when using USER_TAG 9. Fix for Redis where KV gave incorrect return codes
rellermeyer · Feb 12, 2016 · 6e1a255 · 6e1a255
1 parent 5635175
commit 6e1a255
Show file tree

Hide file tree

Showing 103 changed files with 7,083 additions and 3,778 deletions.
diff --git a/README.md b/README.md
@@ -66,24 +66,21 @@ As a developer, to get started:
 Note: Developers have options to enable / disable specific targets (e.g. Big endian PPC64BE vs little endian PPC64EL) or P8 vs P7 tunings. See the customrc.p8be as an example. Creating a new component-specific environment variable is legal, however the env variable should be optional and provide a default that is safe for production.
 
 Current valid options:
-
-ENV Variable            | Component | Usage (BOLD = default) 
------------------------ | --------- | ------------------------
-TARGET_PLATFORM         | (all)     | PPC64BE - Big-Endian Structures
-                        |           | PPC64LE - Little-Endian Structures
-CUSTOMFLAGS             | (all)     | Custom GCC flags. Used typically to enable P8 or P7 tunings, debug, optimization, etc.
-BLOCK_FILEMODE_ENABLED  | block     | Forces Block Layer to redirect all IO to a file instead of a CAPI device. 1 = enabled, 0 = disabled
-BLOCK_KERNEL_MC_ENABLED | block     | Enables block layer to communicate with cxlflash driver built in to the Linux kernel. For more information, see https://www.kernel.org/doc/Documentation/powerpc/cxlflash.txt
-
+|ENV Variable            | Component | Usage (BOLD = default)  |
+|------------------------|-----------|-------------------------|
+|TARGET_PLATFORM         | (all)     | PPC64BE - Big-Endian Structures   |
+|                        |           | PPC64LE - Little-Endian Structures |
+|CUSTOMFLAGS             | (all)     | Custom GCC flags. Used typically to enable P8 or P7 tunings, debug, optimization, etc.  |
+|BLOCK_FILEMODE_ENABLED  | block     | Forces Block Layer to redirect all IO to a file instead of a CAPI device. 1 = enabled, 0 = disabled  |
+|BLOCK_KERNEL_MC_ENABLED | block     | Enables block layer to communicate with cxlflash driver built in to the Linux kernel. For more information, see https://www.kernel.org/doc/Documentation/powerpc/cxlflash.txt |
 
 Prebuild customrc files exist for different environments. Most users will want to use "customrc.p8elblkkermc" or "customrc.p8el:"
 
 CustomRC files:
-
-Filename                | Description
------------------------ | -------------------------------------
-customrc.p8el           | Little-endian Linux, P8 Tunings, Block FileMode enabled
-customrc.p8elblkkermc   | Little-endian Linux, P8 Tunings, Real IO to CXL Flash kernel driver
+|Filename                | Description
+|------------------------|--------------------------------------|
+|customrc.p8el           | Little-endian Linux, P8 Tunings, Block FileMode enabled |
+|customrc.p8elblkkermc   | Little-endian Linux, P8 Tunings, Real IO to CXL Flash kernel driver |
 
 
 Example on a POWER8 Little-endian system:

diff --git a/config.aix.mk b/config.aix.mk
@@ -81,7 +81,8 @@ endif
 #CUSTOMFLAGS += -D__SURELOCK_MODULE=${MODULE}
 #For AIX use the following istead.
 #Suppress infinite loop warnings on AIX (1500-010)
-CUSTOMFLAGS += -qcpluscmt -Dinline=__inline -D_AIX -D__SURELOCK_MODULE=${MODULE} -qsuppress=1500-010
+CUSTOMFLAGS += -qcpluscmt -Dinline=__inline -D_AIX \
+-D__SURELOCK_MODULE=${MODULE} -qsuppress=1500-010 -D_REENTRANT
 CUSTOMFLAGS64 += -q 64 ${CUSTOMFLAGS}
 LIBS += $(addsuffix .so, $(addprefix lib, ${MODULE}))
 LIBS64 += $(addsuffix .64so, $(addprefix lib, ${MODULE}))
@@ -159,19 +160,19 @@ BEAMFLAGS = \
 COMMONFLAGS = ${EXTRACOMMONFLAGS}
 
 ifndef NO_O3
-COMMONFLAGS += -O3
+#COMMONFLAGS += -O3
 endif
 
 #add support for the rev ID header
 GITREVISION:=$(shell git rev-list HEAD | wc -l | sed -e 's/^ *//')-$(shell git rev-parse --short HEAD)
 CUSTOMFLAGS += -DGITREVISION='"${GITREVISION}"'
 
 
-CFLAGS += ${COMMONFLAGS} -g \
+CFLAGS += ${COMMONFLAGS} \
 	 ${CUSTOMFLAGS} \
 	  ${ARCHFLAGS} \
 	${INCFLAGS}
-CFLAGS64 += ${COMMONFLAGS} -g \
+CFLAGS64 += ${COMMONFLAGS} \
 	 ${CUSTOMFLAGS64} \
 	  ${ARCHFLAGS} \
 	${INCFLAGS}

diff --git a/config.linux.mk b/config.linux.mk
@@ -135,10 +135,9 @@ endif
 GITREVISION:=$(shell git rev-list HEAD | wc -l)-$(shell git rev-parse --short HEAD)
 CUSTOMFLAGS += -DGITREVISION='"${GITREVISION}"'
 
-
 CFLAGS += ${COMMONFLAGS} \
 	 -Wall ${CUSTOMFLAGS}  ${ARCHFLAGS} \
-	-R '/opt/at7.1/lib64:$$ORIGIN/../lib:$$ORIGIN:/lib:/usr/lib:/opt/ibm/capikv/lib' \
+	-R '$$ADV_TOOLCHAIN_PATH/lib64:$$ORIGIN/../lib:$$ORIGIN:/lib:/usr/lib:/opt/ibm/capikv/lib' \
 	${INCFLAGS}
 #if ALLOW_WARNINGS is NOT defined, we assume we are compiling production code
 #as such, we adhere to strict compile flags. If this is defined then we warn
@@ -325,6 +324,17 @@ $(foreach _gtest,$(GTESTS_DIR),$(eval $(call GTESTS_template,$(_gtest))))
 CHECK_HEADER = $(shell echo \\\#include $(1) | \
                $(CC) $(CFLAGS) -E - > /dev/null 2>&1 && echo y || echo n)
 
+#Get cxlflash_ioctl.h if we can't find it on the default include path. Note that if the host system already has one in the system's include libs,
+#we won't download a new file. To download a new file, delete the existing one...
+${ROOTPATH}/src/include/scsi/cxlflash_ioctl.h:
+ifeq ($(call CHECK_HEADER,"<scsi/cxlflash_ioctl.h>"),n)
+	@echo "WARNING: Downloading new cxlflash_ioctl.h from Jenkins"
+	mkdir -p ${ROOTPATH}/src/include/scsi
+	wget -P ${ROOTPATH}/src/include/scsi -q http://hydepark.aus.stglabs.ibm.com:8081/job/ga2-kernel/lastSuccessfulBuild/artifact/linux/include/uapi/scsi/cxlflash_ioctl.h
+endif
+
+
+
 $(GTESTS_DIR):
 	$(CXX) $(CFLAGS) $(LDFLAGS) $($(@)_GTESTS_OFILES) $(GTEST_DEPS) $(LINKLIBS) ${LIBPATHS} -o $@
 #-------------------------------------------------------------------------------
@@ -354,7 +364,8 @@ ${LIBRARIES}: ${OBJECTS}
 ${EXTRA_PARTS} ${PROGRAMS}: ${LIBRARIES}
 $(GTESTS_DIR) $(GTESTS_NM_DIR) $(BIN_TESTS): $(GTEST_TARGETS)
 
-dep:       ${SUBDIRS:.d=.dep} ${DEPS}
+#cxlflash_ioctl.h dep is necessary to cause the "wget" wagon to go download a header from the CI server... look below for the appropriate rule / recipe
+dep:       ${SUBDIRS:.d=.dep} ${DEPS} ${ROOTPATH}/src/include/scsi/cxlflash_ioctl.h
 code_pass: ${SUBDIRS} ${LIBRARIES} ${EXTRA_PARTS} ${PROGRAMS}
 test:      ${SUBDIRS:.d=.test} ${BIN_TESTS} ${GTESTS_DIR} ${GTESTS_NM_DIR}
 fvt:       ${SUBDIRS:.d=.fvt}

diff --git a/env.bash b/env.bash
@@ -47,9 +47,8 @@ fi
 export MCP_PATH=/opt/mcp/toolchains/fr_SL1_2014-05-12-194021
 
 #configure advanced toolchain for linux
-AT70PATH=/opt/at7.0
 AT71PATH=/opt/at7.1
-AT80PATH=/opt/at8.0
+AT90PATH=/opt/at9.0-2-rc2
 
 if [ -d $MCP_PATH ]; then
     echo "INFO: Found MCP: $MCP_PATH ."
@@ -61,12 +60,10 @@ fi
 
 
 
-if [ -d $AT70PATH ]; then
-    export ADV_TOOLCHAIN_PATH=$AT70PATH
-elif [ -d $AT71PATH ]; then
+if [ -d $AT71PATH ]; then
     export ADV_TOOLCHAIN_PATH=$AT71PATH
-elif [ -d $AT80PATH ]; then
-    export ADV_TOOLCHAIN_PATH=$AT80PATH
+elif [ -d $AT90PATH ]; then
+    export ADV_TOOLCHAIN_PATH=$AT90PATH
 else
     echo "WARNING: no toolchain was found. Will fall back to system defaults. YMMV."
 fi

diff --git a/src/block/cflash_block.c b/src/block/cflash_block.c
@@ -689,7 +689,7 @@ inline int CBLK_IN_CACHE(cflsh_chunk_t *chunk,void *buf, cflash_offset_t lba, si
 
 
 /*
- * NAME:        CBLK_BUILD_ISSUE_RW_CMD
+ * NAME:        cblk_build_issue_rw_cmd
  *
  * FUNCTION:    Builds and issues a READ16/WRITE16 command
  *
@@ -705,7 +705,7 @@ inline int CBLK_IN_CACHE(cflsh_chunk_t *chunk,void *buf, cflash_offset_t lba, si
  *              
  */
 
-inline int CBLK_BUILD_ISSUE_RW_CMD(cflsh_chunk_t *chunk, int *cmd_index, void *buf,cflash_offset_t lba, 
+int cblk_build_issue_rw_cmd(cflsh_chunk_t *chunk, int *cmd_index, void *buf,cflash_offset_t lba, 
 				   size_t nblocks, int flags, int lib_flags,uint8_t op_code,
 				   cblk_arw_status_t *status)
 {
@@ -910,9 +910,12 @@ inline int CBLK_BUILD_ISSUE_RW_CMD(cflsh_chunk_t *chunk, int *cmd_index, void *b
     if (op_code == SCSI_READ_16) {
 
 	local_flags = CFLASH_READ_DIR_OP;
+	chunk->cmd_info[cmd->index].flags |= CFLSH_MODE_READ;
+
     } else if (op_code == SCSI_WRITE_16) {
 
 	local_flags = CFLASH_WRITE_DIR_OP;
+	chunk->cmd_info[cmd->index].flags |= CFLSH_MODE_WRITE;
 
     }
 
@@ -1177,18 +1180,6 @@ inline int CBLK_BUILD_ISSUE_RW_CMD(cflsh_chunk_t *chunk, int *cmd_index, void *b
     }
 
 
-
-    if (op_code == SCSI_READ_16) {
-
-        cmd->cmdi->flags |= CFLSH_MODE_READ;
-
-    } else if (op_code == SCSI_WRITE_16) {
-
-        cmd->cmdi->flags |= CFLSH_MODE_WRITE;
-
-    }
-
-
     /*
      * Unlock since we are waiting for 
      * a completions. Other threads may
@@ -2096,7 +2087,7 @@ int cblk_read(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblocks,
     }
 
 
-    rc = CBLK_BUILD_ISSUE_RW_CMD(chunk,&cmd_index,buf,lba,nblocks,flags,0,SCSI_READ_16,NULL);
+    rc = cblk_build_issue_rw_cmd(chunk,&cmd_index,buf,lba,nblocks,flags,0,SCSI_READ_16,NULL);
 
 
     if (rc) {
@@ -2290,7 +2281,7 @@ int cblk_write(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblocks
 
     }
 
-    rc = CBLK_BUILD_ISSUE_RW_CMD(chunk,&cmd_index,buf,lba,nblocks,flags,0,SCSI_WRITE_16,NULL);
+    rc = cblk_build_issue_rw_cmd(chunk,&cmd_index,buf,lba,nblocks,flags,0,SCSI_WRITE_16,NULL);
 
 
     if (rc) {
@@ -2475,11 +2466,11 @@ static inline int _cblk_aread(cflsh_chunk_t *chunk,void *buf,cflash_offset_t lba
     }
     /* 
      * NOTE: If data was read from the cache, then the rc 
-     *       from CBLK_BUILD_ISSUE_RW_CMD will be greater
-     *       than 0. If CBLK_BUILD_ISSUE_RW_CMD fails, then
+     *       from cblk_build_issue_rw_cmd will be greater
+     *       than 0. If cblk_build_issue_rw_cmd fails, then
      *       rc = -1.
      */
-    rc = CBLK_BUILD_ISSUE_RW_CMD(chunk,&cmd_index,buf,lba,nblocks,flags,CFLASH_ASYNC_OP,SCSI_READ_16,status);
+    rc = cblk_build_issue_rw_cmd(chunk,&cmd_index,buf,lba,nblocks,flags,CFLASH_ASYNC_OP,SCSI_READ_16,status);
 
     if (rc) {
 
@@ -2648,7 +2639,7 @@ static inline int _cblk_awrite(cflsh_chunk_t *chunk,void *buf,cflash_offset_t lb
 	return -1;
     }
 
-    rc = CBLK_BUILD_ISSUE_RW_CMD(chunk,&cmd_index,buf,lba,nblocks,flags,CFLASH_ASYNC_OP,SCSI_WRITE_16,status);
+    rc = cblk_build_issue_rw_cmd(chunk,&cmd_index,buf,lba,nblocks,flags,CFLASH_ASYNC_OP,SCSI_WRITE_16,status);
 
     if (rc) {
 
@@ -2798,15 +2789,37 @@ static inline int _cblk_aresult(cflsh_chunk_t *chunk,int *tag, uint64_t *status,
 
     *status = 0;
 
-    if (chunk->flags & CFLSH_CHNK_NO_BG_TD && harvest) {
+    if (chunk->flags & CFLSH_CHNK_NO_BG_TD && harvest)
+    {
+        int do_poll=1, lib_flags = CFLASH_ASYNC_OP;
 
-	/*
-	 * Check if any commands completed since the last time we
-	 * checked, but do not wait for any to complete.
-	 */
+        if (!(flags & CBLK_ARESULT_BLOCKING)) 
+	{
 
-	rc = CBLK_WAIT_FOR_IO_COMPLETE(chunk,tag,&transfer_size,FALSE,CFLASH_ASYNC_OP);
+	    lib_flags |= CFLASH_SHORT_POLL;
+	}
 
+        /* peek at the cmd if a tag was provided
+         * -if the cmd is complete, skip the harvest
+         */
+        if (!(flags & CBLK_ARESULT_USER_TAG)  &&
+            *tag >= 0                         &&
+            *tag < chunk->num_cmds)
+        {
+            CFLASH_BLOCK_LOCK(chunk->lock);
+            if ((cmdi = &(chunk->cmd_info[*tag])) &&
+                cmdi->in_use                 &&
+                cmdi->state == CFLSH_MGM_CMP)
+	    {
+                do_poll=0;
+            }
+            CFLASH_BLOCK_UNLOCK(chunk->lock);
+        }
+        if (do_poll) 
+	{
+
+            rc = CBLK_WAIT_FOR_IO_COMPLETE(chunk,tag,&transfer_size,FALSE,lib_flags);
+	}
     }
 
     CFLASH_BLOCK_LOCK(chunk->lock);
@@ -3021,15 +3034,20 @@ static inline int _cblk_aresult(cflsh_chunk_t *chunk,int *tag, uint64_t *status,
     if (flags & CBLK_ARESULT_USER_TAG) {
 
 	cmd = NULL;
+	CBLK_TRACE_LOG_FILE(9,"search for user_tag:%d from 0 to %d",
+	        *tag, chunk->num_cmds);
 
 	for (i = 0; i < chunk->num_cmds;i++) {
 
-	    if (chunk->cmd_info[i].user_tag == *tag) {
+	    if (chunk->cmd_info[i].in_use                      &&
+	        chunk->cmd_info[i].flags & CFLSH_CMD_INFO_UTAG &&
+	        chunk->cmd_info[i].user_tag == *tag) {
 
 		/*
 		 * We found the user tag provided
 		 */
-
+	        CBLK_TRACE_LOG_FILE(9,"found user_tag:%d at %d",
+	                chunk->cmd_info[i].user_tag, i);
 		cmd = &(chunk->cmd_start[i]);
 
 		break;
@@ -3078,6 +3096,7 @@ static inline int _cblk_aresult(cflsh_chunk_t *chunk,int *tag, uint64_t *status,
 
 	rc = -1;
 	CBLK_TRACE_LOG_FILE(1,"null cmdi for cmd_index = 0x%x",cmd->index);
+	CFLASH_BLOCK_UNLOCK(chunk->lock);
 
 	return rc;
 
@@ -3130,7 +3149,10 @@ static inline int _cblk_aresult(cflsh_chunk_t *chunk,int *tag, uint64_t *status,
 	if (chunk->flags & CFLSH_CHNK_NO_BG_TD) {
 
 
-	    rc = CBLK_WAIT_FOR_IO_COMPLETE(chunk,tag,&transfer_size,TRUE,CFLASH_ASYNC_OP); 
+	    CFLASH_BLOCK_UNLOCK(chunk->lock);
+	    rc = CBLK_WAIT_FOR_IO_COMPLETE(chunk, &cmd->index, &transfer_size,
+	                                  TRUE, CFLASH_ASYNC_OP);
+	    CFLASH_BLOCK_LOCK(chunk->lock); 
 
 	} else {
 	    /*