diff --git a/README.md b/README.md index 129d4f57..2d67119b 100644 --- a/README.md +++ b/README.md @@ -66,21 +66,24 @@ As a developer, to get started: Note: Developers have options to enable / disable specific targets (e.g. Big endian PPC64BE vs little endian PPC64EL) or P8 vs P7 tunings. See the customrc.p8be as an example. Creating a new component-specific environment variable is legal, however the env variable should be optional and provide a default that is safe for production. Current valid options: -|ENV Variable | Component | Usage (BOLD = default) | -|------------------------|-----------|-------------------------| -|TARGET_PLATFORM | (all) | PPC64BE - Big-Endian Structures | -| | | PPC64LE - Little-Endian Structures | -|CUSTOMFLAGS | (all) | Custom GCC flags. Used typically to enable P8 or P7 tunings, debug, optimization, etc. | -|BLOCK_FILEMODE_ENABLED | block | Forces Block Layer to redirect all IO to a file instead of a CAPI device. 1 = enabled, 0 = disabled | -|BLOCK_KERNEL_MC_ENABLED | block | Enables block layer to communicate with cxlflash driver built in to the Linux kernel. For more information, see https://www.kernel.org/doc/Documentation/powerpc/cxlflash.txt | + +ENV Variable | Component | Usage (BOLD = default) +----------------------- | --------- | ------------------------ +TARGET_PLATFORM | (all) | PPC64BE - Big-Endian Structures + | | PPC64LE - Little-Endian Structures +CUSTOMFLAGS | (all) | Custom GCC flags. Used typically to enable P8 or P7 tunings, debug, optimization, etc. +BLOCK_FILEMODE_ENABLED | block | Forces Block Layer to redirect all IO to a file instead of a CAPI device. 1 = enabled, 0 = disabled +BLOCK_KERNEL_MC_ENABLED | block | Enables block layer to communicate with cxlflash driver built in to the Linux kernel. For more information, see https://www.kernel.org/doc/Documentation/powerpc/cxlflash.txt + Prebuild customrc files exist for different environments. Most users will want to use "customrc.p8elblkkermc" or "customrc.p8el:" CustomRC files: -|Filename | Description -|------------------------|--------------------------------------| -|customrc.p8el | Little-endian Linux, P8 Tunings, Block FileMode enabled | -|customrc.p8elblkkermc | Little-endian Linux, P8 Tunings, Real IO to CXL Flash kernel driver | + +Filename | Description +----------------------- | ------------------------------------- +customrc.p8el | Little-endian Linux, P8 Tunings, Block FileMode enabled +customrc.p8elblkkermc | Little-endian Linux, P8 Tunings, Real IO to CXL Flash kernel driver Example on a POWER8 Little-endian system: diff --git a/config.aix.mk b/config.aix.mk index a8fea3f3..cf1bf715 100644 --- a/config.aix.mk +++ b/config.aix.mk @@ -67,6 +67,9 @@ GENDIR = ${ROOTPATH}/obj/genfiles IMGDIR = ${ROOTPATH}/img PKGDIR = ${ROOTPATH}/pkg +GTESTDIR = src/test/framework/googletest/googletest +GTESTINC = ${GTESTDIR}/include + ifdef MODULE OBJDIR = ${ROOTPATH}/obj/modules/${MODULE} BEAMDIR = ${ROOTPATH}/obj/beam/${MODULE} @@ -89,7 +92,8 @@ LIBS64 += $(addsuffix .64so, $(addprefix lib, ${MODULE})) AR_LIBS += $(addsuffix .a, $(addprefix lib, ${MODULE})) EXTRAINCDIR += ${GENDIR} ${CURDIR} #EXPFLAGS = -bE:exportfile -LDFLAGS = -bnoentry -bM:SRE $(EXPFLAGS) ${LIBPATHS} -lc +LDFLAGS = -bnoentry -bM:SRE $(EXPFLAGS) ${LIBPATHS} +POSTLDFLAGS = -lc #LDFLAGS = -bnoentry -bM:SRE -bexpall ${LIBPATHS} -lc LDFLAGS64 = -b64 ${LDFLAGS} LIBRARIES = $(addprefix ${OBJDIR}/, ${MEMBER}) @@ -190,7 +194,7 @@ else TESTGEN = ${ROOTPATH}/src/usr/cxxtest/cxxtestgen.pl endif -INCDIR = /usr/include /usr/include/sys ${ROOTPATH}/src/include/ +INCDIR = ${ROOTPATH}/src/include /usr/include /usr/include/sys _INCDIRS = ${INCDIR} ${EXTRAINCDIR} INCFLAGS = $(addprefix -I, ${_INCDIRS} ) ASMINCFLAGS = $(addprefix $(lastword -Wa,-I), ${_INCDIRS}) @@ -231,12 +235,12 @@ ${OBJDIR}/%.o : %.S ifdef MODULE ${OBJDIR}/${MEMBER} : ${OBJECTS} @mkdir -p ${IMGDIR} - ${LD} ${LDFLAGS} ${MODLIBS} -o $@ $(OBJECTS) + ${LD} ${LDFLAGS} -o $@ $(OBJECTS) ${POSTLDFLAGS} ${MODLIBS} ${OBJDIR}/${MEMBER64} : ${OBJECTS64} @mkdir -p ${IMGDIR} - ${LD} ${LDFLAGS64} ${MODLIBS} -o $@ $(OBJECTS64) + ${LD} ${LDFLAGS64} -o $@ $(OBJECTS64) ${POSTLDFLAGS} ${MODLIBS} endif ${IMGDIR}/%.a : ${LIBRARIES} ${LIBRARIES64} diff --git a/config.linux.mk b/config.linux.mk index c532ecb3..26ba10a4 100644 --- a/config.linux.mk +++ b/config.linux.mk @@ -34,7 +34,6 @@ docs: src/build/doxygen/doxygen.conf rm -rf obj/doxygen/* doxygen src/build/doxygen/doxygen.conf - tests: ${MAKE} ship ${MAKE} test @@ -66,6 +65,9 @@ GENDIR = ${ROOTPATH}/obj/genfiles IMGDIR = ${ROOTPATH}/img PKGDIR = ${ROOTPATH}/pkg +GTESTDIR = src/test/framework/googletest/googletest +GTESTINC = ${GTESTDIR}/include + ifdef MODULE OBJDIR = ${ROOTPATH}/obj/modules/${MODULE} BEAMDIR = ${ROOTPATH}/obj/beam/${MODULE} @@ -110,17 +112,25 @@ ifeq ($(USE_ADVANCED_TOOLCHAIN),yes) VPATH_DIRS:= ${ADV_TOOLCHAIN_PATH}/lib64 ${VPATH_DIRS} #see the ld flags below (search for rpath). This puts the atx.x stuff on the front #which is REQUIRED by the toolchain. - RPATH_PREPEND = -rpath,${ADV_TOOLCHAIN_PATH}/lib64 - + RPATH_PREPEND = -rpath,${ADV_TOOLCHAIN_PATH}/lib64 + CFLAGS += ${COMMONFLAGS} \ + -Wall ${CUSTOMFLAGS} ${ARCHFLAGS} \ + -R '$$ADV_TOOLCHAIN_PATH/lib64:$$ORIGIN/../lib:$$ORIGIN:/lib:/usr/lib:/opt/ibm/capikv/lib' \ + ${INCFLAGS} + LDFLAGS = ${COMMONFLAGS} -Wl,${RPATH_PREPEND},-rpath,$$ORIGIN/../lib,-rpath,$$ORIGIN,-rpath,$(DEFAULT_LIB_INSTALL_PATH) else CC_RAW = gcc CXX_RAW = g++ CC = ${CC_RAW} CXX = ${CXX_RAW} - LD = ld + LD = gcc OBJDUMP = objdump #if we are NOT using the atx.x stuff, prepend nothing. - RPATH_PREPEND = + RPATH_PREPEND = + CFLAGS += ${COMMONFLAGS} \ + -Wall ${CUSTOMFLAGS} ${ARCHFLAGS} \ + ${INCFLAGS} + LDFLAGS = ${COMMONFLAGS} endif #TODO: Find correct flags for surelock @@ -135,10 +145,6 @@ endif GITREVISION:=$(shell git rev-list HEAD | wc -l)-$(shell git rev-parse --short HEAD) CUSTOMFLAGS += -DGITREVISION='"${GITREVISION}"' -CFLAGS += ${COMMONFLAGS} \ - -Wall ${CUSTOMFLAGS} ${ARCHFLAGS} \ - -R '$$ADV_TOOLCHAIN_PATH/lib64:$$ORIGIN/../lib:$$ORIGIN:/lib:/usr/lib:/opt/ibm/capikv/lib' \ - ${INCFLAGS} #if ALLOW_WARNINGS is NOT defined, we assume we are compiling production code #as such, we adhere to strict compile flags. If this is defined then we warn #but allow the compile to continue. @@ -156,7 +162,6 @@ endif ASMFLAGS = ${COMMONFLAGS} CXXFLAGS += ${CFLAGS} -fno-rtti -fno-exceptions -Wall #RPATH order is important here. the prepend var lets us throw the ATxx stuff in first if needed. -LDFLAGS = ${COMMONFLAGS} -Wl,${RPATH_PREPEND},-rpath,$$ORIGIN/../lib,-rpath,$$ORIGIN,-rpath,$(DEFAULT_LIB_INSTALL_PATH) INCDIR = ${ROOTPATH}/src/include/ diff --git a/config.mk b/config.mk index 9a805827..f32bb41a 100644 --- a/config.mk +++ b/config.mk @@ -26,21 +26,30 @@ #select a default target architecture #if no target arch is set, select PPC64BE for convenience. ifndef TARGET_PLATFORM - TARGET_PLATFORM=PPC64BE + UNAME=$(shell uname -m) + ifeq ($(UNAME),x86_64) + TARGET_PLATFORM=x86_64 + else + ifeq ($(UNAME),ppc64le) + TARGET_PLATFORM=PPC64EL + else + TARGET_PLATFORM=PPC64BE + endif + endif endif + #If the target architecture is set, pass an architecture #down as a #define to the underlying code -ARCHFLAGS += -DTARGET_ARCH_${TARGET_PLATFORM} +export ARCHFLAGS += -DTARGET_ARCH_${TARGET_PLATFORM} #Determine if this a linux or AIX system - UNAME=$(shell uname) ifeq ($(UNAME),AIX) -include ${ROOTPATH}/config.aix.mk + include ${ROOTPATH}/config.aix.mk else -ifeq ($(UNAME),Darwin) -include ${ROOTPATH}/config.mac.mk -else -include ${ROOTPATH}/config.linux.mk -endif + ifeq ($(UNAME),Darwin) + include ${ROOTPATH}/config.mac.mk + else + include ${ROOTPATH}/config.linux.mk + endif endif diff --git a/customrc.x86_64 b/customrc.x86_64 new file mode 100644 index 00000000..444680f1 --- /dev/null +++ b/customrc.x86_64 @@ -0,0 +1,29 @@ +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: customrc.x86_64 $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +export MAKECMD=gmake +export CUSTOMFLAGS="" +export BLOCK_FILEMODE_ENABLED=1 +export TARGET_PLATFORM=x86_64 diff --git a/env.bash b/env.bash index 16855286..1bbb62eb 100644 --- a/env.bash +++ b/env.bash @@ -23,7 +23,11 @@ # # IBM_PROLOG_END_TAG - +export CUSTOMFLAGS= +export BLOCK_FILEMODE_ENABLED= +export BLOCK_MC_ENABLED= +export TARGET_PLATFORM= +export BLOCK_KERNEL_MC_ENABLED= #allow a user to specify a custom RC file if needed #e.g. disable the advanced toolchain with "export USE_ADVANCED_TOOLCHAIN=no" @@ -49,6 +53,7 @@ export MCP_PATH=/opt/mcp/toolchains/fr_SL1_2014-05-12-194021 #configure advanced toolchain for linux AT71PATH=/opt/at7.1 AT90PATH=/opt/at9.0-2-rc2 +ATPATH=/opt/at10.0 if [ -d $MCP_PATH ]; then echo "INFO: Found MCP: $MCP_PATH ." @@ -60,10 +65,8 @@ fi -if [ -d $AT71PATH ]; then - export ADV_TOOLCHAIN_PATH=$AT71PATH -elif [ -d $AT90PATH ]; then - export ADV_TOOLCHAIN_PATH=$AT90PATH +if [ -d $ATPATH ]; then + export ADV_TOOLCHAIN_PATH=$ATPATH else echo "WARNING: no toolchain was found. Will fall back to system defaults. YMMV." fi diff --git a/src/block/cflash_block.c b/src/block/cflash_block.c index a144416a..d5d3fbbb 100755 --- a/src/block/cflash_block.c +++ b/src/block/cflash_block.c @@ -30,6 +30,7 @@ #define CFLSH_BLK_FILENUM 0x0100 #include "cflash_block_internal.h" #include "cflash_block_inline.h" +#include #ifndef _AIX #include @@ -54,10 +55,7 @@ REVISION_TAGS(block); #else #include #endif - -#define CFLSH_BLK_FILENUM 0x0100 -#include "cflash_block_internal.h" -#include "cflash_block_inline.h" + cflsh_block_t cflsh_blk; @@ -118,71 +116,60 @@ static int cflsh_blk_term = 0; * pointer = chunk found. * */ - inline cflsh_chunk_t *CBLK_GET_CHUNK_HASH(chunk_id_t chunk_id, int check_rdy) - { - cflsh_chunk_t *chunk = NULL; + cflsh_chunk_hash_t *p_hash = NULL; + cflsh_chunk_t *chunk = NULL; + p_hash = &cflsh_blk.hash[chunk_id & CHUNK_HASH_MASK]; - chunk = cflsh_blk.hash[chunk_id & CHUNK_HASH_MASK]; + CFLASH_BLOCK_WR_RWLOCK(p_hash->lock); + chunk = p_hash->head; while (chunk) { - if ( (ulong)chunk & CHUNK_BAD_ADDR_MASK ) { - - CBLK_TRACE_LOG_FILE(1,"Corrupted chunk address = 0x%llx, index = 0x%x", - (uint64_t)chunk, (chunk_id & CHUNK_HASH_MASK)); - - cflsh_blk.num_bad_chunk_ids++; - chunk = NULL; - - break; - - } - - - if (chunk->index == chunk_id) { - + if ( (ulong)chunk & CHUNK_BAD_ADDR_MASK ) { - /* - * Found the specified chunk. Let's - * validate it. - */ - - if (CFLSH_EYECATCH_CHUNK(chunk)) { - /* - * Invalid chunk - */ - - CBLK_TRACE_LOG_FILE(1,"Invalid chunk, chunk_id = %d", - chunk_id); - chunk = NULL; - } else if ((!(chunk->flags & CFLSH_CHNK_RDY)) && - (check_rdy)) { + CBLK_TRACE_LOG_FILE(1,"Corrupted chunk address:0x%llx, index:0x%x", + (uint64_t)chunk, (chunk_id & CHUNK_HASH_MASK)); + cflsh_blk.num_bad_chunk_ids++; + chunk = NULL; + break; + } - /* - * This chunk is not ready - */ - - CBLK_TRACE_LOG_FILE(1,"chunk not ready, chunk_id = %d", - chunk_id); - chunk = NULL; - } - break; - } + if (chunk->index == chunk_id) { + + /* + * Found the specified chunk. Let's + * validate it. + */ + + if (CFLSH_EYECATCH_CHUNK(chunk)) { + /* + * Invalid chunk + */ + CBLK_TRACE_LOG_FILE(1,"Invalid chunk, chunk_id = %d", + chunk_id); + chunk = NULL; + } else if ((!(chunk->flags & CFLSH_CHNK_RDY)) && (check_rdy)) { + + /* + * This chunk is not ready + */ + CBLK_TRACE_LOG_FILE(1,"chunk not ready, chunk_id = %d", + chunk_id); + chunk = NULL; + } + break; + } - chunk = chunk->next; + chunk = chunk->next; } /* while */ - - - - - + CFLASH_BLOCK_RWUNLOCK(p_hash->lock); return (chunk); } @@ -269,6 +256,14 @@ void _cflsh_blk_init(void) char *env_adap_poll_delay = getenv("CFLSH_BLK_ADAP_POLL_DLY"); #endif /* _SKIP_READ_CALL */ int rc; + int i=0; + + memset(cflsh_blk.hash,0,sizeof(cflsh_blk.hash)); + + for (i=0; iindex; + cmd->stime = getticks(); if (flags & CBLK_ARW_USER_STATUS_FLAG) { @@ -1262,10 +1267,6 @@ int cblk_term(void *arg,uint64_t flags) } - - - - /* * NAME: cblk_open * @@ -1290,22 +1291,27 @@ chunk_id_t cblk_open(const char *path, int max_num_requests, int mode, chunk_ext int open_flags; int cleanup_depth; cflsh_chunk_t *chunk = NULL; + int rc = 0; #ifdef _AIX int ext_flags = 0; #endif /* _AIX */ - errno = 0; - #ifdef _AIX cflsh_blk_init(); #endif /* AIX */ + if (flags & CBLK_OPN_GROUP) + { + flags &= ~CBLK_OPN_GROUP; + rc = cblk_cg_open(path,max_num_requests,mode,(int)ext,ext,flags); + return (rc); + } CFLASH_BLOCK_WR_RWLOCK(cflsh_blk.global_lock); - CBLK_TRACE_LOG_FILE(5,"opening %s with max_num_requests = %d, mode = 0x%x, flags = 0x%x for pid = 0x%llx", + CBLK_TRACE_LOG_FILE(4,"opening %s with max_num_requests = %d, mode = 0x%x, flags = 0x%x for pid = 0x%llx", path,max_num_requests,mode,flags,(uint64_t)cflsh_blk.caller_pid); @@ -1320,11 +1326,10 @@ chunk_id_t cblk_open(const char *path, int max_num_requests, int mode, chunk_ext ret_chunk_id = cblk_get_chunk(CFLSH_BLK_CHUNK_SET_UP, max_num_requests); + CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); chunk = CBLK_GET_CHUNK_HASH(ret_chunk_id,FALSE); - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); - if (chunk) { @@ -1350,7 +1355,7 @@ chunk_id_t cblk_open(const char *path, int max_num_requests, int mode, chunk_ext cblk_chunk_open_cleanup(chunk,20); CFLASH_BLOCK_UNLOCK(chunk->lock); - + free(chunk); @@ -1504,6 +1509,13 @@ chunk_id_t cblk_open(const char *path, int max_num_requests, int mode, chunk_ext chunk->fd = openx(chunk->dev_name,open_flags,0,ext_flags); #else + + if (flags & CBLK_OPN_MPIO_FO) { + + + chunk->flags |= CFLSH_CHNK_MPIO_FO; + } + open_flags = O_RDWR | O_NONBLOCK; @@ -1629,7 +1641,11 @@ int cblk_close(chunk_id_t chunk_id, int flags) cflsh_chunk_t *chunk; int loop_cnt = 0; - errno = 0; + if (flags & CBLK_GROUP_MASK) + { + flags &= ~CBLK_GROUP_ID; + return (cblk_cg_close(chunk_id,flags)); + } if ((chunk_id <= NULL_CHUNK_ID) || (chunk_id >= cflsh_blk.next_chunk_id)) { @@ -1649,8 +1665,8 @@ int cblk_close(chunk_id_t chunk_id, int flags) if (chunk == NULL) { - CBLK_TRACE_LOG_FILE(1,"closing failed because chunk not found, chunk_id = %d", - chunk_id); + CBLK_TRACE_LOG_FILE(1,"closing failed because chunk not found, chunk_id = %d, num_active_chunks = 0x%x", + chunk_id,cflsh_blk.num_active_chunks); CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); errno = EINVAL; return -1; @@ -1658,8 +1674,8 @@ int cblk_close(chunk_id_t chunk_id, int flags) if (chunk->in_use == FALSE) { - CBLK_TRACE_LOG_FILE(1,"closing failed because chunk not in use, rchunk_id = %d, path = %s", - chunk_id,chunk->dev_name); + CBLK_TRACE_LOG_FILE(1,"closing failed because chunk not in use, rchunk_id = %d, path = %s, num_active_chunks = 0x%x", + chunk_id,chunk->dev_name,cflsh_blk.num_active_chunks); CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); errno = EINVAL; return -1; @@ -1693,7 +1709,8 @@ int cblk_close(chunk_id_t chunk_id, int flags) return -1; } - CBLK_TRACE_LOG_FILE(9,"closing chunk->dev_name = %s",chunk->dev_name); + CBLK_TRACE_LOG_FILE(9,"closing chunk->dev_name = %s, num_paths = %d", + chunk->dev_name,chunk->num_paths); if (chunk->flags & CFLSH_CHNK_CLOSE) { @@ -1807,8 +1824,13 @@ int cblk_get_lun_size(chunk_id_t chunk_id, size_t *nblocks, int flags) int rc = 0; cflsh_chunk_t *chunk; + if (flags & CBLK_GROUP_MASK) + { + flags &= ~CBLK_GROUP_ID; + return cblk_cg_get_lun_size(chunk_id, nblocks, flags); + } + - errno = 0; if ((chunk_id <= NULL_CHUNK_ID) || (chunk_id >= cflsh_blk.next_chunk_id)) { @@ -1822,11 +1844,9 @@ int cblk_get_lun_size(chunk_id_t chunk_id, size_t *nblocks, int flags) return -1; } - CFLASH_BLOCK_RD_RWLOCK(cflsh_blk.global_lock); - chunk = CBLK_GET_CHUNK_HASH(chunk_id,TRUE); - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); + if (chunk == NULL) {errno=EINVAL; return -1;} *nblocks = chunk->num_blocks_lun; @@ -1859,8 +1879,12 @@ int cblk_get_size(chunk_id_t chunk_id, size_t *nblocks, int flags) int rc = 0; cflsh_chunk_t *chunk; + if (flags & CBLK_GROUP_MASK) + { + flags &= ~CBLK_GROUP_ID; + return cblk_cg_get_size(chunk_id, nblocks, flags); + } - errno = 0; if ((chunk_id <= NULL_CHUNK_ID) || (chunk_id >= cflsh_blk.next_chunk_id)) { @@ -1874,16 +1898,11 @@ int cblk_get_size(chunk_id_t chunk_id, size_t *nblocks, int flags) return -1; } - CFLASH_BLOCK_RD_RWLOCK(cflsh_blk.global_lock); - chunk = CBLK_GET_CHUNK_HASH(chunk_id,TRUE); - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); - *nblocks = chunk->num_blocks; - CBLK_TRACE_LOG_FILE(5,"get_size returned = 0x%llx, rc = %d",(uint64_t)(*nblocks), rc); - + CBLK_TRACE_LOG_FILE(5,"id:%d *nblocks:%ld rc:%d", chunk_id, *nblocks, rc); return rc; } @@ -1914,9 +1933,12 @@ int cblk_set_size(chunk_id_t chunk_id, size_t nblocks, int flags) int rc = 0; cflsh_chunk_t *chunk; + if (flags & CBLK_GROUP_MASK) + { + flags &= ~CBLK_GROUP_ID; + return cblk_cg_set_size(chunk_id, nblocks, flags); + } - - errno = 0; if ((chunk_id <= NULL_CHUNK_ID) || (chunk_id >= cflsh_blk.next_chunk_id)) { @@ -1927,8 +1949,6 @@ int cblk_set_size(chunk_id_t chunk_id, size_t nblocks, int flags) CBLK_TRACE_LOG_FILE(5,"set_size size = 0x%llx",(uint64_t)nblocks); - CFLASH_BLOCK_RD_RWLOCK(cflsh_blk.global_lock); - chunk = CBLK_GET_CHUNK_HASH(chunk_id,TRUE); if (chunk == NULL) { @@ -1936,7 +1956,6 @@ int cblk_set_size(chunk_id_t chunk_id, size_t nblocks, int flags) CBLK_TRACE_LOG_FILE(1,"chunk not found, chunk_id = %d", chunk_id); - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); errno = EINVAL; return -1; } @@ -1944,8 +1963,6 @@ int cblk_set_size(chunk_id_t chunk_id, size_t nblocks, int flags) CFLASH_BLOCK_LOCK(chunk->lock); - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); - if (!(chunk->flags & CFLSH_CHNK_VLUN)) { @@ -2039,9 +2056,12 @@ int cblk_read(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblocks, int pthread_rc; #endif + if (flags & CBLK_GROUP_MASK) + { + flags &= ~CBLK_GROUP_ID; + return cblk_cg_read(chunk_id, buf, lba, nblocks, flags); + } - - errno = 0; CBLK_TRACE_LOG_FILE(5,"chunk_id = %d, lba = 0x%llx, nblocks = 0x%llx, buf = %p",chunk_id,lba,(uint64_t)nblocks,buf); @@ -2051,7 +2071,6 @@ int cblk_read(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblocks, } - CFLASH_BLOCK_RD_RWLOCK(cflsh_blk.global_lock); chunk = CBLK_GET_CHUNK_HASH(chunk_id,TRUE); if (chunk == NULL) { @@ -2059,18 +2078,17 @@ int cblk_read(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblocks, CBLK_TRACE_LOG_FILE(1,"chunk not found, chunk_id = %d", chunk_id); - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); errno = EINVAL; return -1; } - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); if (nblocks > chunk->stats.max_transfer_size) { - CBLK_TRACE_LOG_FILE(1,"nblocks too large = 0x%llx",nblocks); + CBLK_TRACE_LOG_FILE(1,"nblocks too large = 0x%llx, max_transfer_size = 0x%llx",nblocks, + chunk->stats.max_transfer_size); errno = EINVAL; return -1; @@ -2232,8 +2250,13 @@ int cblk_write(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblocks int pthread_rc; #endif + if (flags & CBLK_GROUP_MASK) + { + flags &= ~CBLK_GROUP_ID; + return cblk_cg_write(chunk_id, buf, lba, nblocks, flags); + } + - errno = 0; CBLK_TRACE_LOG_FILE(5,"chunk_id = %d, lba = 0x%llx, nblocks = 0x%llx, buf = %p",chunk_id,lba,(uint64_t)nblocks,buf); if (CBLK_VALIDATE_RW(chunk_id,buf,lba,nblocks)) { @@ -2242,7 +2265,6 @@ int cblk_write(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblocks } - CFLASH_BLOCK_RD_RWLOCK(cflsh_blk.global_lock); chunk = CBLK_GET_CHUNK_HASH(chunk_id,TRUE); if (chunk == NULL) { @@ -2250,17 +2272,15 @@ int cblk_write(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblocks CBLK_TRACE_LOG_FILE(1,"chunk not found, chunk_id = %d", chunk_id); - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); errno = EINVAL; return -1; } - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); - if (nblocks > chunk->stats.max_transfer_size) { - CBLK_TRACE_LOG_FILE(1,"nblocks too large = 0x%llx",nblocks); + CBLK_TRACE_LOG_FILE(1,"nblocks too large = 0x%llx, max_transfer_size = 0x%llx",nblocks, + chunk->stats.max_transfer_size); errno = EINVAL; return -1; @@ -2423,7 +2443,8 @@ static inline int _cblk_aread(cflsh_chunk_t *chunk,void *buf,cflash_offset_t lba if (nblocks > chunk->stats.max_transfer_size) { - CBLK_TRACE_LOG_FILE(1,"nblocks too large = 0x%llx",nblocks); + CBLK_TRACE_LOG_FILE(1,"nblocks too large = 0x%llx, max_transfer_size = 0x%llx",nblocks, + chunk->stats.max_transfer_size); errno = EINVAL; return -1; @@ -2539,7 +2560,11 @@ int cblk_aread(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblocks { cflsh_chunk_t *chunk; - errno = 0; + if (flags & CBLK_GROUP_MASK) + { + errno = EINVAL; + return -1; + } CBLK_TRACE_LOG_FILE(5,"chunk_id = %d, lba = 0x%llx, nblocks = 0x%llx, buf = %p",chunk_id,lba,(uint64_t)nblocks,buf); @@ -2548,7 +2573,6 @@ int cblk_aread(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblocks return -1; } - CFLASH_BLOCK_RD_RWLOCK(cflsh_blk.global_lock); chunk = CBLK_GET_CHUNK_HASH(chunk_id,TRUE); if (chunk == NULL) { @@ -2556,12 +2580,10 @@ int cblk_aread(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblocks CBLK_TRACE_LOG_FILE(1,"chunk not found, chunk_id = %d", chunk_id); - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); errno = EINVAL; return -1; } - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); return (_cblk_aread(chunk,buf,lba,nblocks,tag,status,flags)); } @@ -2594,7 +2616,8 @@ static inline int _cblk_awrite(cflsh_chunk_t *chunk,void *buf,cflash_offset_t lb if (nblocks > chunk->stats.max_transfer_size) { - CBLK_TRACE_LOG_FILE(1,"nblocks too large = 0x%llx",nblocks); + CBLK_TRACE_LOG_FILE(1,"nblocks too large = 0x%llx, max_transfer_size = 0x%llx",nblocks, + chunk->stats.max_transfer_size); errno = EINVAL; return -1; @@ -2705,10 +2728,14 @@ int cblk_awrite(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblock cflsh_chunk_t *chunk; + if (flags & CBLK_GROUP_MASK) + { + errno = EINVAL; + return -1; + } - errno = 0; - - CBLK_TRACE_LOG_FILE(5,"chunk_id = %d, lba = 0x%llx, nblocks = 0x%llx, buf = %p",chunk_id,lba,(uint64_t)nblocks,buf); + CBLK_TRACE_LOG_FILE(5,"chunk_id = %d, lba = 0x%llx, nblocks = 0x%llx, buf = %p flags:%x", + chunk_id,lba,(uint64_t)nblocks,buf, flags); if (CBLK_VALIDATE_RW(chunk_id,buf,lba,nblocks)) { @@ -2717,7 +2744,6 @@ int cblk_awrite(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblock } - CFLASH_BLOCK_RD_RWLOCK(cflsh_blk.global_lock); chunk = CBLK_GET_CHUNK_HASH(chunk_id,TRUE); if (chunk == NULL) { @@ -2725,14 +2751,11 @@ int cblk_awrite(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblock CBLK_TRACE_LOG_FILE(1,"chunk not found, chunk_id = %d", chunk_id); - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); errno = EINVAL; return -1; } - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); - return (_cblk_awrite(chunk,buf,lba,nblocks,tag,status,flags)); } @@ -2748,7 +2771,6 @@ int cblk_awrite(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblock * tag - Tag associated with this request that * completed. * flags - Flags on this request. - * harvest - 0 to skip harvest call, else do harvest * * RETURNS: * 0 for good completion, but requested tag has not yet completed. @@ -2759,7 +2781,7 @@ int cblk_awrite(chunk_id_t chunk_id,void *buf,cflash_offset_t lba, size_t nblock */ static inline int _cblk_aresult(cflsh_chunk_t *chunk,int *tag, uint64_t *status, - int flags, int harvest) + int flags) { int rc = 0; @@ -2789,7 +2811,11 @@ static inline int _cblk_aresult(cflsh_chunk_t *chunk,int *tag, uint64_t *status, *status = 0; - if (chunk->flags & CFLSH_CHNK_NO_BG_TD && harvest) + if (chunk->flags & CFLSH_CHNK_NO_BG_TD +#ifndef BLOCK_FILEMODE_ENABLED + && (flags & CBLK_ARESULT_NO_HARVEST)==0 +#endif + ) { int do_poll=1, lib_flags = CFLASH_ASYNC_OP; @@ -2841,7 +2867,7 @@ static inline int _cblk_aresult(cflsh_chunk_t *chunk,int *tag, uint64_t *status, * No commands are active to wait on. */ - errno = EINVAL; + errno = ENOENT; CFLASH_BLOCK_UNLOCK(chunk->lock); return -1; @@ -2878,7 +2904,11 @@ static inline int _cblk_aresult(cflsh_chunk_t *chunk,int *tag, uint64_t *status, * but has not yet been seen by a caller. */ - *tag = cmdi->index; + if (cmdi->flags & CFLSH_CMD_INFO_UTAG) { + *tag = cmdi->user_tag; + } else { + *tag = cmdi->index; + } break; } @@ -2962,8 +2992,13 @@ static inline int _cblk_aresult(cflsh_chunk_t *chunk,int *tag, uint64_t *status, * This is an async command that completed * but has not yet been seen by a caller. */ - - *tag = cmdi->index; + + + if (cmdi->flags & CFLSH_CMD_INFO_UTAG) { + *tag = cmdi->user_tag; + } else { + *tag = cmdi->index; + } break; } @@ -3210,8 +3245,6 @@ static inline int _cblk_aresult(cflsh_chunk_t *chunk,int *tag, uint64_t *status, rc = cmd->cmdi->transfer_size; - errno = 0; - if (rc == 0) { /* @@ -3238,6 +3271,8 @@ static inline int _cblk_aresult(cflsh_chunk_t *chunk,int *tag, uint64_t *status, } + CBLK_IO_LATENCY(chunk, cmd); + if (cmd->cmdi->flags & CFLSH_ASYNC_IO) { if (cmd->cmdi->flags & CFLSH_MODE_READ) { @@ -3393,8 +3428,11 @@ int cblk_aresult(chunk_id_t chunk_id,int *tag, uint64_t *status, int flags) { cflsh_chunk_t *chunk; - - errno = 0; + if (flags & CBLK_GROUP_MASK) + { + errno = EINVAL; + return -1; + } CBLK_TRACE_LOG_FILE(5,"chunk_id = %d, flags = 0x%x, tag = 0x%x",chunk_id,flags,*tag); @@ -3406,8 +3444,6 @@ int cblk_aresult(chunk_id_t chunk_id,int *tag, uint64_t *status, int flags) return -1; } - CFLASH_BLOCK_RD_RWLOCK(cflsh_blk.global_lock); - chunk = CBLK_GET_CHUNK_HASH(chunk_id,TRUE); if (chunk == NULL) { @@ -3415,14 +3451,11 @@ int cblk_aresult(chunk_id_t chunk_id,int *tag, uint64_t *status, int flags) CBLK_TRACE_LOG_FILE(1,"chunk not found, chunk_id = %d", chunk_id); - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); errno = EINVAL; return -1; } - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); - - return (_cblk_aresult(chunk,tag,status,flags,1)); + return (_cblk_aresult(chunk,tag,status,flags)); } @@ -3470,12 +3503,9 @@ int cblk_listio(chunk_id_t chunk_id, int avail_completions; int harvest=1;/* send TRUE to aresult for the first cmd in the list only */ - errno = 0; - CBLK_TRACE_LOG_FILE(5,"chunk_id = %d, issue_items = 0x%x, pending_items = 0x%x, wait_items = 0x%x, timeout = 0x%llx,flags = 0x%x", chunk_id,issue_items,pending_items,wait_items,timeout,flags); - CFLASH_BLOCK_RD_RWLOCK(cflsh_blk.global_lock); chunk = CBLK_GET_CHUNK_HASH(chunk_id,TRUE); if (chunk == NULL) { @@ -3483,15 +3513,11 @@ int cblk_listio(chunk_id_t chunk_id, CBLK_TRACE_LOG_FILE(1,"chunk not found, chunk_id = %d", chunk_id); - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); errno = EINVAL; return -1; } - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); - - /* * Do some validation on lists past before we do any processing. @@ -3842,10 +3868,11 @@ int cblk_listio(chunk_id_t chunk_id, } - rc = _cblk_aresult(chunk,&(io->tag),&status,io_flags,harvest); + rc = _cblk_aresult(chunk,&(io->tag),&status,io_flags); if (harvest) { - harvest=0; + io_flags |= CBLK_ARESULT_NO_HARVEST; + harvest=0; } if (rc < 0) { @@ -4233,9 +4260,12 @@ int cblk_clone_after_fork(chunk_id_t chunk_id, int mode, int flags) int rc = 0; cflsh_chunk_t *chunk; - errno = 0; - + if (flags & CBLK_GROUP_MASK) + { + flags &= ~CBLK_GROUP_ID; + return cblk_cg_clone_after_fork(chunk_id, mode, flags); + } CBLK_TRACE_LOG_FILE(5,"orig_chunk_id = %d mode = 0x%x, flags = 0x%x", chunk_id,mode,flags); @@ -4247,9 +4277,10 @@ int cblk_clone_after_fork(chunk_id_t chunk_id, int mode, int flags) return -1; } - CFLASH_BLOCK_WR_RWLOCK(cflsh_blk.global_lock); chunk = CBLK_GET_CHUNK_HASH(chunk_id,TRUE); + CFLASH_BLOCK_WR_RWLOCK(cflsh_blk.global_lock); + if (chunk == NULL) { @@ -4276,31 +4307,7 @@ int cblk_clone_after_fork(chunk_id_t chunk_id, int mode, int flags) CFLASH_BLOCK_LOCK(chunk->lock); - /* - * Since we forked the child, get its new PID - * and clear its old process name if known. - */ - - cflsh_blk.caller_pid = getpid(); - - - if (cflsh_blk.process_name) { - - free(cflsh_blk.process_name); - } - - - - /* - * Since we forked the child, if we have tracing turned - * on for a trace file per process, then we need to - * open the new file for this child's PID. The routine - * cblk_setup_trace_files will handle the situation - * where multiple chunks are cloned and using the same - * new trace file. - */ - cblk_setup_trace_files(TRUE); if (chunk->num_active_cmds > 0) { @@ -4445,9 +4452,8 @@ int cblk_clone_after_fork(chunk_id_t chunk_id, int mode, int flags) */ - // TODO: ?? this assumes rrq size matches this chunk's num_cmds - bzero((void *)chunk->path[chunk->cur_path]->afu->p_hrrq_start , - (sizeof(*(chunk->path[chunk->cur_path]->afu->p_hrrq_start)) * chunk->path[chunk->cur_path]->afu->num_rrqs)); + + bzero((void *)chunk->path[chunk->cur_path]->afu->p_hrrq_start ,chunk->path[chunk->cur_path]->afu->size_rrq); chunk->path[chunk->cur_path]->afu->p_hrrq_curr = chunk->path[chunk->cur_path]->afu->p_hrrq_start; @@ -4465,6 +4471,20 @@ int cblk_clone_after_fork(chunk_id_t chunk_id, int mode, int flags) chunk->cmd_curr = chunk->cmd_start; + if (chunk->path[chunk->cur_path]->afu->flags & CFLSH_AFU_SQ) { + + /* + * If this AFU is using Submission Queues (SQ) + * then, reinitialize the SQ. + */ + bzero((void *)chunk->path[chunk->cur_path]->afu->p_sq_start ,chunk->path[chunk->cur_path]->afu->size_sq); + + chunk->path[chunk->cur_path]->afu->p_sq_curr = chunk->path[chunk->cur_path]->afu->p_sq_start; + + + } + + } @@ -4499,10 +4519,12 @@ int cblk_get_stats(chunk_id_t chunk_id, chunk_stats_t *stats, int flags) int rc = 0; cflsh_chunk_t *chunk; + if (flags & CBLK_GROUP_MASK) + { + flags &= ~CBLK_GROUP_ID; + return cblk_cg_get_stats(chunk_id, stats, flags); + } - - errno = 0; - CBLK_TRACE_LOG_FILE(6,"flags = 0x%x",flags); @@ -4522,9 +4544,10 @@ int cblk_get_stats(chunk_id_t chunk_id, chunk_stats_t *stats, int flags) return -1; } - CFLASH_BLOCK_RD_RWLOCK(cflsh_blk.global_lock); chunk = CBLK_GET_CHUNK_HASH(chunk_id,TRUE); + CFLASH_BLOCK_RD_RWLOCK(cflsh_blk.global_lock); + if (chunk == NULL) { diff --git a/src/block/cflash_block_cgroup.c b/src/block/cflash_block_cgroup.c new file mode 100644 index 00000000..b059eac1 --- /dev/null +++ b/src/block/cflash_block_cgroup.c @@ -0,0 +1,918 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/block/cflash_block_cgroup.c $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ + +#define CFLSH_BLK_FILENUM 0x0700 +#include "cflash_block_cgroup.h" + +int init_done=0; +cflash_block_grp_global_t cflash_block_grp_global; + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_init + * + * FUNCTION: init chunk_grp hash table structs + * + * INPUTS: + * + * RETURNS: + * 0 success, otherwise error. + ******************************************************************************/ +int cblk_cg_init(void) +{ + int rc = 0; + int i = 0; + + if (init_done) {return 0;} + + memset(&cflash_block_grp_global,0,sizeof(cflash_block_grp_global)); + + for (i=0; ichunk_list = malloc(sizeof(cflash_block_chunk_entry_t)*num_chunks); + if (!p_cg->chunk_list) + { + CBLK_TRACE_LOG_FILE(9, "malloc failed, errno:%d", errno); + rc=-1; goto error; + } + + bzero(p_cg->chunk_list, sizeof(cflash_block_chunk_entry_t)*num_chunks); + + p_cg->num_chunks = num_chunks; + + for (i=0; ichunk_list[i].id = NULL_CHUNK_ID;} + + pthread_mutex_lock(&cflash_block_grp_global.lock); + p_cg->id = cflash_block_grp_global.next_group_id++; + pthread_mutex_unlock(&cflash_block_grp_global.lock); + + *id = p_cg->id; + + CBLK_GET_CG_HASH_PTR(p_hash, *id); + if (!p_hash) + { + CBLK_TRACE_LOG_FILE(9, "no CG for id:%d", *id); + rc=-3; goto error; + } + + /* insert new CG at hash tail */ + pthread_mutex_lock(&p_hash->lock); + if (!p_hash->head) + { + CBLK_TRACE_LOG_FILE(9, "new p_cg:%p at p_hash:%p, head", p_cg, p_hash); + p_hash->head = p_cg; + } + else + { + CBLK_TRACE_LOG_FILE(9, "new p_cg:%p at p_hash:%p, tail", p_cg, p_hash); + p_cg->prev = p_hash->tail; + p_hash->tail->next = p_cg; + } + p_hash->tail = p_cg; + pthread_mutex_unlock(&p_hash->lock); + + *p_ret_cg = p_cg; + goto done; + +error: + if (p_cg && p_cg->chunk_list) {free(p_cg->chunk_list);} + if (p_cg) {free(p_cg);} + +done: + CBLK_TRACE_LOG_FILE(9, "id:%d num_chunks:%d rc:%d", *id, num_chunks, rc); + return 0; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_get_next_chunk_id + * + * FUNCTION: Get the next chunk id from the chunk group + * for I/O. + * + * INPUTS: + * file index - file identifier + * + * RETURNS: + * NULL_CHUNK_ID for error. + * Otherwise the chunk_id is returned. + ******************************************************************************/ +chunk_id_t cblk_cg_get_next_chunk_id(chunk_cg_id_t cgid) +{ + chunk_id_t chunk_id = NULL_CHUNK_ID; + cflash_block_grp_fd_t *p_cg = NULL; + int next = 0; + + p_cg = CBLK_GET_CG_HASH(cgid); + + if (p_cg) + { + if (p_cg->chunk_list == NULL) + { + CBLK_TRACE_LOG_FILE(9, "not found, cgid:%d", cgid); + return chunk_id; + } + next = p_cg->cur_chunk_index; + chunk_id = p_cg->chunk_list[next].id; + p_cg->cur_chunk_index = (next+1) % p_cg->num_chunks; + } + + CBLK_TRACE_LOG_FILE(9, "return cgid:%d", chunk_id); + return chunk_id; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_open + * + * FUNCTION: Opens a chunk group + * + * INPUTS: + * flags - Flags for open + * + * RETURNS: + * ULL_CHUNK_CG_ID for error. + * Otherwise the chunk_id is returned. + ******************************************************************************/ +chunk_cg_id_t cblk_cg_open(const char *path, + int max_num_requests, + int mode, + int num_chunks, + chunk_ext_arg_t ext, + int flags) +{ + chunk_cg_id_t id = NULL_CHUNK_CG_ID; + cflash_block_grp_fd_t *p_cg = NULL; + int i = 0; + + if (!init_done) {cblk_cg_init();} + + errno = 0; + CBLK_TRACE_LOG_FILE(4, "path:%s num_chunks:%d ext:%d flags:%x", + path, num_chunks, ext, flags); + + if (num_chunks<=0) + { + CBLK_TRACE_LOG_FILE(9, "bad parms, num_chunks=0"); + errno=EINVAL; goto error_open; + } + + if (flags & CBLK_GROUP_RAID0) + { + id = cblk_r0_open(path, max_num_requests, mode, ext, flags); + } + else + { + if (flags & CBLK_OPN_VIRT_LUN) + { + CBLK_TRACE_LOG_FILE(4, "VLUN not supported for CG\n"); + errno = EINVAL; + goto error_open; + } + flags &= ~CBLK_GROUP_ID; + + if (cblk_cg_alloc(&id, num_chunks, &p_cg)) + { + CBLK_TRACE_LOG_FILE(9, "cg_alloc failed"); + goto error_open; + } + if (p_cg == NULL) + { + CBLK_TRACE_LOG_FILE(9, "NULL from cg_alloc"); + goto error_open; + } + + for (i=0; inum_chunks; i++) + { + p_cg->chunk_list[i].id = cblk_open(path, max_num_requests, + mode, ext, flags); + if (p_cg->chunk_list[i].id == NULL_CHUNK_ID) + { + CBLK_TRACE_LOG_FILE(9, "open failed errno:%d", errno); + goto error_open; + } + CBLK_TRACE_LOG_FILE(2, "new %d:%d", id, p_cg->chunk_list[i].id); + } + } + goto done; + +error_open: + if (id != NULL_CHUNK_CG_ID) {cblk_cg_close(id, flags);} + id = NULL_CHUNK_CG_ID; + +done: + CBLK_TRACE_LOG_FILE(9, "cgid:%d", id); + return id; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_close + * + * FUNCTION: Closes a chunk group. + * + * INPUTS: + * flags - Flags for close + * + * RETURNS: + * 0 for good completion, ERRNO on error + ******************************************************************************/ +int cblk_cg_close(chunk_cg_id_t cgid, int flags) +{ + cflash_block_grp_fd_hash_t *p_hash = NULL; + cflash_block_grp_fd_t *p_cg = NULL; + int rc = 0; + int i = 0; + + p_cg = CBLK_GET_CG_HASH(cgid); + if (p_cg == NULL) + { + CBLK_TRACE_LOG_FILE(9, "cgid:%d not found", cgid); + errno=EINVAL; rc=-1; goto done; + } + + flags &= ~CBLK_GROUP_MASK; + + for (i=0; inum_chunks; i++) + { + if (p_cg->chunk_list[i].id != NULL_CHUNK_ID) + { + CBLK_TRACE_LOG_FILE(9, "closing %d:%d",cgid,p_cg->chunk_list[i].id); + rc=cblk_close(p_cg->chunk_list[i].id, flags); + } + else + { + CBLK_TRACE_LOG_FILE(9, "cannot close NULL_CHUNK_ID, cgid:%d", cgid); + } + } + + CBLK_GET_CG_HASH_PTR(p_hash, cgid); + + pthread_mutex_lock(&p_hash->lock); + if (p_hash->head == p_cg) + { + CBLK_TRACE_LOG_FILE(9, "delete p_cg:%p at p_hash:%p head",p_cg, p_hash); + p_hash->head = p_cg->next; + if (p_cg->next) {p_cg->next->prev = NULL;} + if (p_hash->tail == p_cg) {p_hash->tail = NULL;} + } + else if (p_hash->tail == p_cg) + { + CBLK_TRACE_LOG_FILE(9, "delete p_cg:%p at p_hash:%p tail",p_cg, p_hash); + p_cg->prev->next = NULL; + p_hash->tail = p_cg->prev; + } + else + { + CBLK_TRACE_LOG_FILE(9, "delete p_cg:%p at p_hash:%p mid", p_cg, p_hash); + p_cg->prev->next = p_cg->next; + p_cg->next->prev = p_cg->prev; + } + pthread_mutex_unlock(&p_hash->lock); + + if (p_cg->chunk_list) {free(p_cg->chunk_list);} + if (p_cg) {free(p_cg);} + +done: + CBLK_TRACE_LOG_FILE(9, "closed cgid:%d flags:%x rc:%d", cgid, flags, rc); + return rc; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_get_stats + * + * FUNCTION: Return statistics for this chunk group. + * + * + * INPUTS: + * chunk_id - Chunk identifier + * tag - Pointer to stats returned + * + * RETURNS: + * 0 for good completion, ERRNO on error + ******************************************************************************/ +int cblk_cg_get_stats(chunk_cg_id_t cgid, chunk_stats_t *stats, int flags) +{ + cflash_block_grp_fd_t *p_cg = NULL; + chunk_stats_t cs = {0}; + int rc = 0; + int i = 0; + + if (!stats) {errno=EINVAL; rc=-1; goto done;} + memset(stats,0,sizeof(chunk_stats_t)); + + p_cg = CBLK_GET_CG_HASH(cgid); + if (p_cg == NULL) {errno=EINVAL; rc=-1; goto done;} + + flags &= ~CBLK_GROUP_MASK; + + for (i=0; inum_chunks; i++) + { + if (p_cg->chunk_list[i].id == NULL_CHUNK_ID) {continue;} + + if (cblk_get_stats(p_cg->chunk_list[i].id, &cs, flags) == 0) + { + stats->max_transfer_size += cs.max_transfer_size; + stats->num_reads += cs.num_reads; + stats->num_writes += cs.num_writes; + stats->num_areads += cs.num_areads; + stats->num_awrites += cs.num_awrites; + stats->num_act_reads += cs.num_act_reads; + stats->num_act_writes += cs.num_act_writes; + stats->num_act_areads += cs.num_act_areads; + stats->num_act_awrites += cs.num_act_awrites; + stats->max_num_act_writes += cs.max_num_act_writes; + stats->max_num_act_reads += cs.max_num_act_reads; + stats->max_num_act_awrites += cs.max_num_act_awrites; + stats->max_num_act_areads += cs.max_num_act_areads; + stats->num_blocks_read += cs.num_blocks_read; + stats->num_blocks_written += cs.num_blocks_written; + stats->num_errors += cs.num_errors; + stats->num_aresult_no_cmplt += cs.num_aresult_no_cmplt; + stats->num_retries += cs.num_retries; + stats->num_timeouts += cs.num_timeouts; + stats->num_no_cmds_free += cs.num_no_cmds_free; + stats->num_no_cmd_room += cs.num_no_cmd_room; + stats->num_no_cmds_free_fail += cs.num_no_cmds_free_fail; + stats->num_fc_errors += cs.num_fc_errors; + stats->num_port0_linkdowns += cs.num_port0_linkdowns; + stats->num_port1_linkdowns += cs.num_port1_linkdowns; + stats->num_port0_no_logins += cs.num_port0_no_logins; + stats->num_port1_no_logins += cs.num_port1_no_logins; + stats->num_port0_fc_errors += cs.num_port0_fc_errors; + stats->num_port1_fc_errors += cs.num_port1_fc_errors; + stats->num_cc_errors += cs.num_cc_errors; + stats->num_afu_errors += cs.num_afu_errors; + stats->num_capi_false_reads += cs.num_capi_false_reads; + stats->num_capi_adap_resets += cs.num_capi_adap_resets; + stats->num_capi_afu_errors += cs.num_capi_afu_errors; + stats->num_capi_afu_intrpts += cs.num_capi_afu_intrpts; + stats->num_capi_unexp_afu_intrpts += cs.num_capi_unexp_afu_intrpts; + stats->num_active_threads += cs.num_active_threads; + stats->max_num_act_threads += cs.max_num_act_threads; + stats->num_cache_hits += cs.num_cache_hits; + } + } + +done: + return rc; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_get_num_chunks + * + * FUNCTION: Return num_chunks for this chunk group. + * + * + * INPUTS: + * chunk_id - Chunk identifier + * + * RETURNS: + * <0 fail, >0 good completion + ******************************************************************************/ +int cblk_cg_get_num_chunks(chunk_cg_id_t cgid, int flags) +{ + cflash_block_grp_fd_t *p_cg = NULL; + int num = 0; + + p_cg = CBLK_GET_CG_HASH(cgid); + if (!p_cg) + { + CBLK_TRACE_LOG_FILE(9, "not found, cgid:%d", cgid); + errno=EINVAL; num=-1; goto done; + } + + num = p_cg->num_chunks; + +done: + return num; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_get_lun_size + * + * FUNCTION: Returns the number of blocks + * associated with the physical lun(s) + * that contains this chunk group + * + * INPUTS: + * chunk_id - Chunk identifier + * nblocks - Address of returned number of + * blocks for this lun + * flags - Flags for open + * + * RETURNS: + * 0 for good completion, ERRNO on error + ******************************************************************************/ +int cblk_cg_get_lun_size(chunk_cg_id_t cgid, size_t *nblocks, int flags) +{ + cflash_block_grp_fd_t *p_cg = NULL; + int rc = 0; + int i = 0; + size_t blks = 0; + + if (!nblocks) {errno=EINVAL; rc=-1; goto done;} + *nblocks=0; + + p_cg = CBLK_GET_CG_HASH(cgid); + if (!p_cg) {errno=ENODEV; rc=-1; goto done;} + + if (p_cg->flags & CBLK_GROUP_RAID0) + { + for (i=0; inum_chunks; i++) + { + rc = cblk_get_lun_size(p_cg->chunk_list[i].id, &blks, 0); + if (rc) {goto done;} + *nblocks += blks; + } + } + else {rc = cblk_get_lun_size(p_cg->chunk_list[0].id, nblocks, 0);} + +done: + CBLK_TRACE_LOG_FILE(4, "cgid:%d *nblocks:%d flags:%x rc:%d", + cgid, *nblocks, flags, rc); + return rc; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_get_size + * + * FUNCTION: Returns the number of blocks + * associated with this chunk + * + * INPUTS: + * chunk_id - Chunk identifier + * nblocks - Address of returned number of + * blocks for this lun + * flags - Flags for open + * + * RETURNS: + * 0 for good completion, ERRNO on error + ******************************************************************************/ +int cblk_cg_get_size(chunk_cg_id_t cgid, size_t *nblocks, int flags) +{ + cflash_block_grp_fd_t *p_cg = NULL; + int rc = 0; + int i = 0; + size_t blks = 0; + + if (!nblocks) {errno=EINVAL; rc=-1; goto done;} + *nblocks=0; + + p_cg = CBLK_GET_CG_HASH(cgid); + if (!p_cg) {errno=EINVAL; rc=-1; goto done;} + + if (p_cg->flags & CBLK_GROUP_RAID0) + { + for (i=0; inum_chunks; i++) + { + rc = cblk_get_size(p_cg->chunk_list[i].id, &blks, 0); + if (rc) {goto done;} + *nblocks += blks; + } + } + else {rc = cblk_get_size(p_cg->chunk_list[0].id, nblocks, 0);} + +done: + CBLK_TRACE_LOG_FILE(4, "cgid:%d *nblocks:%d flags:%x rc:%d", + cgid, *nblocks, flags, rc); + return rc; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_set_size + * + * FUNCTION: Sets the number of blocks + * associated with the physical lun that + * contains this chunk + * + * INPUTS: + * chunk_id - Chunk identifier + * nblocks - Address of returned number of + * blocks for this lun + * flags - Flags for open + * + * RETURNS: + * 0 for good completion, ERRNO on error + ******************************************************************************/ +int cblk_cg_set_size(chunk_cg_id_t cgid, size_t nblocks, int flags) +{ + int rc = 0; + + if (flags & CBLK_GROUP_RAID0) + { + rc = cblk_r0_set_size(cgid, nblocks); + } + else + { + errno = EINVAL; + rc = -1; + } + return rc; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_read + * + * FUNCTION: Reads data from the specified offset in the chunk + * group and places that data in the specified buffer. + * This request is a blocking read request (i.e. + * it will not return until either data is read or + * an error is encountered). + * + * + * INPUTS: + * chunk_id - Chunk identifier + * buf - Buffer to read data into + * lba - starting LBA (logical Block Address) + * in chunk to read data from. + * nblocks - Number of blocks to read. + * + * RETURNS: + * 0 for good completion, ERRNO on error + ******************************************************************************/ +int cblk_cg_read(chunk_cg_id_t cgid, + void *pbuf, + cflash_offset_t lba, + size_t nblocks, + int flags) +{ + int rc = 0; + chunk_id_t chunk_id = 0; + + if (flags & CBLK_GROUP_RAID0) + { + rc = cblk_r0_read(cgid, pbuf, lba, nblocks, flags); + } + else + { + errno = 0; + chunk_id = cblk_cg_get_next_chunk_id(cgid); + + if (chunk_id == NULL_CHUNK_ID) + { + errno = EINVAL; + rc = -1; + goto done; + } + flags &= ~CBLK_GROUP_MASK; + rc = cblk_read(chunk_id, pbuf, lba, nblocks, flags); + } + +done: + CBLK_TRACE_LOG_FILE(9, "cgid:%2d:%2d lba:%d nblocks:%d flags:%x rc:%d", + cgid, chunk_id, lba, nblocks, flags, rc); + return rc; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_write + * + * FUNCTION: Writes data to the specified offset in the chunk + * group from the specified buffer. This request is + * a blocking write request (i.e.it will not + * return until either data is written or + * an error is encountered). + * + * INPUTS: + * chunk_id - Chunk identifier + * buf - Buffer to write data from + * lba - starting LBA (logical Block Address) + * in chunk to write data to. + * nblocks - Number of blocks to write. + * + * RETURNS: + * 0 for good completion, ERRNO on error + ******************************************************************************/ +int cblk_cg_write(chunk_cg_id_t cgid, + void *pbuf, + cflash_offset_t lba, + size_t nblocks, + int flags) +{ + int rc = 0; + chunk_id_t chunk_id = 0; + + if (flags & CBLK_GROUP_RAID0) + { + rc = cblk_r0_write(cgid, pbuf, lba, nblocks, flags); + } + else + { + errno = 0; + chunk_id = cblk_cg_get_next_chunk_id(cgid); + + if (chunk_id == NULL_CHUNK_ID) + { + errno = EINVAL; + rc = -1; + goto done; + } + flags &= ~CBLK_GROUP_MASK; + rc = cblk_write(chunk_id, pbuf, lba, nblocks, flags); + } + +done: + CBLK_TRACE_LOG_FILE(9, "cgid:%d lba:%d nblocks:%d flags:%x rc:%d", + cgid, lba, nblocks, flags, rc); + return rc; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_aread + * + * FUNCTION: Reads data from the specified offset in the chunk + * group and places that data in the specified buffer. + * This request is an asynchronous read request (i.e. + * it will return as soon as it has issued the request + * to the device. It will not wait for a response from + * the device.). + * + * INPUTS: + * chunk_id - Chunk identifier + * buf - Buffer to read data into + * lba - starting LBA (logical Block Address) + * in chunk to read data from. + * nblocks - Number of blocks to read. + * tag - Tag associated with this request. + * + * RETURNS: + * 0 for good completion, + * >0 Read data was in cache and was read. + * -1 for error with ERRNO set + ******************************************************************************/ +int cblk_cg_aread(chunk_cg_id_t cgid, + void *pbuf, + cflash_offset_t lba, + size_t nblocks, + cflsh_cg_tag_t *ptag, + cblk_arw_status_t *p_arwstatus, + int flags) +{ + int rc = 0; + + if (flags & CBLK_GROUP_RAID0) + { + rc = cblk_r0_aread(cgid, pbuf, lba, nblocks, ptag, p_arwstatus, flags); + } + else + { + errno = EINVAL; + rc = -1; + } + return rc; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_awrite + * + * FUNCTION: Writes data to the specified offset in the chunk + * group from the specified buffer. This request is + * an asynchronous write request (i.e.it will + * return as soon as it has issued the request + * to the device. It will not wait for a response from + * the device.). + * + * + * INPUTS: + * chunk_id - Chunk identifier + * buf - Buffer to write data from + * lba - starting LBA (logical Block Address) + * in chunk to write data to. + * nblocks - Number of blocks to write. + * tag - Tag associated with this request. + * + * RETURNS: + * 0 for good completion, ERRNO on error + ******************************************************************************/ +int cblk_cg_awrite(chunk_cg_id_t cgid, + void *pbuf, + cflash_offset_t lba, + size_t nblocks, + cflsh_cg_tag_t *ptag, + cblk_arw_status_t *p_arwstatus, + int flags) +{ + int rc = 0; + + if (flags & CBLK_GROUP_RAID0) + { + rc = cblk_r0_awrite(cgid, pbuf, lba, nblocks, ptag, p_arwstatus, flags); + } + else + { + errno = EINVAL; + rc = -1; + } + return rc; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_aresult + * + * FUNCTION: Waits for asynchronous read or writes to complete + * and returns the status of them. + * + * + * INPUTS: + * chunk_id - Chunk identifier + * tag - Tag associated with this request that + * completed. + * flags - Flags on this request. + * + * RETURNS: + * 0 for good completion, but requested tag has not yet completed. + * -1 for error and errno is set. + * > 0 for good completion where the return value is the + * number of blocks transferred. + ******************************************************************************/ +int cblk_cg_aresult(chunk_cg_id_t cgid, + cflsh_cg_tag_t *ptag, + uint64_t *p_arwstatus, + int flags) +{ + int rc = 0; + + if (flags & CBLK_GROUP_RAID0) + { + rc = cblk_r0_aresult(cgid, ptag, p_arwstatus, flags); + } + else + { + errno = EINVAL; + rc = -1; + } + return rc; +} + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: cblk_cg_clone_after_fork + * + * FUNCTION: clone a (virtual lun) chunk group with an existing (original) + * chunk group. This is useful if a process is forked and the + * child wants to read data from the parents chunk group. + * + * INPUTS: + * chunk_id - Chunk id to be cloned. + * mode - Access mode for the chunk + * (O_RDONLY, O_WRONLY, O_RDWR) + * flags - Flags on this request. + * + * RETURNS: + * 0 for good completion, ERRNO on error + ******************************************************************************/ +int cblk_cg_clone_after_fork(chunk_cg_id_t cgid, int mode, int flags) +{ + cflash_block_grp_fd_t *p_cg = NULL; + int rc = 0; + int i = 0; + + p_cg = CBLK_GET_CG_HASH(cgid); + + if (p_cg == NULL) {errno=EINVAL; rc=-1; goto done;} + + if (flags & CBLK_GROUP_RAID0) + { + flags &= ~CBLK_GROUP_MASK; + for (i=0; inum_chunks; i++) + { + rc = cblk_clone_after_fork(p_cg->chunk_list[i].id, mode, flags); + if (rc) {break;} + } + } + else + { + errno = EINVAL; + rc = -1; + } + +done: + CBLK_TRACE_LOG_FILE(9, "cgid:%d mode:%d flags:%x rc:%d", + cgid, mode, flags, rc); + return rc; +} diff --git a/src/block/cflash_block_cgroup.h b/src/block/cflash_block_cgroup.h new file mode 100644 index 00000000..103227a4 --- /dev/null +++ b/src/block/cflash_block_cgroup.h @@ -0,0 +1,133 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/block/cflash_block_group.h $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ + +#ifndef _H_CFLASH_BLOCK_GROUP +#define _H_CFLASH_BLOCK_GROUP + +#include "cflash_block_internal.h" +#include "cflash_block_protos.h" + + +#define MAX_CFLASH_GRP_FDS 64 +#define MAX_CG_HASH (MAX_CFLASH_GRP_FDS-1) + +/** + ******************************************************************************* + * \brief + * data structure for chunk + ******************************************************************************/ +typedef struct cflash_block_chunk_entry_s +{ + uint64_t flags; + chunk_id_t id; /* Chunk id */ +} cflash_block_chunk_entry_t; + +/** + ******************************************************************************* + * \brief + * data structure for a chunk group + ******************************************************************************/ +typedef struct cflash_block_grp_fd_s +{ + struct cflash_block_grp_fd_s *prev; /* Previous group in list */ + struct cflash_block_grp_fd_s *next; /* Next group in list */ + uint64_t flags; + chunk_cg_id_t id; /* file descriptor; */ + int num_chunks; /* Number of chunks used */ + int cur_chunk_index; /* cur index in chunk_list */ + cflash_block_chunk_entry_t *chunk_list; + cflsh_blk_lock_t lock; +} cflash_block_grp_fd_t; + +/** + ******************************************************************************* + * \brief + * root data structure pointing to a linked list of chunk groups + ******************************************************************************/ +typedef struct cflash_block_grp_fd_hash_s +{ + pthread_mutex_t lock; + cflash_block_grp_fd_t *head; + cflash_block_grp_fd_t *tail; +} cflash_block_grp_fd_hash_t; + +/** + ******************************************************************************* + * \brief + * global anchor block for Chunk Groups + ******************************************************************************/ +typedef struct cflash_block_grp_global_s +{ + pthread_mutex_t lock; + cflash_block_grp_fd_hash_t ht[MAX_CFLASH_GRP_FDS]; + int flags; + int next_group_id; +} cflash_block_grp_global_t; + +extern cflash_block_grp_global_t cflash_block_grp_global; + +#define CBLK_GET_CG_HASH_PTR(_p_hash, _id) \ + _p_hash=&cflash_block_grp_global.ht[_id & MAX_CG_HASH]; + +/** + ******************************************************************************* + * \brief + * \details + * + * NAME: CBLK_GET_CG_HASH + * + * FUNCTION: Find empty chunk group fd from file descriptor in the hash table + * + * INPUTS: + * file index - file identifier + * + * RETURNS: + * NULL = No chunk group found. + * pointer = chunk group found. + ******************************************************************************/ +static inline cflash_block_grp_fd_t* CBLK_GET_CG_HASH(int id) +{ + cflash_block_grp_fd_hash_t *p_hash = NULL; + cflash_block_grp_fd_t *p_grp = NULL; + + CBLK_GET_CG_HASH_PTR(p_hash, id); + + pthread_mutex_lock(&p_hash->lock); + p_grp = p_hash->head; + while (p_grp && p_grp->id != id) {p_grp = p_grp->next;} + pthread_mutex_unlock(&p_hash->lock); + + if (p_grp && p_grp->chunk_list == NULL) {p_grp = NULL;} + + return p_grp; +} + +/* cflash_block_cgroup.c protos */ +int cblk_cg_init(void); +int cblk_cg_alloc(int *id,int num_chunks,cflash_block_grp_fd_t **ret_chunk_grp); +int cblk_cg_free(cflash_block_grp_fd_t *chunk_grp); +chunk_id_t cblk_cg_get_next_chunk_id(chunk_cg_id_t cgid); + +#endif /* _H_CFLASH_BLOCK_GROUP */ diff --git a/src/block/cflash_block_inline.h b/src/block/cflash_block_inline.h index 881f0e17..e9c2b10b 100755 --- a/src/block/cflash_block_inline.h +++ b/src/block/cflash_block_inline.h @@ -30,7 +30,7 @@ #include "cflash_block_internal.h" #include "cflash_block_protos.h" - +#include @@ -379,16 +379,16 @@ static inline int CBLK_INVALID_CMD_CMDI(cflsh_chunk_t *chunk, - CBLK_TRACE_LOG_FILE(1,"cmdi = %p with index %d is invalid, chunk->num_cmds = %d", - cmdi,cmdi->index, chunk->num_cmds,fcn_name); + CBLK_TRACE_LOG_FILE(1,"cmdi = %p with index %d is invalid, chunk->num_cmds = %d, chunk->flags =0x%x for fcn = %s", + cmdi,cmdi->index, chunk->num_cmds,chunk->flags,fcn_name); return -1; } if (CFLSH_EYECATCH_CMDI(cmdi)) { - CBLK_TRACE_LOG_FILE(1,"Invalid eyecatcher cmdi = %p is invalid, chunk->num_cmds = %d", - cmdi,cmdi->index, chunk->num_cmds,fcn_name); + CBLK_TRACE_LOG_FILE(1,"Invalid eyecatcher cmdi = %p is invalid, chunk->num_cmds = %d, chunk->flags =0x%x for fcn = %s", + cmdi,cmdi->index, chunk->num_cmds,chunk->flags,fcn_name); return -1; } @@ -452,7 +452,7 @@ static inline int CBLK_INVALID_CMD_CMDI(cflsh_chunk_t *chunk, CBLK_TRACE_LOG_FILE(1, - "cmdi = %p with index %d has invalid free_next pointer = %p, chunk->num_cmds = %d for fcn = %s", + "cmdi = %p with index %d has invalid active_next pointer = %p, chunk->num_cmds = %d for fcn = %s", cmdi,cmdi->index, tmp_cmdi,chunk->num_cmds,fcn_name); return -1; @@ -471,7 +471,7 @@ static inline int CBLK_INVALID_CMD_CMDI(cflsh_chunk_t *chunk, CBLK_TRACE_LOG_FILE(1, - "cmdi = %p with index %d has invalid free_next pointer = %p, chunk->num_cmds = %d for fcn = %s", + "cmdi = %p with index %d has invalid active_prev pointer = %p, chunk->num_cmds = %d for fcn = %s", cmdi,cmdi->index, tmp_cmdi,chunk->num_cmds,fcn_name); return -1; @@ -773,8 +773,7 @@ static inline uint64_t CBLK_GET_INTRPT_STATUS(cflsh_chunk_t *chunk, int path_in /* * NAME: CBLK_INC_RRQ * - * FUNCTION: This routine is called whenever an RRQ has been processed and - * the path lock is needed. + * FUNCTION: This routine is called whenever an RRQ has been processed. * * * NOTE; This routine assumes the caller is holding chunk->lock. @@ -819,13 +818,66 @@ static inline void CBLK_INC_RRQ(cflsh_chunk_t *chunk, int path_index) + return ; +} + +/* + * NAME: CBLK_INC_SQ + * + * FUNCTION: This routine is called whenever an SQ entry has been queued. + * + * + * + * NOTE; This routine assumes the caller is holding chunk->lock. + * + * RETURNS: None + * + * + */ + +static inline void CBLK_INC_SQ(cflsh_chunk_t *chunk, int path_index) +{ + + + + if (chunk == NULL) { + + errno = EINVAL; + + return; + } + + if (chunk->path[path_index] == NULL) { + + CBLK_TRACE_LOG_FILE(1,"path is null"); + + errno = EINVAL; + + return; + + } + + if (chunk->path[path_index]->fcn_ptrs.inc_sq == NULL) { + + errno = EINVAL; + + return; + } + + + + chunk->path[path_index]->fcn_ptrs.inc_sq(chunk,path_index); + + + return ; } /* * NAME: CBLK_INC_RRQ_LOCK * - * FUNCTION: This routine is called whenever an RRQ has been processed. + * FUNCTION: This routine is called whenever an RRQ has been processed and + * the path lock is needed. * * * NOTE; This routine assumes the caller is holding chunk->lock. @@ -1122,6 +1174,79 @@ static inline int CBLK_ADAP_SETUP(cflsh_chunk_t *chunk, int path_index) return rc; } +/* + * NAME: CBLK_ADAP_SQ_SETUP + * + * FUNCTION: Sets up an adapter's SQ + * + * + * NOTE; This routine assumes the caller is holding chunk->lock. + * + * RETURNS: None + * + * + */ +static inline int CBLK_ADAP_SQ_SETUP(cflsh_chunk_t *chunk, int path_index) +{ + int rc = 0; + + if (chunk == NULL) { + + errno = EINVAL; + + return -1; + } + + if (chunk->path[path_index] == NULL) { + + CBLK_TRACE_LOG_FILE(1,"path is null"); + + errno = EINVAL; + + return -1; + + } + + if (chunk->path[path_index]->afu == NULL) { + + CBLK_TRACE_LOG_FILE(1,"afu is null"); + + errno = EINVAL; + + return -1; + + } + + if (chunk->path[path_index]->fcn_ptrs.adap_sq_setup == NULL) { + + errno = EINVAL; + + return -1; + } + + CFLASH_BLOCK_AFU_SHARE_LOCK(chunk->path[path_index]->afu); + + if (chunk->path[path_index]->afu->flags & CFLSH_AFU_HALTED) { + + /* + * If path is in a halted state then fail + * from this routine + */ + + + CFLASH_BLOCK_AFU_SHARE_UNLOCK(chunk->path[path_index]->afu); + + return -1; + + } + + rc = chunk->path[path_index]->fcn_ptrs.adap_sq_setup(chunk,path_index); + + CFLASH_BLOCK_AFU_SHARE_UNLOCK(chunk->path[path_index]->afu); + + return rc; +} + /* * NAME: CBLK_BUILD_ADAP_CMD @@ -2128,6 +2253,51 @@ static inline void CBLK_FREE_CMD(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t *cmd) } +/* + * NAME: CBLK_FREE_CMDI + * + * FUNCTION: Marks command info as free and ready for reuse + * + * + * INPUTS: + * chunk - The chunk to which a free + * command is needed. + * + * cmd - Pointer to found command. + * + * RETURNS: + * 0 - Command was found. + * otherwise - error + * + */ + +static inline void CBLK_FREE_CMDI(cflsh_chunk_t *chunk, cflsh_cmd_info_t *cmdi) +{ + if ((chunk == NULL) || + (cmdi == NULL)) { + + + return; + } + + cmdi->in_use = 0; + + /* + * Remove command from active list + */ + + CBLK_DQ_NODE(chunk->head_act,chunk->tail_act,cmdi,act_prev,act_next); + + /* + * Place command on free list + */ + + CBLK_Q_NODE_TAIL(chunk->head_free,chunk->tail_free,cmdi,free_prev,free_next); + + return; +} + + /* * NAME: CBLK_ISSUE_CMD * @@ -2222,6 +2392,50 @@ static inline int CBLK_ISSUE_CMD(cflsh_chunk_t *chunk, return rc; } +/* + * NAME: CBLK_IO_LATENCY + * + * FUNCTION: update latency vars for a rd/wr cmd, + * trace latency once per second per chunk + * + * Environment: This routine assumes the chunk mutex + * lock is held by the caller. + * + * INPUTS: + * chunk - Chunk the cmd is associated. + * cmd - Cmd which just completed + * + * RETURNS: + */ +static inline void CBLK_IO_LATENCY(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t *cmd) +{ + /* calc and save r/w latency */ + if (cmd->cmdi->flags & CFLSH_MODE_READ) + { + chunk->rcmd += 1; + chunk->rlat += UDELTA(cmd->stime,cflsh_blk.nspt); + } + else if (cmd->cmdi->flags & CFLSH_MODE_WRITE) + { + chunk->wcmd += 1; + chunk->wlat += UDELTA(cmd->stime,cflsh_blk.nspt); + } + + /* trace latency */ + if (cblk_log_verbosity >= 4) + { + if (SDELTA(chunk->ptime,cflsh_blk.nspt) > 1) + { + chunk->ptime = getticks(); + CBLK_TRACE_LOG_FILE(4, "id:%4ld tlat:%6ld rlat:%6ld wlat:%6ld", + chunk->index, + (chunk->rlat+chunk->wlat)/(chunk->rcmd+chunk->wcmd), + chunk->rlat/chunk->rcmd, + chunk->wlat/chunk->wcmd); + } + } +} + /* * NAME: CBLK_COMPLETE_CMD * @@ -2289,7 +2503,8 @@ static inline int CBLK_COMPLETE_CMD(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t *cmd, rc = -1; } - + CBLK_IO_LATENCY(chunk, cmd); + /* * This command completed, * clean it up. @@ -2384,9 +2599,13 @@ static inline int CBLK_COMPLETE_CMD(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t *cmd, CBLK_FREE_CMD(chunk,cmd); } - - CBLK_TRACE_LOG_FILE(8,"cmdi->in_use= 0x%x cmdi->lba = 0x%llx, rc = %d, chunk->index = %d, cmdi->flags = 0x%x", - cmdi->in_use,cmdi->lba,rc,chunk->index,cmdi->flags); + + CBLK_TRACE_LOG_FILE(5,"in_use:0x%x lba:%8llx rc:%d lat:%6ld " + "flags:0x%x id:%2d rd:%d", + cmdi->in_use,cmdi->lba,rc, + UDELTA(cmd->stime,cflsh_blk.nspt), + cmdi->flags, chunk->index, + ((cmdi->flags & CFLSH_MODE_READ)!=0)); return (rc); } @@ -2419,6 +2638,7 @@ static inline int CBLK_CHECK_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_index cflsh_cmd_info_t *cmdi; cflsh_cmd_mgm_t *p_cmd; int rc = 0; + int cmd_rc = 0; time_t timeout; @@ -2504,11 +2724,10 @@ static inline int CBLK_CHECK_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_index if (cmdi->cmd_time < timeout) { - CBLK_TRACE_LOG_FILE(1,"Timeout for for cmd lba = 0x%llx, cmd = 0x%llx cmd_index = %d, chunk->index = %d", + CBLK_TRACE_LOG_FILE(1,"Timeout for cmd lba = 0x%llx, cmd = 0x%llx cmd_index = %d, chunk->index = %d", cmdi->lba,(uint64_t)cmd,cmd->index,chunk->index); - - cmdi->status = ETIMEDOUT; + cmdi->status = 0; cmdi->transfer_size = 0; @@ -2527,11 +2746,29 @@ static inline int CBLK_CHECK_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_index CFLASH_BLOCK_LOCK(chunk->lock); - *cmd_complete = TRUE; + if ((cmdi->retry_count >= CAPI_CMD_MAX_RETRIES) || + (cmdi->status)) { + + /* + * Don't fail yet, until all retries have + * been attempted. Also if the resume command + * logic in the block library fails to re-issue + * the command, then cmdi->status will be non-zero. + * NOTE: We cleared cmdi->status before unlocking + * and calling reset_context. + */ - errno = ETIMEDOUT; + CBLK_TRACE_LOG_FILE(1,"Timeout for exceeded retries for cmd lba = 0x%llx, cmd = 0x%llx cmdi->retry_count = %d, cmdi->status = 0x%x", + cmdi->lba,(uint64_t)cmd,cmdi->retry_count,cmdi->status); - rc = -1; + cmdi->status = ETIMEDOUT; + + *cmd_complete = TRUE; + + errno = ETIMEDOUT; + + rc = -1; + } } } else { @@ -2596,7 +2833,7 @@ static inline int CBLK_CHECK_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_index p_cmdi->state = CFLSH_MGM_CMP; - rc = cblk_process_cmd(chunk,path_index,p_cmd); + cmd_rc = cblk_process_cmd(chunk,path_index,p_cmd); if (p_cmd == cmd) { @@ -2606,8 +2843,8 @@ static inline int CBLK_CHECK_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_index */ - if ((rc != CFLASH_CMD_RETRY_ERR) && - (rc != CFLASH_CMD_DLY_RETRY_ERR)) { + if ((cmd_rc != CFLASH_CMD_RETRY_ERR) && + (cmd_rc != CFLASH_CMD_DLY_RETRY_ERR)) { /* * Since we found our command completed and @@ -2629,6 +2866,11 @@ static inline int CBLK_CHECK_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_index #endif *cmd_complete = TRUE; + + if (cmd_rc == CFLASH_CMD_FATAL_ERR) { + + rc = -1; + } } @@ -2696,10 +2938,10 @@ static inline int CBLK_WAIT_FOR_IO_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_ } - CBLK_TRACE_LOG_FILE(9,"waiting for cmd with cmd_index = 0x%x on chunk->index = %d", - *cmd_index,chunk->index); - CBLK_TRACE_LOG_FILE(9,"chunk->fd = %d, poll_fd= %d", - chunk->fd,chunk->path[path_index]->afu->poll_fd); + CBLK_TRACE_LOG_FILE(9,"waiting for cmd with cmd_index = 0x%x on chunk->index = %d, path_index = %d", + *cmd_index,chunk->index,path_index); + CBLK_TRACE_LOG_FILE(9,"chunk->fd = %d, poll_fd= %d, afu = %p", + chunk->fd,chunk->path[path_index]->afu->poll_fd,chunk->path[path_index]->afu); #ifndef BLOCK_FILEMODE_ENABLED @@ -2798,7 +3040,7 @@ static inline int CBLK_WAIT_FOR_IO_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_ CFLASH_BLOCK_LOCK(chunk->lock); - CBLK_TRACE_LOG_FILE(8,"poll_ret = 0x%x, chunk->index = %d",poll_ret,chunk->index); + CBLK_TRACE_LOG_FILE(8,"poll_ret = 0x%x, chunk->index = %d, path_index = %d",poll_ret,chunk->index,path_index); @@ -3069,8 +3311,9 @@ static inline int CBLK_WAIT_FOR_IO_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_ if (chunk->path[path_index]->afu->p_hrrq_curr) { - CBLK_TRACE_LOG_FILE(7,"*(chunk->path[path_index]->afu->p_hrrq_curr) = 0x%llx, chunk->path[path_index]->toggle = 0x%llx , chunk->index = %d", - *(chunk->path[path_index]->afu->p_hrrq_curr),chunk->path[path_index]->afu->toggle, + CBLK_TRACE_LOG_FILE(7,"*(chunk->path[%d]->afu->p_hrrq_curr) = 0x%llx, chunk->path[path_index]->toggle = 0x%llx , chunk->index = %d", + path_index, + CBLK_READ_ADDR_CHK_UE(chunk->path[path_index]->afu->p_hrrq_curr),chunk->path[path_index]->afu->toggle, chunk->index); } @@ -3147,11 +3390,20 @@ static inline int CBLK_WAIT_FOR_IO_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_ if (cmdi->cmd_time < timeout) { + /* + * Don't fail yet, until all retries have + * been attempted. Also if the resume command + * logic in the block library fails to re-issue + * the command, then cmdi->status will be non-zero. + * NOTE: We cleared cmdi->status before unlocking + * and calling reset_context. + */ + CBLK_TRACE_LOG_FILE(1,"Timeout for for cmd lba = 0x%llx, cmd = 0x%llx cmd_index = %d, chunk->index = %d", cmdi->lba,(uint64_t)cmd,cmd->index,chunk->index); - cmdi->status = ETIMEDOUT; + cmdi->status = 0; cmdi->transfer_size = 0; @@ -3167,11 +3419,19 @@ static inline int CBLK_WAIT_FOR_IO_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_ CFLASH_BLOCK_LOCK(chunk->lock); - *cmd_complete = TRUE; + if ((cmdi->retry_count >= CAPI_CMD_MAX_RETRIES) || + (cmdi->status)) { - errno = ETIMEDOUT; + CBLK_TRACE_LOG_FILE(1,"Timeout for exceeded retries for cmd lba = 0x%llx, cmd = 0x%llx cmdi->retry_count = %d, cmdi->status = 0x%x", + cmdi->lba,(uint64_t)cmd,cmdi->retry_count,cmdi->status); - rc = -1; + *cmd_complete = TRUE; + + cmdi->status = ETIMEDOUT; + errno = ETIMEDOUT; + + rc = -1; + } } /* @@ -3252,13 +3512,6 @@ static inline int CBLK_WAIT_FOR_IO_COMPLETE_PATH(cflsh_chunk_t *chunk, int path_ */ rc = CBLK_PROCESS_ADAP_CONVERT_INTRPT(chunk,&cmd,CFLSH_BLK_INTRPT_CMD_CMPLT,cmd_complete, transfer_size); -#ifdef _PERF_TEST - CFLASH_BLOCK_UNLOCK(chunk->lock); - usleep(cflsh_blk.adap_poll_delay); - - CFLASH_BLOCK_LOCK(chunk->lock); -#endif /* _PERF_TEST */ - #endif @@ -3337,16 +3590,7 @@ static inline int CBLK_INTR_DELAY_WAIT(cflsh_chunk_t *chunk, int lib_flags, int * If there is a back ground thread for * processing interrupts. */ - - if (loop_cnt > CFLASH_MIN_POLL_RETRIES) { - - /* - * Do not start delaying until we have done - * some checking for completion - */ - - usleep(CFLASH_DELAY_NO_CMD_INTRPT); - } + usleep(CFLASH_DELAY_NO_CMD_INTRPT); if ( (rc) && (loop_cnt > CFLASH_MAX_POLL_RETRIES)) { @@ -3379,16 +3623,7 @@ static inline int CBLK_INTR_DELAY_WAIT(cflsh_chunk_t *chunk, int lib_flags, int * until a command completes or we time-out. */ - if (loop_cnt > CFLASH_MIN_POLL_RETRIES) { - - /* - * Do not start delaying until we have done - * some checking for completion - */ - usleep(CFLASH_DELAY_NO_CMD_INTRPT); - } - } else if ( (rc) && (loop_cnt > CFLASH_MAX_POLL_RETRIES)) { @@ -3511,8 +3746,8 @@ static inline int CBLK_WAIT_FOR_IO_COMPLETE(cflsh_chunk_t *chunk, if ( (cmd->cmdi->in_use == 0) || (cmd->cmdi->state == CFLSH_MGM_ASY_CMP)) { - CBLK_TRACE_LOG_FILE(1,"cmd->cmdi->in_use = 0 flags = 0x%x lba = 0x%llx, chunk->index = %d", - cmd->cmdi->flags,cmd->cmdi->lba,chunk->index); + CBLK_TRACE_LOG_FILE(1,"cmd->cmdi->in_use = 0 flags = 0x%x lba = 0x%llx, chunk->index = %d chunk->flags = %d", + cmd->cmdi->flags,cmd->cmdi->lba,chunk->index,chunk->flags); errno = EINVAL; @@ -3533,7 +3768,7 @@ static inline int CBLK_WAIT_FOR_IO_COMPLETE(cflsh_chunk_t *chunk, poll_fail_retries = 0; while ((!cmd_complete) && - (loop_cnt < CFLASH_MAX_WAIT_LOOP_CNT)) { + (loop_cnt < CFLASH_DELAY_LOOP_CNT)) { rc = CBLK_WAIT_FOR_IO_COMPLETE_PATH(chunk, path_index,cmd,cmd_index, transfer_size,wait, @@ -3593,6 +3828,9 @@ static inline int CBLK_WAIT_FOR_IO_COMPLETE(cflsh_chunk_t *chunk, } else { + CBLK_TRACE_LOG_FILE(9,"chunk->index = %d, num_paths = %d", + chunk->index,chunk->num_paths); + for (path_index=0;path_index < chunk->num_paths;path_index++) { @@ -3609,7 +3847,7 @@ static inline int CBLK_WAIT_FOR_IO_COMPLETE(cflsh_chunk_t *chunk, while ((!cmd_complete) && - (loop_cnt < CFLASH_MAX_WAIT_LOOP_CNT)) { + (loop_cnt < CFLASH_DELAY_LOOP_CNT)) { rc = CBLK_WAIT_FOR_IO_COMPLETE_PATH(chunk, path_index,cmd,cmd_index, transfer_size,wait, &cmd_complete,&poll_retry,&poll_fail_retries); @@ -3671,6 +3909,7 @@ static inline int CBLK_WAIT_FOR_IO_COMPLETE(cflsh_chunk_t *chunk, } if (cmd) { + if (rc==0) {errno=0;} CBLK_TRACE_LOG_FILE(5,"waiting returned for cmd with lba = 0x%llx, with rc = %d, errno = %d in_use = %d, chunk->index = %d", cmd->cmdi->lba, rc, errno,cmd->cmdi->in_use,chunk->index); diff --git a/src/block/cflash_block_int.c b/src/block/cflash_block_int.c index d2bd406a..7022fdc3 100644 --- a/src/block/cflash_block_int.c +++ b/src/block/cflash_block_int.c @@ -32,6 +32,9 @@ #ifdef BLOCK_FILEMODE_ENABLED #include #endif +#ifdef _AIX +#include +#endif char cblk_filename[PATH_MAX]; @@ -368,6 +371,133 @@ int cblk_valid_endianess(void) return rc; } +/* ---------------------------------------------------------------------------- + * + * NAME: cblk_get_host_type + * + * FUNCTION: Determine if this OS is running on Bare Metal, pHyp + * or PowerKVM. + * + * + * + * CALLED BY: + * + * + * INTERNAL PROCEDURES CALLED: + * + * + * + * EXTERNAL PROCEDURES CALLED: + * + * + * + * RETURNS: + * + * ---------------------------------------------------------------------------- + */ + + +void cblk_get_host_type(void) +{ +#ifdef _AIX + + /* + * For AIX always assume pHyp platform type + */ + + cflsh_blk.host_type = CFLASH_HOST_PHYP; + return; + +#else +#define CFLASH_BLOCK_HOST_BUF_SIZE 256 + FILE *fp; + char buf[CFLASH_BLOCK_HOST_BUF_SIZE]; + char *platform_line; + + /* + * Set host type to unknown until we had a chance + * to deterine otherwise. + */ + + cflsh_blk.host_type = CFLASH_HOST_UNKNOWN; + + if ((fp = fopen(CFLASH_BLOCK_HOST_TYPE_FILE,"r")) == NULL) { + + CBLK_TRACE_LOG_FILE(1,"Can not determine host type, failed to open = %s", + CFLASH_BLOCK_HOST_TYPE_FILE); + return; + } + + + + while (fgets(buf,CFLASH_BLOCK_HOST_BUF_SIZE,fp)) { + + + /* + * The /proc/cpuinfo file will have a line + * that starts with the format: + * + * platform : + * + * after the ":" will be the platform/host type + * + * So we first search for platform, then extract out + * platform/host type. + */ + + platform_line = strstr(buf,"platform"); + + if ((platform_line) && + (strstr(platform_line,":")) ) { + + CBLK_TRACE_LOG_FILE(9,"Platform_line = %s",platform_line); + + + /* + * NOTE: The order below is import, because two of the + * platform strings start with pSeries. + */ + + if (strstr(platform_line,"PowerNV")) { + + CBLK_TRACE_LOG_FILE(9,"BML host type"); + cflsh_blk.host_type = CFLASH_HOST_NV; + + break; + } else if (strstr(platform_line,"pSeries (emulated by qemu)")) { + + CBLK_TRACE_LOG_FILE(9,"PowerKVM host type"); + cflsh_blk.host_type = CFLASH_HOST_KVM; + + } else if (strstr(platform_line,"pSeries")) { + + CBLK_TRACE_LOG_FILE(9,"pHyp host type"); + cflsh_blk.host_type = CFLASH_HOST_PHYP; + + } else { + CBLK_TRACE_LOG_FILE(1,"Unknown platform string= %s", + platform_line); + } + + break; + + + } + + } + + if (cflsh_blk.host_type == CFLASH_HOST_UNKNOWN) { + + CBLK_TRACE_LOG_FILE(9,"could not determine host type"); + } + + + fclose (fp); + + return; +#endif +} + /* ---------------------------------------------------------------------------- * * NAME: cblk_chunk_sigsev_handler @@ -416,7 +546,7 @@ void cblk_chunk_sigsev_handler (int signum, siginfo_t *siginfo, void *uctx) for (i=0; i < MAX_NUM_CHUNKS_HASH; i++) { - chunk = cflsh_blk.hash[i]; + chunk = cflsh_blk.hash[i].head; while (chunk) { @@ -455,6 +585,180 @@ void cblk_chunk_sigsev_handler (int signum, siginfo_t *siginfo, void *uctx) return; } +/* + * NAME: cblk_reopen_after_fork + * + * FUNCTION: After a fork, this routine reuses an existing chunk + * for a physical lun and reopens it for use by the child. + * + * NOTES: Assumes caller as the cflsh_blk.global_lock. + * + * INPUTS: + * path - Path of device to open + * + * flags - Flags for open + * + * RETURNS: + * None + * + */ + +void cblk_reopen_after_fork(cflsh_chunk_t *chunk, int flags) +{ + int cleanup_depth; +#ifndef BLOCK_FILEMODE_ENABLED + int mode = 0; + int i; +#endif /* BLOCK_FILEMODE_ENABLED */ + cflsh_cmd_info_t *cmdi; + cflsh_cmd_info_t *next_cmdi; + + errno = 0; + + /* + * Reopen after fork only works for physical luns. + */ + + if (chunk->flags & CFLSH_CHNK_VLUN) { + + + + CBLK_TRACE_LOG_FILE(1,"virtual lun attempted to reopen after fork flags = 0x%x", + chunk->flags); + return; + + } + + CFLASH_BLOCK_LOCK(chunk->lock); + + CBLK_TRACE_LOG_FILE(9,"reopening for %s starting",chunk->dev_name); + /* + * Clear some statistics. We can not clear + * all stats, because the code below will not + * correctly reset some of them. Also the num_paths + * is correct initially until we clean up below and + * then it will get reset to the current number of paths. + */ + + chunk->stats.num_active_threads = 0; + chunk->stats.num_reads = 0; + chunk->stats.num_areads = 0; + chunk->stats.num_writes = 0; + chunk->stats.num_awrites = 0; + + + cmdi = chunk->head_act; + + while (cmdi) { + + next_cmdi = cmdi->act_next; + + + CBLK_FREE_CMDI(chunk,cmdi); + + + cmdi = next_cmdi; + } + +#ifndef BLOCK_FILEMODE_ENABLED + + if (chunk->flags & (CFLSH_CHNK_RD_AC|CFLSH_CHNK_WR_AC)) { + mode = O_RDWR; + + } else if (chunk->flags & CFLSH_CHNK_RD_AC) { + + mode = O_RDONLY; + } else if (chunk->flags & CFLSH_CHNK_WR_AC) { + + mode = O_WRONLY; + } + + /* + * Unmap all mmio space in use + */ + + cblk_chunk_unmap(chunk,FALSE); + + + + CBLK_TRACE_LOG_FILE(9,"reopening successful for %s",chunk->dev_name); + cleanup_depth = 30; + + + for (i=0;i < chunk->num_paths;i++) { + + + if (cblk_chunk_attach_process_map_path(chunk,i,mode, + chunk->path[i]->afu->adap_devno, + chunk->dev_name, + chunk->path[i]->fd,0, + &cleanup_depth,FALSE)) { + + CBLK_TRACE_LOG_FILE(1,"Unable to attach errno = %d",errno); + chunk->path[i]->flags &= ~CFLSH_PATH_ACT; + + chunk->flags |= CFLSH_CHUNK_FAIL_IO; + + return; + } + } + + CBLK_TRACE_LOG_FILE(9,"reopening attach/map successful for %s",chunk->dev_name); + +#endif /* ! BLOCK_FILEMODE_ENABLED */ + + cleanup_depth = 40; + +#ifdef _COMMON_INTRPT_THREAD + + + if (cblk_start_common_intrpt_thread(chunk)) { + + + CBLK_TRACE_LOG_FILE(1,"cblk_start_common_intrpt thread failed with errno= %d", + errno); + + + cblk_chunk_open_cleanup(chunk,cleanup_depth); + + CFLASH_BLOCK_UNLOCK(chunk->lock); + + free(chunk); + + + return; + } + + cleanup_depth = 45; +#endif + +#ifndef BLOCK_FILEMODE_ENABLED + CBLK_TRACE_LOG_FILE(9,"started background interrupt thread for child for %s",chunk->dev_name); + if (cblk_chunk_get_mc_device_resources(chunk,&cleanup_depth)) { + + + CBLK_TRACE_LOG_FILE(5,"cblk_get_device_info failed errno = %d", + errno); + + cblk_chunk_open_cleanup(chunk,cleanup_depth); + + CFLASH_BLOCK_UNLOCK(chunk->lock); + + free(chunk); + + + return; + + } +#endif /* ! BLOCK_FILEMODE_ENABLED */ + + CFLASH_BLOCK_UNLOCK(chunk->lock); + + CBLK_TRACE_LOG_FILE(5,"reopening for %s completed",chunk->dev_name); + + return; +} + /* ---------------------------------------------------------------------------- * @@ -476,6 +780,7 @@ void cblk_chunk_sigsev_handler (int signum, siginfo_t *siginfo, void *uctx) void cblk_prepare_fork (void) { cflsh_chunk_t *chunk = NULL; + cflsh_afu_t *afu = NULL; int i; @@ -486,7 +791,7 @@ void cblk_prepare_fork (void) for (i=0; i < MAX_NUM_CHUNKS_HASH; i++) { - chunk = cflsh_blk.hash[i]; + chunk = cflsh_blk.hash[i].head; while (chunk) { @@ -499,7 +804,16 @@ void cblk_prepare_fork (void) } /* for */ + afu = cflsh_blk.head_afu; + + + while (afu) { + + CFLASH_BLOCK_LOCK(afu->lock); + + afu = afu->next; + } return; } @@ -523,6 +837,7 @@ void cblk_prepare_fork (void) void cblk_parent_post_fork (void) { cflsh_chunk_t *chunk = NULL; + cflsh_afu_t *afu = NULL; int i; int rc; @@ -530,7 +845,7 @@ void cblk_parent_post_fork (void) for (i=0; i < MAX_NUM_CHUNKS_HASH; i++) { - chunk = cflsh_blk.hash[i]; + chunk = cflsh_blk.hash[i].head; while (chunk) { @@ -544,6 +859,17 @@ void cblk_parent_post_fork (void) } /* for */ + afu = cflsh_blk.head_afu; + + + while (afu) { + + + CFLASH_BLOCK_UNLOCK(afu->lock); + + afu = afu->next; + } + rc = pthread_mutex_unlock(&cblk_log_lock); @@ -556,7 +882,7 @@ void cblk_parent_post_fork (void) for (i=0; i < MAX_NUM_CHUNKS_HASH; i++) { - chunk = cflsh_blk.hash[i]; + chunk = cflsh_blk.hash[i].head; if (chunk) { @@ -573,8 +899,6 @@ void cblk_parent_post_fork (void) } } - - CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); return; } @@ -600,13 +924,14 @@ void cblk_parent_post_fork (void) void cblk_child_post_fork (void) { cflsh_chunk_t *chunk = NULL; + cflsh_afu_t *afu = NULL; int i; int rc; for (i=0; i < MAX_NUM_CHUNKS_HASH; i++) { - chunk = cflsh_blk.hash[i]; + chunk = cflsh_blk.hash[i].head; while (chunk) { @@ -620,6 +945,18 @@ void cblk_child_post_fork (void) } /* for */ + afu = cflsh_blk.head_afu; + + + while (afu) { + + + afu->num_issued_cmds = 0; + afu->cmd_room = 0; + CFLASH_BLOCK_UNLOCK(afu->lock); + + afu = afu->next; + } rc = pthread_mutex_unlock(&cblk_log_lock); @@ -633,7 +970,7 @@ void cblk_child_post_fork (void) for (i=0; i < MAX_NUM_CHUNKS_HASH; i++) { - chunk = cflsh_blk.hash[i]; + chunk = cflsh_blk.hash[i].head; if (chunk) { @@ -649,8 +986,67 @@ void cblk_child_post_fork (void) cblk_notify_mc_err(chunk,0,0x206,rc, CFLSH_BLK_NOTIFY_SFW_ERR,NULL); } + } else { + + /* + * Since we forked the child, get its new PID + * and clear its old process name if known. + */ + cflsh_blk.caller_pid = getpid(); + + + if (cflsh_blk.process_name) { + + free(cflsh_blk.process_name); + } + + /* + * Since we forked the child, if we have tracing turned + * on for a trace file per process, then we need to + * open the new file for this child's PID. The routine + * cblk_setup_trace_files will handle the situation + * where multiple chunks are cloned and using the same + * new trace file. + */ + + cblk_setup_trace_files(TRUE); + + CBLK_TRACE_LOG_FILE(9,"child_fork"); + + + for (i=0; i < MAX_NUM_CHUNKS_HASH; i++) { + + chunk = cflsh_blk.hash[i].head; + + + while (chunk) { + + if (!(chunk->flags & CFLSH_CHNK_VLUN)) { + + /* + * Only handle physical luns here. Virtual luns + * are handled separately with an explicit call + * to cblk_clone_after_fork. + * + * For physical luns we will reuse the chunk. + */ + + cblk_reopen_after_fork(chunk,0); + } + + chunk = chunk->next; + + } /* while */ + + + } /* for */ + + + + CBLK_TRACE_LOG_FILE(9,"child_fork"); } + CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); @@ -869,6 +1265,17 @@ cflsh_block_chunk_type_t cblk_get_chunk_type(const char *path, int arch_type) { cflsh_block_chunk_type_t chunk_type; +#ifdef BLOCK_FILEMODE_ENABLED + char *env_arch_type = getenv("CFLSH_BLK_ARCH_TYPE"); + + + if (env_arch_type) { + + chunk_type = atoi(env_arch_type); + + return chunk_type; + } +#endif /* BLOCK_FILEMODE_ENABLED */ if (arch_type) { @@ -925,6 +1332,7 @@ int cblk_set_fcn_ptrs(cflsh_path_t *path) switch (path->type) { case CFLASH_BLK_CHUNK_SIS_LITE: + case CFLASH_BLK_CHUNK_SIS_LITE_SQ: /* * SIS Lite adapter/AFU type @@ -994,21 +1402,38 @@ int cblk_alloc_hrrq_afu(cflsh_afu_t *afu, int num_cmds) } /* - * Align RRQ on cacheline boundary. + * Align start/end RRQ on cacheline boundary. + * + * NOTE: We will pad out the end of the rrq to + * to the next cacheline. This is needed + * in case we encounter a UE on the RRQ. + * The dcbz interface to clear the poison bits + * works only on cachelines. Thus we can not + * allow someone else to potentiallly share the + * cacheline with the last RRQ entry pointers. */ - if ( posix_memalign((void *)&(afu->p_hrrq_start),128, - (sizeof(*(afu->p_hrrq_start)) * num_cmds))) { + afu->size_rrq = sizeof(*(afu->p_hrrq_start)) * num_cmds; + + + if ((afu->size_rrq) % CBLK_CACHE_LINE_SIZE ) { + + + afu->size_rrq = ((afu->size_rrq)/CBLK_CACHE_LINE_SIZE + 1) * CBLK_CACHE_LINE_SIZE; + } + + + + if ( posix_memalign((void *)&(afu->p_hrrq_start),CBLK_CACHE_LINE_SIZE,afu->size_rrq)) { - CBLK_TRACE_LOG_FILE(1,"Failed posix_memalign for rrq errno= %d",errno); + CBLK_TRACE_LOG_FILE(1,"Failed posix_memalign for rrq errno= %d, size_rrq = %d",errno,afu->size_rrq); return -1; } - bzero((void *)afu->p_hrrq_start , - (sizeof(*(afu->p_hrrq_start)) * num_cmds)); + bzero((void *)afu->p_hrrq_start ,afu->size_rrq); afu->p_hrrq_end = afu->p_hrrq_start + (num_cmds - 1); @@ -1075,6 +1500,145 @@ int cblk_free_hrrq_afu(cflsh_afu_t *afu) +/* + * NAME: cblk_alloc_sq_afu + * + * FUNCTION: This routine allocates and initializes + * the SQ (submission queue) per AFU. + * + * NOTE: This routine assumes the caller + * has the cflsh_blk.global_lock. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 - Success + * nonzero - Failure + * + */ +int cblk_alloc_sq_afu(cflsh_afu_t *afu, int num_cmds) +{ + int rc = 0; + + + if (num_cmds == 0) { + + CBLK_TRACE_LOG_FILE(1,"num_cmds = 0"); + + return -1; + + } + + if (afu == NULL) { + + CBLK_TRACE_LOG_FILE(1,"No AFU provided"); + + return -1; + + } + + + if (afu->p_sq_start) { + + //TODO:?? CBLK_TRACE_LOG_FILE(5,"SQ allocated already"); + + // TODO:?? Maybe this should be failed eventually. + return 0; + } + + /* + * Align start SQ on page boundary and end of SQ on + * cache line boundary. + * + * NOTE: We will pad out the end of the sq to + * to the next cacheline. This is being + * done to prevent the end of the SQ from being + * shared by some host memory that could be written + * from the AFU. For that case then that cache + * line could potentially encounter a UE. Thus + * we can not share the cache line of the last + * SQ entry. + */ + + afu->size_sq = sizeof(*(afu->p_sq_start)) * (num_cmds + 1); + + + if ((afu->size_sq) % CBLK_CACHE_LINE_SIZE ) { + + + afu->size_sq = ((afu->size_sq)/CBLK_CACHE_LINE_SIZE + 1) * CBLK_CACHE_LINE_SIZE; + } + + + + if ( posix_memalign((void *)&(afu->p_sq_start),4096,afu->size_sq)) { + + + CBLK_TRACE_LOG_FILE(1,"Failed posix_memalign for rrq errno= %d, size_rrq = %d",errno,afu->size_sq); + + return -1; + + } + + bzero((void *)afu->p_sq_start ,afu->size_sq); + + + afu->p_sq_end = afu->p_sq_start + (num_cmds); + + afu->p_sq_curr = afu->p_sq_start; + + + + return rc; +} + + +/* + * NAME: cblk_free_sq_afu + * + * FUNCTION: This routine frees the + * SQ (submission queu) per AFU. + * + * NOTE: This routine assumes the caller + * has the cflsh_blk.global_lock. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 - Success + * nonzero - Failure + * + */ +int cblk_free_sq_afu(cflsh_afu_t *afu) +{ + + + if (afu == NULL) { + + return 0; + + } + + + if (afu->p_sq_start == NULL) { + + return 0; + } + + + free(afu->p_sq_start); + + afu->p_sq_start = NULL; + afu->p_sq_end = NULL; + + return 0; +} + + /* * NAME: cblk_get_afu * @@ -1290,6 +1854,28 @@ cflsh_afu_t *cblk_get_afu(cflsh_path_t *path, char *dev_name, dev64_t adap_devno return NULL; } + if (type == CFLASH_BLK_CHUNK_SIS_LITE_SQ) { + + /* + * This AFU is using a Submission Queue (SQ) + * to issue IOARCBs to it. Let's allocate + * the SQ now. + */ + + + if (cblk_alloc_sq_afu(afu,num_cmds)) { + + + cblk_free_hrrq_afu(afu); + free(afu); + + return NULL; + } + + afu->flags |= CFLSH_AFU_SQ; + + afu->hrrq_to_ioarcb_off = sizeof(sisl_ioarcb_t); + } pthread_rc = pthread_cond_init(&(afu->resume_event),NULL); @@ -1317,6 +1903,7 @@ cflsh_afu_t *cblk_get_afu(cflsh_path_t *path, char *dev_name, dev64_t adap_devno afu->eyec = CFLSH_EYEC_AFU; + path->afu_share_type = *in_use; CBLK_Q_NODE_TAIL(cflsh_blk.head_afu,cflsh_blk.tail_afu,afu,prev,next); } @@ -1379,6 +1966,34 @@ int cblk_update_afu_type(cflsh_afu_t *afu,cflsh_block_chunk_type_t type) } afu->type = type; + + if ((type == CFLASH_BLK_CHUNK_SIS_LITE_SQ) && + !(afu->flags & CFLSH_AFU_SQ)) { + + + /* + * This AFU is using a Submission Queue (SQ) + * to issue IOARCBs to it and we have not + * yet allocated the SWQ. So let's do that + * now. + */ + + + if (cblk_alloc_sq_afu(afu,afu->num_rrqs)) { + + + cblk_free_hrrq_afu(afu); + errno = ENOMEM; + + return -1; + } + + afu->flags |= CFLSH_AFU_SQ; + + afu->hrrq_to_ioarcb_off = sizeof(sisl_ioarcb_t); + + + } } return rc; @@ -1421,6 +2036,7 @@ void cblk_release_afu(cflsh_path_t *path,cflsh_afu_t *afu) } cblk_free_hrrq_afu(afu); + cblk_free_sq_afu(afu); free(afu); } @@ -1451,7 +2067,7 @@ void cblk_release_afu(cflsh_path_t *path,cflsh_afu_t *afu) * */ -cflsh_path_t *cblk_get_path(cflsh_chunk_t *chunk, dev64_t adap_devno,cflsh_block_chunk_type_t type,int num_cmds, +cflsh_path_t *cblk_get_path(cflsh_chunk_t *chunk, dev64_t adap_devno,char *path_name,cflsh_block_chunk_type_t type,int num_cmds, cflsh_afu_in_use_t *in_use, int share) { cflsh_path_t *path = NULL; @@ -1519,6 +2135,11 @@ cflsh_path_t *cblk_get_path(cflsh_chunk_t *chunk, dev64_t adap_devno,cflsh_block } +#ifndef _AIX + if (path_name) { + strcpy(path->dev_name,path_name); + } +#endif /* _AIX */ path->eyec = CFLSH_EYEC_PATH; @@ -1581,6 +2202,11 @@ void cblk_release_path(cflsh_chunk_t *chunk, cflsh_path_t *path) cblk_release_afu(path,path->afu); path->afu = NULL; + +#ifndef _AIX + bzero(path->dev_name,PATH_MAX); +#endif /* !_AIX */ + path->eyec = 0; free(path); chunk->num_paths--; @@ -1613,6 +2239,7 @@ void cblk_release_path(cflsh_chunk_t *chunk, cflsh_path_t *path) int cblk_update_path_type(cflsh_chunk_t *chunk, cflsh_path_t *path, cflsh_block_chunk_type_t type) { int rc = 0; + int sq_flags; if (path == NULL) { @@ -1623,8 +2250,26 @@ int cblk_update_path_type(cflsh_chunk_t *chunk, cflsh_path_t *path, cflsh_block_ return -1; } + sq_flags = path->afu->flags & CFLSH_AFU_SQ; + rc = cblk_update_afu_type(path->afu,type); + if (!sq_flags && + (path->afu->mmio) && + (path->afu->flags & CFLSH_AFU_SQ)) { + /* + * If we changed from not having an SQ for this AFU + * to now having one, then we need to register + * the SQ with the AFU. This can happen + * in situations where we determine the + * AFU type after we have already done + * the initial setup. + */ + + rc = CBLK_ADAP_SQ_SETUP(chunk,path->path_index); + + } + return rc; } @@ -1653,7 +2298,7 @@ chunk_id_t cblk_get_chunk(int flags,int max_num_cmds) { chunk_id_t ret_chunk_id = NULL_CHUNK_ID; cflsh_chunk_t *chunk = NULL; - cflsh_chunk_t *tmp_chunk; + cflsh_chunk_hash_t *p_hash = NULL; int j; @@ -1807,62 +2452,20 @@ chunk_id_t cblk_get_chunk(int flags,int max_num_cmds) chunk->eyec = CFLSH_EYEC_CHUNK; + p_hash = &cflsh_blk.hash[chunk->index & CHUNK_HASH_MASK]; - if (cflsh_blk.hash[chunk->index & CHUNK_HASH_MASK] == NULL) { - - cflsh_blk.hash[chunk->index & CHUNK_HASH_MASK] = chunk; - } else { - - tmp_chunk = cflsh_blk.hash[chunk->index & CHUNK_HASH_MASK]; - - while (tmp_chunk) { - - if ((ulong)tmp_chunk & CHUNK_BAD_ADDR_MASK ) { - - /* - * Chunk addresses are allocated - * on certain alignment. If this - * potential chunk address does not - * have the correct alignment then fail - * this request. - */ - - cflsh_blk.num_bad_chunk_ids++; - - CBLK_TRACE_LOG_FILE(1,"Corrupted chunk address = 0x%p, hash[] = 0x%p index = 0x%x", - tmp_chunk, cflsh_blk.hash[chunk->index & CHUNK_HASH_MASK], - (chunk->index & CHUNK_HASH_MASK)); - - CBLK_LIVE_DUMP_THRESHOLD(5,"0x200"); - - free(chunk->cmd_info); - free(chunk->cmd_start); - - chunk->eyec = 0; - - free(chunk); - - errno = EFAULT; - return NULL_CHUNK_ID; - } - - - if (tmp_chunk->next == NULL) { - - tmp_chunk->next = chunk; - - chunk->prev = tmp_chunk; - break; - } - - tmp_chunk = tmp_chunk->next; - - } /* while */ + CFLASH_BLOCK_WR_RWLOCK(p_hash->lock); + if (p_hash->tail) + { + chunk->prev = p_hash->tail; + p_hash->tail->next = chunk; } - - } + if (!p_hash->head) {p_hash->head = chunk;} + p_hash->tail = chunk; + CFLASH_BLOCK_RWUNLOCK(p_hash->lock); + } if (ret_chunk_id == NULL_CHUNK_ID) { @@ -1907,11 +2510,26 @@ cflash_cmd_err_t cblk_process_cmd(cflsh_chunk_t *chunk,int path_index, cflsh_cmd int rc2 = 0; size_t transfer_size = 0; int pthread_rc; + int i; + if (CBLK_INVALID_CMD_CMDI(chunk,cmd,cmd->cmdi,__FUNCTION__)) { - CBLK_TRACE_LOG_FILE(1,"cmd/cmdi is valid"); + CBLK_TRACE_LOG_FILE(1,"cmd/cmdi is invalid for chunk = %s, path_index = 0x%x chunk->flags = 0x%x", + chunk->dev_name,path_index, chunk->flags); + + for (i = 0; i< chunk->num_cmds;i++) { + + if (cmd == &chunk->cmd_start[i]) { + + + CBLK_TRACE_LOG_FILE(1,"cmd/cmdi (cont) cmd found with index = %d, cmd->index = %d, cmdi = %p", + i,cmd->index,&(chunk->cmd_info[i])); + + break; + } + } errno = EINVAL; rc = CFLASH_CMD_FATAL_ERR; @@ -2012,6 +2630,7 @@ cflash_cmd_err_t cblk_process_cmd(cflsh_chunk_t *chunk,int path_index, cflsh_cmd cmd->cmdi->retry_count++; + cmd->cmdi->cmd_time = time(NULL); /* * Since the caller used CFLASH_BLOCK_AFU_SHARE_LOCK, @@ -2238,7 +2857,19 @@ cflash_cmd_err_t cblk_process_cmd(cflsh_chunk_t *chunk,int path_index, cflsh_cmd if (CBLK_INVALID_CMD_CMDI(chunk,cmd,cmd->cmdi,__FUNCTION__)) { - CBLK_TRACE_LOG_FILE(1,"cmd/cmdi is valid"); + CBLK_TRACE_LOG_FILE(1,"cmd/cmdi is invalid"); + + for (i = 0; i< chunk->num_cmds;i++) { + + if (cmd == &chunk->cmd_start[i]) { + + + CBLK_TRACE_LOG_FILE(1,"cmd/cmdi (cont) cmd found with index = %d, cmd->index = %d, cmdi = %p", + i,cmd->index,&(chunk->cmd_info[i])); + + break; + } + } return rc; } @@ -2333,7 +2964,7 @@ int cblk_find_free_cmd(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t **cmd,int flags) */ - while ((!found) && (loop_cnt < CFLASH_BLOCK_MAX_CMD_WAIT_RETRIES)) { + while ((!found) && (loop_cnt++ < CFLASH_BLOCK_MAX_CMD_WAIT_RETRIES)) { CFLASH_BLOCK_UNLOCK(chunk->lock); @@ -2353,8 +2984,9 @@ int cblk_find_free_cmd(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t **cmd,int flags) return -1; } - CBLK_TRACE_LOG_FILE(1,"No free command found num_active_cmds = %d, num_in_use = %d", - chunk->num_active_cmds,num_in_use); + CBLK_TRACE_LOG_FILE(1,"No free command found num_cmds = %d " + "num_active_cmds = %d, num_in_use = %d flags:%x", + chunk->num_cmds, chunk->num_active_cmds, num_in_use, flags); cmdi = chunk->head_free; @@ -2411,14 +3043,14 @@ int cblk_find_free_cmd(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t **cmd,int flags) } if (chunk->cmd_start == NULL) { - CBLK_TRACE_LOG_FILE(1,"cmd_start is NULL"); + CBLK_TRACE_LOG_FILE(1,"cmd_start is NULL, chunk->flags =0x%x",chunk->flags); errno = EINVAL; return -1; } if (cmdi == NULL) { - CBLK_TRACE_LOG_FILE(1,"cmdi is NULL"); + CBLK_TRACE_LOG_FILE(1,"cmdi is NULL, chunk->flags =0x%x",chunk->flags); errno = EINVAL; return -1; } @@ -2428,8 +3060,8 @@ int cblk_find_free_cmd(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t **cmd,int flags) - CBLK_TRACE_LOG_FILE(1,"cmdi = %p with index %d is invalid, chunk->num_cmds = %d", - cmdi,cmdi->index, chunk->num_cmds); + CBLK_TRACE_LOG_FILE(1,"cmdi = %p with index %d is invalid, chunk->num_cmds = %d, chunk->flags =0x%x", + cmdi,cmdi->index, chunk->num_cmds,chunk->flags); errno = EINVAL; return -1; @@ -2437,8 +3069,8 @@ int cblk_find_free_cmd(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t **cmd,int flags) if (CFLSH_EYECATCH_CMDI(cmdi)) { - CBLK_TRACE_LOG_FILE(1,"Invalid eyecatcher cmdi = %p is invalid, chunk->num_cmds = %d", - cmdi,cmdi->index, chunk->num_cmds); + CBLK_TRACE_LOG_FILE(1,"Invalid eyecatcher cmdi = %p is invalid, chunk->num_cmds = %d, chunk->flags =0x%x", + cmdi,cmdi->index, chunk->num_cmds,chunk->flags); errno = EINVAL; return -1; @@ -2446,12 +3078,12 @@ int cblk_find_free_cmd(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t **cmd,int flags) if (cmdi->index > chunk->num_cmds) { - CBLK_TRACE_LOG_FILE(1,"cmdi = %p index is too large = %d, chunk->num_cmds = %d", + CBLK_TRACE_LOG_FILE(1,"cmdi = %p index is too large = %d, chunk->num_cmds = %d, chunk->flags =0x%x", cmdi,cmdi->index, chunk->num_cmds); CBLK_TRACE_LOG_FILE(1,"user_tag = 0x%x, *user_status = %p, path_index = %d", - cmdi->user_tag,cmdi->user_status,cmdi->path_index); + cmdi->user_tag,cmdi->user_status,cmdi->path_index,chunk->flags); errno = EINVAL; return -1; } @@ -2469,8 +3101,8 @@ int cblk_find_free_cmd(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t **cmd,int flags) - CBLK_TRACE_LOG_FILE(1,"cmd = %p with index = %d, is invalid, chunk->num_cmds = %d", - *cmd,cmdi->index, chunk->num_cmds); + CBLK_TRACE_LOG_FILE(1,"cmd = %p with index = %d, is invalid, chunk->num_cmds = %d, chunk->flags =0x%x", + *cmd,cmdi->index, chunk->num_cmds,chunk->flags); errno = EINVAL; return -1; @@ -2482,8 +3114,8 @@ int cblk_find_free_cmd(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t **cmd,int flags) if (pthread_rc) { - CBLK_TRACE_LOG_FILE(1,"pthread_cond_init failed rc = %d errno= %d", - pthread_rc,errno); + CBLK_TRACE_LOG_FILE(1,"pthread_cond_init failed rc = %d errno= %d, cmdi = %p", + pthread_rc,errno,cmdi); rc = -1; return rc; @@ -3354,6 +3986,7 @@ void cblk_open_cleanup_wait_thread(cflsh_chunk_t *chunk) void cblk_chunk_open_cleanup(cflsh_chunk_t *chunk, int cleanup_depth) { + cflsh_chunk_hash_t *p_hash = NULL; int i; CBLK_TRACE_LOG_FILE(5,"cleanup = %d",cleanup_depth); @@ -3453,6 +4086,10 @@ void cblk_chunk_open_cleanup(cflsh_chunk_t *chunk, int cleanup_depth) } } + if (chunk->udid) { + free(chunk->udid); + } + chunk->num_blocks = 0; chunk->flags = 0; chunk->in_use = FALSE; @@ -3485,17 +4122,27 @@ void cblk_chunk_open_cleanup(cflsh_chunk_t *chunk, int cleanup_depth) } - if (chunk->prev) { - chunk->prev->next = chunk->next; + p_hash = &cflsh_blk.hash[chunk->index & CHUNK_HASH_MASK]; - } else { + CFLASH_BLOCK_WR_RWLOCK(p_hash->lock); - cflsh_blk.hash[chunk->index & CHUNK_HASH_MASK] = chunk->next; + if (p_hash->head == chunk) + { + p_hash->head = chunk->next; + if (chunk->next) {chunk->next->prev = NULL;} + if (p_hash->tail == chunk) {p_hash->tail = NULL;} } - - if (chunk->next) { + else if (p_hash->tail == chunk) + { + chunk->prev->next = NULL; + p_hash->tail = chunk->prev; + } + else + { + chunk->prev->next = chunk->next; chunk->next->prev = chunk->prev; } + CFLASH_BLOCK_RWUNLOCK(p_hash->lock); } @@ -4755,6 +5402,7 @@ void cblk_resume_all_halted_cmds(cflsh_chunk_t *chunk, int increment_retries, void cblk_reset_context_shared_afu(cflsh_afu_t *afu) { cflsh_chunk_t *chunk = NULL; + cflash_block_check_os_status_t status; cflsh_path_t *path; int reset_context_success = TRUE; int detach = FALSE; @@ -4772,6 +5420,16 @@ void cblk_reset_context_shared_afu(cflsh_afu_t *afu) timeout = time(NULL) - 1; CFLASH_BLOCK_LOCK(afu->lock); + + if (afu->flags & (CFLSH_AFU_HALTED|CFLSH_AFU_RECOV)) { + + CBLK_TRACE_LOG_FILE(5,"skipping context reset, since AFU is halted or recovering is pending"); + + CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); + CFLASH_BLOCK_UNLOCK(afu->lock); + + return; + } if (afu->reset_time > timeout) { @@ -4815,7 +5473,8 @@ void cblk_reset_context_shared_afu(cflsh_afu_t *afu) /* * We found at least one chunk associated with this afu. - * Thus we'll reset the context and then all commands + * Thus we'll reset the context by driving it from this + * chunk. After this all commands * need to be failed/resumed. */ @@ -4826,10 +5485,46 @@ void cblk_reset_context_shared_afu(cflsh_afu_t *afu) /* * Reset context failed */ - CBLK_TRACE_LOG_FILE(1,"reset context failed for path_index of %d",path->path_index); reset_context_success = FALSE; - // TODO:?? should we do unmap detach here instead of below. + /* + * Check for adapter reset + */ + + CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); + + CFLASH_BLOCK_LOCK(chunk->lock); + + status = cblk_check_os_adap_err(chunk,path_index); + + CFLASH_BLOCK_UNLOCK(chunk->lock); + + CBLK_TRACE_LOG_FILE(1,"reset context failed for path_index of %d, status = %d",path->path_index,status); + + if (status != CFLSH_BLK_CHK_OS_NO_RESET) { + + /* + * If some type of reset was done, then + * exit here. If a reset was done then + * this supersedes context reset. If reset + * failed then it should have also done the + * clean up and failing of all I/O for this + * chunk. + */ + + CBLK_TRACE_LOG_FILE(9,"AFU reset was done assuming context reset is unnecessary"); + + return; + + + } + + /* + * Drop thru and clean up + */ + + CFLASH_BLOCK_WR_RWLOCK(cflsh_blk.global_lock); + } /* @@ -5018,7 +5713,7 @@ int cblk_retry_new_path(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t *cmd, int delay_ne /* * If we have more than one path, then - * allow retry down that path. + * attempt retry down alternate path. */ @@ -5043,7 +5738,21 @@ int cblk_retry_new_path(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t *cmd, int delay_ne chunk->cmd_info[cmd->index].path_index = i; CBLK_TRACE_LOG_FILE(9,"Retry path_index = %d",i); - rc = CFLASH_CMD_RETRY_ERR; + + + if ((chunk->path[cur_path]->afu == chunk->path[i]->afu) && + (delay_needed_same_afu)) { + + /* + * If the new path has the same AFU + * and the caller requested delay on + * on the same AFU, then do so. + */ + + rc = CFLASH_CMD_DLY_RETRY_ERR; + } else { + rc = CFLASH_CMD_RETRY_ERR; + } chunk->stats.num_path_fail_overs++; @@ -5054,10 +5763,49 @@ int cblk_retry_new_path(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t *cmd, int delay_ne } + if (chunk->cmd_info[cmd->index].path_index == cur_path) { + + /* + * If no new paths are found, then + * give retry any way on same path. + */ + + if (delay_needed_same_afu) { + + /* + * If no alternate path was found, + * then only do retry down current path + * if delay is needed for same AFU. + */ + + rc = CFLASH_CMD_DLY_RETRY_ERR; + } else { + rc = CFLASH_CMD_RETRY_ERR; + + + } + + } + #endif /* AIX */ - } + } else { + + + /* + * This routine is only called for situations where a + * retry might be valid. Thus at the very minimum do + * the retry, but for the same path. If the caller + * indicated a delay also, then do a delay retry. + */ + + if (delay_needed_same_afu) { + rc = CFLASH_CMD_DLY_RETRY_ERR; + } else { + rc = CFLASH_CMD_RETRY_ERR; + } + } if ((rc == CFLASH_CMD_DLY_RETRY_ERR) || @@ -5076,17 +5824,7 @@ int cblk_retry_new_path(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t *cmd, int delay_ne - } else { - - - /* - * This routine is only called for situations where a - * retry might be valid. Thus at the very minimum do - * the retry, but for the same path. - */ - - rc = CFLASH_CMD_RETRY_ERR; - } + } #else @@ -5214,15 +5952,26 @@ void cblk_filemode_io(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t *cmd) * from cmd into p_hrrq_curr. So * we use a two step process. */ - tmp_val = (uint32_t)cmd; + + if (chunk->path[chunk->cur_path]->afu->flags & CFLSH_AFU_SQ) { + tmp_val = (uint32_t)&(cmd->sisl_cmd.sa); + } else { + tmp_val = (uint32_t)cmd; + } *(chunk->path[chunk->cur_path]->afu->p_hrrq_curr) = tmp_val | chunk->path[chunk->cur_path]->afu->toggle; #else - *(chunk->path[chunk->cur_path]->afu->p_hrrq_curr) = (uint64_t) cmd | chunk->path[chunk->cur_path]->afu->toggle; + + if (chunk->path[chunk->cur_path]->afu->flags & CFLSH_AFU_SQ) { + *(chunk->path[chunk->cur_path]->afu->p_hrrq_curr) = (uint64_t)&(cmd->sisl_cmd.sa)| chunk->path[chunk->cur_path]->afu->toggle; + } else { + + *(chunk->path[chunk->cur_path]->afu->p_hrrq_curr) = (uint64_t) cmd | chunk->path[chunk->cur_path]->afu->toggle; + } #endif CBLK_TRACE_LOG_FILE(7,"*(chunk->path[chunk->cur_path].p_hrrq_curr) = 0x%llx, chunk->path[chunk->cur_path].toggle = 0x%llx, chunk->index = %d", - *(chunk->path[chunk->cur_path]->afu->p_hrrq_curr),chunk->path[chunk->cur_path]->afu->toggle, + CBLK_READ_ADDR_CHK_UE(chunk->path[chunk->cur_path]->afu->p_hrrq_curr),chunk->path[chunk->cur_path]->afu->toggle, chunk->index); } @@ -5571,8 +6320,8 @@ void *cblk_intrpt_thread(void *data) } - CBLK_TRACE_LOG_FILE(9,"chunk index = %d thread_flags = %d num_active_cmds = 0x%x", - chunk->index,chunk->thread_flags,chunk->num_active_cmds); + CBLK_TRACE_LOG_FILE(9,"chunk index = %d thread_flags = %d num_active_cmds = 0x%x, num_paths = %d", + chunk->index,chunk->thread_flags,chunk->num_active_cmds,chunk->num_paths); if (chunk->thread_flags & CFLSH_CHNK_EXIT_INTRPT) { @@ -5631,11 +6380,11 @@ void *cblk_intrpt_thread(void *data) if ((chunk->num_active_cmds) && (chunk->head_act) && - !(chunk->flags & CFLSH_CHNK_HALTED) ) { + !(chunk->flags & (CFLSH_CHNK_HALTED|CFLSH_CHNK_RECOV_AFU))) { /* * We need to check for dropped commands here if - * we are not in a halted state. + * we are not in a halted nor AFU recovery state. * For common threads there is no effective mechanism in * CBLK_WAIT_FOR_IO_COMPLETE to detect commmands that time-out. * So we will do that here. First find the oldest command, @@ -5784,6 +6533,19 @@ void *cblk_intrpt_thread(void *data) CBLK_GET_INTRPT_STATUS(chunk,i); + + if (chunk->flags & CFLSH_CHNK_RECOV_AFU) { + + /* + * We could detect UE when reading interrupt status. + * So give up on this recovery, if AFU recovery + * is in progress. + */ + + CBLK_TRACE_LOG_FILE(5,"AFU Recovery in progress: abandon context reset chunk->index = %d, chunk->flags 0x%x", + chunk->index, chunk->flags); + break; + } CFLASH_BLOCK_UNLOCK(chunk->lock); cblk_reset_context_shared_afu(chunk->path[i]->afu); @@ -6131,6 +6893,12 @@ void cblk_trace_log_data_ext(trace_log_ext_arg_t *ext_arg, FILE *logfp,char *fil va_list ap; struct timeb cur_time, log_time, delta_time; uint print_log_number; +#ifndef TIMELEN +#define TIMELEN 26 /* Linux does have a define for the minium size of the a timebuf */ + /* However linux man pages say it is 26 */ +#endif + char timebuf[TIMELEN+1]; + time_t curtime; if (ext_arg == NULL) { @@ -6182,6 +6950,9 @@ void cblk_trace_log_data_ext(trace_log_ext_arg_t *ext_arg, FILE *logfp,char *fil */ fprintf(logfp,"---------------------------------------------------------------------------\n"); fprintf(logfp,"Date for %s is %s at %s\n",__FILE__,__DATE__,__TIME__); + + curtime = time(NULL); + fprintf(logfp,"Trace started at %s\n",ctime_r(&curtime,timebuf)); fprintf(logfp,"Index Sec msec delta dmsec Filename function, line ...\n"); fprintf(logfp,"------- ----- ----- ----- ----- -------------------- ---------------------\n"); @@ -6262,12 +7033,16 @@ void cblk_display_stats(cflsh_chunk_t *chunk, int verbosity) #endif CBLK_TRACE_LOG_FILE(verbosity,"flags 0x%x",cflsh_blk.flags); + CBLK_TRACE_LOG_FILE(verbosity,"host_type 0x%x",cflsh_blk.host_type); CBLK_TRACE_LOG_FILE(verbosity,"lun_id 0x%llx",cflsh_blk.lun_id); CBLK_TRACE_LOG_FILE(verbosity,"next_chunk_id 0x%llx",cflsh_blk.next_chunk_id); CBLK_TRACE_LOG_FILE(verbosity,"num_active_chunks 0x%x",cflsh_blk.num_active_chunks); CBLK_TRACE_LOG_FILE(verbosity,"num_max_active_chunks 0x%x",cflsh_blk.num_max_active_chunks); CBLK_TRACE_LOG_FILE(verbosity,"num_bad_chunk_ids 0x%x",cflsh_blk.num_bad_chunk_ids); CBLK_TRACE_LOG_FILE(verbosity,"chunk_id 0x%llx",chunk->index); + if(chunk->udid) { + CBLK_TRACE_LOG_FILE(verbosity,"udid %s",chunk->udid); + } CBLK_TRACE_LOG_FILE(verbosity,"chunk_block size 0x%x",chunk->stats.block_size); CBLK_TRACE_LOG_FILE(verbosity,"num_paths 0x%x",chunk->num_paths); CBLK_TRACE_LOG_FILE(verbosity,"primary_path_id 0x%x",chunk->path[0]->path_id); @@ -6326,6 +7101,9 @@ void cblk_display_stats(cflsh_chunk_t *chunk, int verbosity) CBLK_TRACE_LOG_FILE(verbosity,"num_fail_detach_threads 0x%llx",chunk->stats.num_fail_detach_threads); CBLK_TRACE_LOG_FILE(verbosity,"num_active_threads 0x%llx",chunk->stats.num_active_threads); CBLK_TRACE_LOG_FILE(verbosity,"max_num_act_threads 0x%llx",chunk->stats.max_num_act_threads); + CBLK_TRACE_LOG_FILE(verbosity,"Avg latency %ld",(chunk->rlat+chunk->wlat)/(chunk->rcmd+chunk->wcmd)); + CBLK_TRACE_LOG_FILE(verbosity,"Read latency %ld",chunk->rlat/chunk->rcmd); + CBLK_TRACE_LOG_FILE(verbosity,"Write latency %ld",chunk->wlat/chunk->wcmd); return; } @@ -6544,7 +7322,7 @@ void cblk_dump_debug_data(const char *reason,const char *reason_filename,const for (i=0;ipath[j]->path_id); fprintf(cblk_dumpfp," path_id_mask = 0x%x\n",chunk->path[j]->path_id_mask); fprintf(cblk_dumpfp," num_ports = 0x%x\n",chunk->path[j]->num_ports); +#ifndef _AIX + fprintf(cblk_dumpfp," dev_name = %s\n",chunk->path[j]->dev_name); +#endif /* !_AIX */ + fprintf(cblk_dumpfp," fd = %d\n",chunk->path[j]->fd); fprintf(cblk_dumpfp," chunk = %p",chunk->path[j]->chunk); if (chunk->path[j]->chunk == chunk) { fprintf(cblk_dumpfp," (Valid)\n"); @@ -6887,6 +7669,11 @@ void cblk_dump_debug_data(const char *reason,const char *reason_filename,const fprintf(cblk_dumpfp," hrrq_curr = %p\n",afu->p_hrrq_curr); fprintf(cblk_dumpfp," hrrq_end = %p\n",afu->p_hrrq_end); fprintf(cblk_dumpfp," num_rrqs = 0x%x\n",afu->num_rrqs); + fprintf(cblk_dumpfp," size_rrq = 0x%x\n",afu->size_rrq); + fprintf(cblk_dumpfp," sq_start = %p\n",afu->p_sq_start); + fprintf(cblk_dumpfp," sq_curr = %p\n",afu->p_sq_curr); + fprintf(cblk_dumpfp," sq_end = %p\n",afu->p_sq_end); + fprintf(cblk_dumpfp," size_sq = 0x%x\n",afu->size_sq); fprintf(cblk_dumpfp," num_issued_cmds = 0x%x\n",afu->num_issued_cmds); fprintf(cblk_dumpfp," cmd_room = 0x%"PRIx64"\n",afu->cmd_room); @@ -7072,7 +7859,6 @@ int cblk_setup_sigusr1_dump(void) */ void cblk_dcbz_buffer(void *buf, size_t len) { -#define CBLK_DCBZ_CACHE_LINE_SZ 128 ulong start_addr; ulong end_addr; @@ -7107,16 +7893,18 @@ void cblk_dcbz_buffer(void *buf, size_t len) */ while (addr < end_addr) { +#ifndef TARGET_ARCH_x86_64 asm volatile ("dcbz 0, %0" : : "r" (addr) : "memory"); - +#endif addr += CBLK_DCBZ_CACHE_LINE_SZ; } /* issue sync for cache lines */ +#ifndef TARGET_ARCH_x86_64 asm volatile ( "lwsync" : : ); - +#endif return; } @@ -7155,6 +7943,147 @@ void cblk_clear_poison_bits(void *buf, size_t len) } +/* + * NAME: cblk_query_ue + * + * FUNCTION: Checks if UE has occurred in + * the specified buffer. + * + * RETURNS: + * + * 1 if UE found. O otherwise + * + */ + +int cblk_query_ue(void *buf, size_t len) +{ + int ue_found = FALSE; + +#if defined(_CBLK_UE_SAFE) && defined(_AIX) + + if (ue_query(buf,len)) { + + ue_found = TRUE; + CBLK_TRACE_LOG_FILE(9,"buf = %p ue_found = %d",buf,ue_found); + } + +#endif + + return ue_found; + +} + +/* + * NAME: cblk_check_cmd_data_buf_for_ue + * + * FUNCTION: Checks if UE has occurred in + * the specified data buffer of a command. + * If so then that command is failed with + * ENOTRECOVERABLE. + * + * RETURNS: + * + * None + * + */ + +void cblk_check_cmd_data_buf_for_ue(cflsh_chunk_t *chunk,cflsh_cmd_mgm_t *cmd) +{ + + cflsh_cmd_info_t *cmdi; +#ifdef _COMMON_INTRPT_THREAD + int pthread_rc; +#endif /* _COMMON_INTRPT_THREAD */ + + + if (chunk == NULL) { + + return; + } + + if (cmd == NULL) { + + return; + } + + cmdi = cmd->cmdi; + + if (cmdi == NULL) { + + return; + } + + + if (cblk_query_ue(cmdi->buf,(cmdi->nblocks*CAPI_FLASH_BLOCK_SIZE))) { + + + /* + * If this request's data buffer has a UE, then + * that request needs to be failed immediately + * with an errno of ENOTRECOVERABLE, since we + * can not clear the data buffer because it is owned + * by another command. The caller maybe able to retry + * and recover if they can get the data from some other + * place then this buffer. + */ + + + cmdi->status = ENOTRECOVERABLE; + cmdi->transfer_size = 0; + + + cblk_notify_mc_err(chunk,cmdi->path_index,0x20e,0, + CFLSH_BLK_NOTIFY_AFU_ERROR,cmd); + + CBLK_TRACE_LOG_FILE(1,"Command with UE lba = 0x%llx flags = 0x%x, chunk->index = %d", + cmd->cmdi->lba,cmd->cmdi->flags,chunk->index); + + + /* + * Fail command back. + */ + + cmdi->state = CFLSH_MGM_CMP; +#ifdef _COMMON_INTRPT_THREAD + + if (!(chunk->flags & CFLSH_CHNK_NO_BG_TD)) { + + /* + * If we are using a common interrupt thread + */ + + pthread_rc = pthread_cond_signal(&(cmdi->thread_event)); + + if (pthread_rc) { + + CBLK_TRACE_LOG_FILE(5,"pthread_cond_signal failed rc = %d,errno = %d, chunk->index = %d", + pthread_rc,errno,chunk->index); + } + + + /* + * Signal any one waiting for any command to complete. + */ + + pthread_rc = pthread_cond_signal(&(chunk->cmd_cmplt_event)); + + if (pthread_rc) { + + CBLK_TRACE_LOG_FILE(5,"pthread_cond_signal failed for cmd_cmplt_event rc = %d,errno = %d, chunk->index = %d", + pthread_rc,errno,chunk->index); + } + } + +#endif /* _COMMON_INTRPT_THREAD */ + + } + + return; + +} + + + /* * NAME: cblk_clear_poison_bits_chunk * @@ -7198,14 +8127,35 @@ void cblk_clear_poison_bits(void *buf, size_t len) void cblk_clear_poison_bits_chunk(cflsh_chunk_t *chunk, int path_index, int all_paths) { - int i; + int i,j; scsi_cdb_t *cdb = NULL; cflsh_cmd_mgm_t *cmd = NULL; - cflsh_cmd_info_t *cmdi; + cflsh_cmd_info_t *cmdi, *cmdi2; int local_flags = 0; uint8_t op_code = 0; + ulong cmdi_start_addr; + ulong cmdi_end_addr; + ulong cmdi2_start_addr; + ulong cmdi2_end_addr; + + + + + CBLK_TRACE_LOG_FILE(9,"chunk = %p,path_index = %d, all_paths = %d", + chunk,path_index,all_paths); + /* + * Clear the RRQ here, since it could + * have poison bits. + * + * NOTE: When the RRQ was allocated, we ensured + * it started and ended on cacheline boundaries. + * Since cblk_clear_poison_bits only clears poison + * bits in a cacheline this was needed. + */ + + cblk_clear_poison_bits(chunk->path[path_index]->afu->p_hrrq_start,chunk->path[path_index]->afu->size_rrq); /* @@ -7219,7 +8169,7 @@ void cblk_clear_poison_bits_chunk(cflsh_chunk_t *chunk, int path_index, int all_ * buffers to prevent UEs when access the read * data buffer. . */ - + for (i=0; i < chunk->num_cmds; i++) { if ((chunk->cmd_info[i].in_use) && (chunk->cmd_info[i].state == CFLSH_MGM_HALTED) && @@ -7267,8 +8217,8 @@ void cblk_clear_poison_bits_chunk(cflsh_chunk_t *chunk, int path_index, int all_ } /* - * Only the adapter specific area could have - * poison bits, but for simplicity (to avoid + * Only the adapter specific area of a command + * could have poison bits, but for simplicity (to avoid * using a an adapter specific routine to clear * the poison bits), we will just clear the whole * cmd. @@ -7291,12 +8241,112 @@ void cblk_clear_poison_bits_chunk(cflsh_chunk_t *chunk, int path_index, int all_ if (cmdi->flags & CFLSH_MODE_READ) { - /* - * Clear read data buffer of any potential poison bits + /* + * Clear cache aligned portion of read data buffer of any potential poison bits */ cblk_clear_poison_bits(cmdi->buf,(cmdi->nblocks*CAPI_FLASH_BLOCK_SIZE)); + + if ((ulong)cmdi->buf & (CBLK_DCBZ_CACHE_LINE_SZ-1)) { + + + /* + * This data buffer is not cache aligned and + * the non-aligned portion could have UE that was not + * cleared when we cleared the cache aligned portion above. + * For the non-cache aligned part we need to see + * if fortunately one of the other read commands has the remainder + * of this cache line and then clear the poison bits on that cache + * line. Otherwise this will be a fatal error for this request. + * + * NOTE: This code assumes that a cache line size is smaller than + * a disk block size. Thus at most there will be one other read + * request that shares the same cache line with the + * beginning of this read/s buffers and one read request + * that shares the same cache line with the end of this read's + * buffer. + * + */ + + + + CBLK_TRACE_LOG_FILE(1,"non-aligned Read command with UE lba = 0x%llx flags = 0x%x, chunk->index = %d", + cmdi->lba,cmd->cmdi->flags,chunk->index); + + cmdi_start_addr = (ulong)((ulong)cmdi->buf & ~(CBLK_DCBZ_CACHE_LINE_SZ-1)); + + cmdi_end_addr = (ulong)(((ulong)cmdi->buf +(cmdi->nblocks*CAPI_FLASH_BLOCK_SIZE)) & ~(CBLK_DCBZ_CACHE_LINE_SZ-1)); + + for (j=0; j < chunk->num_cmds; j++) { + if ((chunk->cmd_info[j].in_use) && + (chunk->cmd_info[j].state == CFLSH_MGM_HALTED) && + (j != i) && + (all_paths || (chunk->cmd_info[j].path_index == path_index))) { + + cmdi2 = &chunk->cmd_info[j]; + + if (cmdi2->flags & CFLSH_MODE_READ) { + + /* + * This is also a read so that we can clear poison bits + * if necessary. + */ + + cmdi2_start_addr = (ulong)((ulong)cmdi2->buf & ~(CBLK_DCBZ_CACHE_LINE_SZ-1)); + + cmdi2_end_addr = (ulong)(((ulong)cmdi2->buf +(cmdi2->nblocks*CAPI_FLASH_BLOCK_SIZE)) & ~(CBLK_DCBZ_CACHE_LINE_SZ-1)); + + if (cmdi2_start_addr == cmdi_end_addr) { + + /* + * This cacheline is starts with cmdi2's data buffer + * and ends at the 1st cacheline boundary in cmdi's + * data buffer. Lets clear the poison bits in it. + */ + + cblk_clear_poison_bits((void *)cmdi2_start_addr,CBLK_DCBZ_CACHE_LINE_SZ); + + } else if (cmdi_start_addr == cmdi2_end_addr) { + + /* + * This cacheline is starts with cmdi's data buffer + * and ends at the 1st cacheline boundary in cmdi2's + * data buffer. Lets clear the poison bits in it. + */ + + cblk_clear_poison_bits((void *)cmdi_start_addr,CBLK_DCBZ_CACHE_LINE_SZ); + } + } + } + + } /* inner for loop */ + } + } + + + /* + * According to the CAPI architecture, we should never get a UE on a host buffer + * that is write only to storage and is cache aligned. UE's arise when the + * AFU can checkout a cacheline to modify it (a Read from the host perspective) and the + * CAPI link is dropped. Since a data buffer may not be cache aligned, either a read + * or write request could have a data buffer that contains a cache line shared + * by another command that is a read buffer. Thus that portion of the cacheline could + * have a UE. We will call the following routine to determine if this is is the case. + * and fail it if so. + */ + + cblk_check_cmd_data_buf_for_ue(chunk,cmd); + + + + /* + * Set new time + * in command + */ + + cmdi->cmd_time = time(NULL); + } diff --git a/src/block/cflash_block_internal.h b/src/block/cflash_block_internal.h index 77721bd1..abc2106c 100644 --- a/src/block/cflash_block_internal.h +++ b/src/block/cflash_block_internal.h @@ -49,6 +49,14 @@ #include #endif #include +#ifdef _AIX +#ifndef ENOTRECOVERABLE +#define ENOTRECOVERABLE 94 /* If we are compiling on an old AIX level */ + /* that does not have this errno, then */ + /* define it here to value used by recent */ + /* levels of AIX. */ +#endif +#endif /* _AIX */ #include #ifndef _MACOSX #include @@ -95,7 +103,6 @@ typedef uint64_t dev64_t; //#define _SKIP_POLL_CALL 1 //#define _SKIP_READ_CALL 1 //#define _MASTER_CONTXT 1 -//#define _PERF_TEST 1 #define _ERROR_INTR_MODE 1 /* Only rely on interrupts for errors. */ /* Normal command completions are polled */ /* from RRQ. */ @@ -179,6 +186,10 @@ typedef uint64_t dev64_t; #define CAPI_FLASH_BLOCK_SIZE 4096 +#define CBLK_CACHE_LINE_SIZE 128 /* Size of OS cachine */ + +#define CBLK_DCBZ_CACHE_LINE_SZ 128 /* Size of cacheline assumed by dcbz */ + #define CFLASH_BLOCK_MAX_WAIT_ROOM_RETRIES 5000 /* Number of times to check for */ /* command room before giving up */ #define CFLASH_BLOCK_DELAY_ROOM 1000 /* Number of microseconds to delay */ @@ -225,6 +236,23 @@ typedef uint64_t dev64_t; +/************************************************************************/ +/* Unrecoverable Error (UE) macros */ +/************************************************************************/ + +/* + * Read (dereference) memory address,but + * verify there is no UE here. Currently + * this macro is a no op, since the new AIX + * interface is not yet ready. + */ + + +#if defined(_CBLK_UE_SAFE) && defined(_AIX) +#define CBLK_READ_ADDR_CHK_UE(_address_) (ue_load((void *)_address_)) +#else +#define CBLK_READ_ADDR_CHK_UE(_address_) (*(_address_)) +#endif /* Linux or old AIX level */ /************************************************************************/ /* Miscellaneous */ @@ -389,7 +417,11 @@ do { \ #elif _MACOSX #define CBLK_LWSYNC() asm volatile ("mfence") #else +#ifndef TARGET_ARCH_x86_64 #define CBLK_LWSYNC() __asm__ __volatile__ ("lwsync") +#else +#define CBLK_LWSYNC() +#endif #endif @@ -857,21 +889,9 @@ do { \ #endif /* !_ERROR_INTR_MODE */ -#ifndef _PERF_TEST - #ifdef _AIX - #define CFLASH_MAX_WAIT_LOOP_CNT 100000 - #else - #define CFLASH_MAX_WAIT_LOOP_CNT 500000 - #endif -#else -#define CFLASH_MAX_WAIT_LOOP_CNT 1 -#endif +#define CFLASH_DELAY_LOOP_CNT 50000000 -#ifdef _AIX -#define CFLASH_DELAY_NO_CMD_INTRPT 500/* Time in microseconds to delay */ -#else -#define CFLASH_DELAY_NO_CMD_INTRPT 100/* Time in microseconds to delay */ -#endif +#define CFLASH_DELAY_NO_CMD_INTRPT 1 /* Time in microseconds to delay */ /* between checking RRQ when */ /* running without command */ /* completion interrupts. */ @@ -884,13 +904,7 @@ do { \ /* to reach 50 seconds */ -#define CFLASH_MIN_POLL_RETRIES 100 /* Minimum number of poll retries*/ - /* before we start delaying. */ -#ifdef _AIX -#define CFLASH_MAX_POLL_RETRIES 20000 -#else -#define CFLASH_MAX_POLL_RETRIES 100000 -#endif +#define CFLASH_MAX_POLL_RETRIES 200000 #define CFLASH_MAX_POLL_FAIL_RETRIES 10 @@ -1039,6 +1053,7 @@ typedef struct cflsh_cmd_mgm_s { }; struct cflsh_cmd_info_s *cmdi; /* Associated command info */ int index; /* index of command */ + uint64_t stime; /* ticks of cmd start */ #if !defined(__64BIT__) && defined(_AIX) int reserved2[10]; /* Reserved for future use */ #else @@ -1120,10 +1135,12 @@ typedef struct cflsh_cmd_info_s { typedef enum { - CFLASH_BLK_CHUNK_NONE = 0x0, /* No command type */ - CFLASH_BLK_CHUNK_SIS_LITE = 0x1, /* Chunk for SIS Lite device*/ - CFLASH_BLK_CHUNK_SIS_SAS64 = 0x2, /* Future SISSAS64 chunk */ - /* type. */ + CFLASH_BLK_CHUNK_NONE = 0x0, /* No command type */ + CFLASH_BLK_CHUNK_SIS_LITE = 0x1, /* Chunk for SIS Lite device*/ + CFLASH_BLK_CHUNK_SIS_LITE_SQ = 0x2, /* Chunk for SIS Lite device*/ + /* using SQ. */ + CFLASH_BLK_CHUNK_SIS_SAS64 = 0x3, /* Future SISSAS64 chunk */ + /* type. */ } cflsh_block_chunk_type_t; @@ -1136,8 +1153,10 @@ typedef struct cflsh_chunk_fcn_ptrs { int (*get_num_interrupts)(struct cflsh_chunk_s *chunk, int path_index); uint64_t (*get_cmd_room)(struct cflsh_chunk_s *chunk, int path_index); int (*adap_setup)(struct cflsh_chunk_s *chunk, int path_index); + int (*adap_sq_setup)(struct cflsh_chunk_s *chunk, int path_index); uint64_t (*get_intrpt_status)(struct cflsh_chunk_s *chunk, int path_index); void (*inc_rrq)(struct cflsh_chunk_s *chunk, int path_index); + void (*inc_sq)(struct cflsh_chunk_s *chunk, int path_index); uint32_t (*get_cmd_data_length)(struct cflsh_chunk_s *chunk, cflsh_cmd_mgm_t *cmd); scsi_cdb_t *(*get_cmd_cdb)(struct cflsh_chunk_s *chunk, cflsh_cmd_mgm_t *cmd); cflsh_cmd_mgm_t *(*get_cmd_rsp)(struct cflsh_chunk_s *chunk, int path_index); @@ -1194,6 +1213,10 @@ typedef struct cflsh_afu_s { int flags; /* Flags for this path */ #define CFLSH_AFU_SHARED 0x1 /* This AFU can be shared */ #define CFLSH_AFU_HALTED 0x2 /* AFU is in a halted state */ +#define CFLSH_AFU_RECOV 0x4 /* AFU is in a recovery state */ +#define CFLSH_AFU_SQ 0x8 /* This AFU is using an sub- */ + /* mission quue (SQ). */ + int ref_count; /* Reference count for this */ /* path */ int poll_fd; /* File descriptor for poll or */ @@ -1209,16 +1232,33 @@ typedef struct cflsh_afu_s { /* with the AFU. */ uint64_t toggle; /* Toggle bit for RRQ */ cflsh_block_chunk_type_t type;/* CAPI block AFU type */ -#ifdef _AIX dev64_t adap_devno; /* Devno of adapter. */ -#endif /* _AIX */ uint64_t *p_hrrq_start; /* Start of Host */ /* Request/Response Queue */ uint64_t *p_hrrq_end; /* End of Host */ /* Request/Response Queue */ volatile uint64_t *p_hrrq_curr;/* Current Host */ /* Request/Response Queue Entry*/ + int64_t hrrq_to_ioarcb_off;/* This is the value that */ + /* needs to be subtraced from */ + /* a given RRQ returned value */ + /* to get the offset for the */ + /* associated IOARCB. This is */ + /* based on the fact this */ + /* library always places the */ + /* IOARCB before the IOASA in */ + /* the command. */ + + + sisl_ioarcb_t *p_sq_start; /* Start of host submission */ + /* queue (SQ). */ + sisl_ioarcb_t *p_sq_end; /* End of host submission */ + /* queue (SQ). */ + volatile sisl_ioarcb_t *p_sq_curr; /* Current submission */ + /* queue (SQ) Queue Entry */ int num_rrqs; /* Number of RRQ elements */ + int size_rrq; /* Size in bytes of RRQ */ + int size_sq; /* Size in bytes of SQ */ int32_t num_issued_cmds; /* Number of issued commands */ void *mmio_mmap; /* MMIO address returned by */ /* MMAP. The value returned */ @@ -1275,7 +1315,12 @@ typedef struct cflsh_path_s { #define CFLSH_CHNK_SIGH 0x0002 /* MMIO signal handler is setup*/ #define CFLSH_PATH_RST 0x0004 /* Unprocessed context reset */ #define CFLSH_PATH_A_RST 0x0008 /* Unprocessed adap reset */ - +#define CFLSH_PATH_ATTACH 0x0010 /* Did an attach for this path */ +#define CFLSH_PATH_CLOSE_POLL_FD 0x0020 /* For certain error recovery */ + /* actions (detach, recover */ + /* clone we need to close this */ + /* path's file descriptor. */ + cflsh_afu_in_use_t afu_share_type; int path_index; /* Path to issue command */ uint16_t path_id; /* Path id of selected path */ uint32_t path_id_mask; /* paths to use to access this */ @@ -1309,6 +1354,17 @@ typedef struct cflsh_path_s { jmp_buf jmp_read; /* Used to long jump around */ /* read hangs */ #endif /* REMOVE */ +#ifndef _AIX + char dev_name[PATH_MAX]; /* Device special filename */ + /* for this path. */ + +#endif /* !_AIX */ + int fd; /* File descriptor. For linux */ + /* each path will have a */ + /* separate fd. For AIX each */ + /* path uses the same fd as */ + /* as the chunk, because of */ + /* MPIO. */ eye_catch4b_t eyec; /* Eye catcher */ } cflsh_path_t; @@ -1324,7 +1380,7 @@ typedef struct cflsh_path_s { #define MAX_NUM_CHUNKS_HASH 64 -#define CHUNK_HASH_MASK 0x0000003f +#define CHUNK_HASH_MASK (MAX_NUM_CHUNKS_HASH-1) /************************************************************************/ @@ -1430,10 +1486,45 @@ typedef struct cflsh_chunk_s { uint64_t num_blocks_lun; /* Maximum size in blocks of */ /* this lpysical lun */ cflsh_path_t *path[CFLSH_BLK_MAX_NUM_PATHS]; /* Adapter paths for this chunk */ + char *udid; /* Unique Device Identifier */ + + uint64_t ptime; /* track when to trace lat */ + uint64_t rcmd; /* #cmds in read lat calc */ + uint64_t wcmd; /* #cmds in write lat calc */ + uint64_t rlat; /* total rd latency in ticks */ + uint64_t wlat; /* total wr latency in ticks */ + eye_catch4b_t eyec; /* Eye catcher */ } cflsh_chunk_t; +/************************************************************************/ +/* Anchor block for each Hash list of chunks */ +/************************************************************************/ +typedef struct cflsh_chunk_hash_s +{ + cflsh_blk_lock_t lock; + cflsh_chunk_t *head; + cflsh_chunk_t *tail; +} cflsh_chunk_hash_t; + + +#ifndef _AIX +#define CFLASH_BLOCK_HOST_TYPE_FILE "/proc/cpuinfo" + + + +#endif /* ! _AIX */ + +typedef +enum { + CFLASH_HOST_UNKNOWN = 0, /* Unknown host type */ + CFLASH_HOST_NV = 1, /* Bare Metal (or No virtualization */ + /* host type. */ + CFLASH_HOST_PHYP = 2, /* pHyp host type */ + CFLASH_HOST_KVM = 3, /* KVM host type */ +} cflash_host_type_t; + /************************************************************************/ @@ -1451,6 +1542,7 @@ typedef struct cflsh_block_s { #define CFLSH_G_SYSLOG 0x0008 /* Use syslog for all tracing */ int next_chunk_id; /* Chunk id of next allocated */ /* chunk. */ + cflash_host_type_t host_type; /* Host type */ pid_t caller_pid; /* Process ID of caller of */ /* this library. */ @@ -1471,7 +1563,7 @@ typedef struct cflsh_block_s { cflsh_afu_t *head_afu; /* Head of list of AFUs */ cflsh_afu_t *tail_afu; /* Tail of list of AFUs */ - cflsh_chunk_t *hash[MAX_NUM_CHUNKS_HASH]; + cflsh_chunk_hash_t hash[MAX_NUM_CHUNKS_HASH]; uint64_t next_chunk_starting_lba; /* This is the starting LBA */ @@ -1498,7 +1590,8 @@ typedef struct cflsh_block_s { uint32_t thread_log_mask; /* Mask used to hash thread */ /* logs into specific files. */ cflsh_thread_log_t *thread_logs; /* Array of log files per thread*/ - + double nspt; /* nanoseconds per tick */ + #ifdef _SKIP_READ_CALL int adap_poll_delay;/* Adapter poll delay time in */ /* microseconds */ @@ -1552,4 +1645,22 @@ enum { } cflash_block_notify_reason_t; +/************************************************************************/ +/* check os status codes */ +/************************************************************************/ + +typedef +enum { + CFLSH_BLK_CHK_OS_NO_RESET = 0, /* No adapter reset done */ + CFLSH_BLK_CHK_OS_RESET_SUCC = 1, /* Adapter reset was */ + /* successful */ + CFLSH_BLK_CHK_OS_RESET_FAIL = 2, /* Adapter reset failed */ + CFLSH_BLK_CHK_OS_RESET_PEND = 3, /* Adapter reset pending */ + CFLSH_BLK_CHK_OS_RESET = 4, /* Adapter reset done, */ + /* but no information on */ + /* whether it succeeded */ + /* or failed. */ + +} cflash_block_check_os_status_t; + #endif /* _H_CFLASH_BLOCK_INT */ diff --git a/src/block/cflash_block_kern_mc.c b/src/block/cflash_block_kern_mc.c index bb11fc03..1a35d89e 100644 --- a/src/block/cflash_block_kern_mc.c +++ b/src/block/cflash_block_kern_mc.c @@ -57,6 +57,9 @@ typedef struct dk_capi_recover_context dk_capi_recover_context_t; #include #include +#ifdef DK_CXLFLASH_CONTEXT_SQ_CMD_MODE +#ident "$Debug: SQ_CMD_MODE $" +#endif #ifndef DK_CAPI_ATTACH /* * Create common set of defines/types across @@ -199,11 +202,29 @@ cflsh_block_chunk_type_t cblk_get_os_chunk_type(const char *path, int arch_typ * evaluate it. */ - if (arch_type != DK_ARCH_SISLITE) { + switch (arch_type) { + case DK_ARCH_SISLITE: + /* + * Original SISLITE (default) type + */ + chunk_type = CFLASH_BLK_CHUNK_SIS_LITE; + break; +#ifdef DK_ARCH_SISLITE_SQ + case DK_ARCH_SISLITE_SQ: + /* + * Sislite type that uses submission + * queue (SQ). + */ + chunk_type = CFLASH_BLK_CHUNK_SIS_LITE_SQ; + break; +#endif /* DK_ARCH_SISLITE_SQ */ + default: return CFLASH_BLK_CHUNK_NONE; } + + return chunk_type; } #endif @@ -217,12 +238,14 @@ cflsh_block_chunk_type_t cblk_get_os_chunk_type(const char *path, int arch_typ return chunk_type; } +#ifndef _AIX /* ---------------------------------------------------------------------------- * - * NAME: cblk_find_parent_dev + * NAME: cblk_extract_udid_from_vpd83_file * - * FUNCTION: Find parent string of lun. + * FUNCTION: Parse the inquiry page 83 file in sysfs + * * * * @@ -241,19 +264,319 @@ cflsh_block_chunk_type_t cblk_get_os_chunk_type(const char *path, int arch_typ * * ---------------------------------------------------------------------------- */ +char *cblk_extract_udid_from_vpd83_file(char *vpd_pg83_filename) +{ -#ifdef _AIX -char *cblk_find_parent_dev(char *device_name) + +#define CFLASH_BLOCK_BUF_SIZE 256 +#define CFLASH_WWID_SIZE 256 + + FILE *fp; + char buf[CFLASH_BLOCK_BUF_SIZE]; + char *udid = NULL; + char wwid[CFLASH_WWID_SIZE]; + int length; + int rc; + + + CBLK_TRACE_LOG_FILE(9,"vpd_pg83 name = %s",vpd_pg83_filename); + + + + /* + * The data read out of the vpd_pg83 file is the + * inquiry data of page 0x83, which is the + * Device Identification VPD page. + */ + + if ((fp = fopen(vpd_pg83_filename,"r")) == NULL) { + + CBLK_TRACE_LOG_FILE(1,"failed to open = %s", + vpd_pg83_filename); + return udid; + } + + + length = fread(buf,1,CFLASH_BLOCK_BUF_SIZE,fp); + + if (length < sizeof(struct inqry_pg83_data)) { + + + + CBLK_TRACE_LOG_FILE(1,"failed to read page 0x83 header, read %d bytes, but expected *%d bytes, errno = %d", + length,sizeof(struct inqry_pg83_data),errno); + + fclose(fp); + + return udid; + } + + bzero(wwid,CFLASH_WWID_SIZE); + + + rc = cflash_process_scsi_inquiry_dev_id_page(buf,length,wwid); + + if (rc) { + + CBLK_TRACE_LOG_FILE(1,"failed to parse page 0x83 with rc = %d, and errno = %d", + rc,errno); + } else { + + + udid = strdup(wwid); + } + + fclose (fp); + return udid; + + +} + +/* ---------------------------------------------------------------------------- + * + * NAME: cblk_find_other_paths_to_device + * + * FUNCTION: Walk all cflash disks in the same and looks for those + * that have the same udid (universal device ID). + * + * + * + * + * CALLED BY: + * + * + * INTERNAL PROCEDURES CALLED: + * + * + * + * EXTERNAL PROCEDURES CALLED: + * + * + * + * RETURNS: + * + * ---------------------------------------------------------------------------- + */ +int cblk_find_other_paths_to_device(char *subsystem_name, char *orig_udid, + char *cur_devname, char *dev_name_list[], + int size_devname_list) { - return NULL; + struct udev *udev_lib; + struct udev_device *device, *parent; + struct udev_enumerate *device_enumerate; + struct udev_list_entry *device_list, *device_entry; + const char *device_mode = NULL; + char *parent_name = NULL; + const char *path = NULL; + char vpd_pg83_filename[PATH_MAX]; + char *udid = NULL; + char *devname = NULL; + int num_matches_found = 0; + + + CBLK_TRACE_LOG_FILE(9,"subsystem_path = %s, cur_devname = %s",subsystem_name,cur_devname); + + + /* + * Extract cur_devname with absolute path removed + */ + + cur_devname = rindex(cur_devname,'/'); + + cur_devname++; + + + udev_lib = udev_new(); + + if (udev_lib == NULL) { + CBLK_TRACE_LOG_FILE(1,"udev_new failed with errno = %d",errno); + + + return 0; + + } + + + device_enumerate = udev_enumerate_new(udev_lib); + + if (device_enumerate == NULL) { + + CBLK_TRACE_LOG_FILE(1,"udev_enumerate_new failed with errno = %d",errno); + + + return 0; + + } + + + udev_enumerate_add_match_subsystem(device_enumerate,subsystem_name); + + udev_enumerate_scan_devices(device_enumerate); + + device_list = udev_enumerate_get_list_entry(device_enumerate); + + + for (device_entry = device_list; device_entry != NULL; device_entry = udev_list_entry_get_next(device_entry)) { + + path = udev_list_entry_get_name(device_entry); + + if (path == NULL) { + + CBLK_TRACE_LOG_FILE(1,"udev_list_entry_get_name failed with errno = %d",errno); + continue; + } + + + CBLK_TRACE_LOG_FILE(9,"path = %s",path); + + + /* + * Extract filename with absolute path removed + */ + + devname = rindex(path,'/'); + devname++; + + if (!(strcmp(devname,cur_devname))) { + + /* + * Skip over our device name + */ + + continue; + } + + device = udev_device_new_from_syspath(udev_lib,path); + + if (device == NULL) { + + CBLK_TRACE_LOG_FILE(1,"udev_device_new_from_subsystem_syspath failed with errno = %d",errno); + + continue; + + } + + device_mode = udev_device_get_sysattr_value(device,"device/mode"); + + + if (device_mode == NULL) { + + CBLK_TRACE_LOG_FILE(1,"no mode for this device "); + + + continue; + + } else { + + + if (strcmp(device_mode,"superpipe")) { + + CBLK_TRACE_LOG_FILE(1,"Device is not in superpipe mode = %s",device_mode); + + continue; + } + + CBLK_TRACE_LOG_FILE(9,"disk in superpipe mode found"); + + parent = udev_device_get_parent(device); + + if (parent == NULL) { + + CBLK_TRACE_LOG_FILE(1,"udev_device_get_parent failed with errno = %d",errno); + + continue;; + + } + + parent_name = (char *)udev_device_get_devpath(parent); + + CBLK_TRACE_LOG_FILE(9,"parent name = %s",parent_name); + + sprintf(vpd_pg83_filename,"/sys%s/vpd_pg83",parent_name); + + udid = cblk_extract_udid_from_vpd83_file(vpd_pg83_filename); + + if (!(strcmp(udid,orig_udid))) { + + + devname = rindex(path,'/'); + devname++; + + CBLK_TRACE_LOG_FILE(9,"found device %s with same udid = %s",devname,vpd_pg83_filename); + + + if (num_matches_found < size_devname_list) { + + dev_name_list[num_matches_found] = malloc(PATH_MAX); + + if (dev_name_list[num_matches_found] == NULL) { + CBLK_TRACE_LOG_FILE(1,"malloc for dev_name = %s on %d entry failed with errno = %d", + devname,num_matches_found,errno); + break; + } + + bzero(dev_name_list[num_matches_found],PATH_MAX); + + sprintf(dev_name_list[num_matches_found],"/dev/%s",devname); + + + } else { + + /* + * We have exceeded the size of + * this list + */ + + break; + } + + num_matches_found++; + + + + } + + + } + + + } + + CBLK_TRACE_LOG_FILE(9,"num_matches_found = %d",num_matches_found); + + return num_matches_found; } -#else -char *cblk_find_parent_dev(char *device_name) + +/* ---------------------------------------------------------------------------- + * + * NAME: cblk_find_full_parent_name + * + * FUNCTION: Find parent name, that also includes child name portion + * for this device. + * + * + * + * CALLED BY: + * + * + * INTERNAL PROCEDURES CALLED: + * + * + * + * EXTERNAL PROCEDURES CALLED: + * + * + * + * RETURNS: + * + * ---------------------------------------------------------------------------- + */ +char *cblk_find_full_parent_name(char *device_name) { + char *parent_name = NULL; - char *child_part = NULL; const char *device_mode = NULL; char *subsystem_name = NULL; char *devname = NULL; @@ -330,6 +653,8 @@ char *cblk_find_parent_dev(char *device_name) } + CBLK_TRACE_LOG_FILE(9,"device found for subsystem name = %s",subsystem_name); + device_mode = udev_device_get_sysattr_value(device,"device/mode"); @@ -351,6 +676,8 @@ char *cblk_find_parent_dev(char *device_name) } } + CBLK_TRACE_LOG_FILE(9,"device_mode found for subsystem name = %s",subsystem_name); + parent = udev_device_get_parent(device); if (parent == NULL) { @@ -361,11 +688,71 @@ char *cblk_find_parent_dev(char *device_name) } + + CBLK_TRACE_LOG_FILE(9,"parent found for subsystem name = %s",subsystem_name); + parent_name = (char *)udev_device_get_devpath(parent); CBLK_TRACE_LOG_FILE(9,"parent name = %s",parent_name); + /* udev is setting errno to non-zero, but there is no real error */ + errno=0; + return (parent_name); + +} +#endif /* !_AIX */ + + +/* ---------------------------------------------------------------------------- + * + * NAME: cblk_find_parent_dev + * + * FUNCTION: Find parent string of lun. + * + * + * + * CALLED BY: + * + * + * INTERNAL PROCEDURES CALLED: + * + * + * + * EXTERNAL PROCEDURES CALLED: + * + * + * + * RETURNS: + * + * ---------------------------------------------------------------------------- + */ + +#ifdef _AIX +char *cblk_find_parent_dev(char *device_name) +{ + + return NULL; +} +#else + +char *cblk_find_parent_dev(char *device_name) +{ + char *parent_name = NULL; + char *child_part = NULL; + + + + + parent_name = cblk_find_full_parent_name(device_name); + + if (parent_name == NULL) { + + return parent_name; + } + + + /* * This parent name string will actually have sysfs directories * associated with the connection information of the child. We @@ -387,6 +774,64 @@ char *cblk_find_parent_dev(char *device_name) } #endif /* !AIX */ + + + + +/* ---------------------------------------------------------------------------- + * + * NAME: cblk_get_udid + * + * FUNCTION: Get device's unique identifier + * + * + * + * CALLED BY: + * + * + * INTERNAL PROCEDURES CALLED: + * + * + * + * EXTERNAL PROCEDURES CALLED: + * + * + * + * RETURNS: + * + * ---------------------------------------------------------------------------- + */ + +#ifdef _AIX +char *cblk_get_udid(char *device_name) +{ + + return NULL; +} +#else + +char *cblk_get_udid(char *device_name) +{ + char vpd_pg83_filename[PATH_MAX]; + char *udid = NULL; + char *parent_name = NULL; + + + parent_name = cblk_find_full_parent_name(device_name); + + + sprintf(vpd_pg83_filename,"/sys%s/vpd_pg83",parent_name); + + udid = cblk_extract_udid_from_vpd83_file(vpd_pg83_filename); + + + return udid; +} + +#endif /* !AIX */ + + + /* ---------------------------------------------------------------------------- * * NAME: cblk_chunk_attach_path @@ -481,14 +926,14 @@ int cblk_chunk_attach_path(cflsh_chunk_t *chunk, int path_index,int mode, // TODO:?? Is this needed disk_attach.flags = CXL_START_WORK_NUM_IRQS; - rc = ioctl(chunk->fd,DK_CAPI_ATTACH,&disk_attach); + rc = ioctl(chunk->path[path_index]->fd,DK_CAPI_ATTACH,&disk_attach); if (rc) { - CBLK_TRACE_LOG_FILE(1,"Unable to attach errno = %d, return_flags = 0x%llx", - errno,disk_attach.return_flags); + CBLK_TRACE_LOG_FILE(1,"Unable to attach errno = %d, return_flags = 0x%llx, num_active_chunks = 0x%x", + errno,disk_attach.return_flags,cflsh_blk.num_active_chunks); @@ -620,14 +1065,56 @@ int cblk_chunk_attach_path(cflsh_chunk_t *chunk, int path_index,int mode, chunk->path[path_index]->afu->mmap_size = CAPI_FLASH_REG_SIZE; -#endif /* !_MASTER_CONTXT */ - -#ifdef _AIX - chunk->path[path_index]->afu->contxt_id = disk_attach.ctx_token; - chunk->path[path_index]->afu->contxt_handle = 0xffffffff & disk_attach.ctx_token; -#else - chunk->path[path_index]->afu->contxt_id = disk_attach.context_id; - chunk->path[path_index]->afu->contxt_handle = 0xffffffff & disk_attach.context_id; +#endif /* !_MASTER_CONTXT */ + +#ifdef _AIX + chunk->path[path_index]->afu->contxt_id = disk_attach.ctx_token; + chunk->path[path_index]->afu->contxt_handle = 0xffffffff & disk_attach.ctx_token; +#else + chunk->path[path_index]->afu->contxt_id = disk_attach.context_id; + chunk->path[path_index]->afu->contxt_handle = 0xffffffff & disk_attach.context_id; + +#ifdef DK_CXLFLASH_CONTEXT_SQ_CMD_MODE + if (disk_attach.return_flags & DK_CXLFLASH_CONTEXT_SQ_CMD_MODE) { + + + /* + * driver is indicating this AFU supports SQ. So switch + * the chunk/afu type to CFLASH_BLK_CHUNK_SIS_LITE_SQ) + */ + + CBLK_TRACE_LOG_FILE(9,"set,DK_CXLFLASH_CONTEXT_SQ_CMD_MODE contxt_id = 0x%llx",chunk->path[path_index]->afu->contxt_id); + + + + if (cblk_update_path_type(chunk,chunk->path[path_index],CFLASH_BLK_CHUNK_SIS_LITE_SQ)) { + + + CBLK_TRACE_LOG_FILE(1,"failed to update chunk path type for SQ for contxt_id = 0x%llx", + chunk->path[path_index]->afu->contxt_id); + + + + cblk_release_path(chunk,(chunk->path[path_index])); + + chunk->path[path_index] = NULL; + + return -1; + } + + } +#endif /*DK_CXLFLASH_CONTEXT_SQ_CMD_MODE */ + + +#ifdef DK_CXLFLASH_APP_CLOSE_ADAP_FD + if (disk_attach.return_flags & DK_CXLFLASH_APP_CLOSE_ADAP_FD) { + + CBLK_TRACE_LOG_FILE(9,"DK_CXLFLASH_APP_CLOSE_ADAP_FD set, contxt_id = 0x%llx",chunk->path[path_index]->afu->contxt_id); + chunk->path[path_index]->flags |= CFLSH_PATH_CLOSE_POLL_FD; + + } +#endif /* DK_CXLFLASH_APP_CLOSE_ADAP_FD */ + #endif /* !AIX */ @@ -641,7 +1128,7 @@ int cblk_chunk_attach_path(cflsh_chunk_t *chunk, int path_index,int mode, chunk->path[path_index]->afu->mmio_mmap = disk_attach.mmio_start; - chunk->path[path_index]->afu->mmio = chunk->path[chunk->cur_path]->afu->mmio_mmap; + chunk->path[path_index]->afu->mmio = chunk->path[path_index]->afu->mmio_mmap; chunk->path[path_index]->afu->mmap_size = disk_attach.mmio_size; #else @@ -667,7 +1154,8 @@ int cblk_chunk_attach_path(cflsh_chunk_t *chunk, int path_index,int mode, #endif - + + chunk->path[path_index]->flags |= CFLSH_PATH_ATTACH; return rc; } @@ -701,7 +1189,7 @@ int cblk_chunk_attach_path(cflsh_chunk_t *chunk, int path_index,int mode, * ---------------------------------------------------------------------------- */ int cblk_chunk_attach_process_map_path(cflsh_chunk_t *chunk, int path_index,int mode, - dev64_t adap_devno, int arch_type, + dev64_t adap_devno, char *path_dev_name, int fd, int arch_type, int *cleanup_depth, int assign_path) { @@ -743,7 +1231,7 @@ int cblk_chunk_attach_process_map_path(cflsh_chunk_t *chunk, int path_index,int share = TRUE; } - path = cblk_get_path(chunk,adap_devno,chunk_type,chunk->num_cmds,&in_use,share); + path = cblk_get_path(chunk,adap_devno,path_dev_name,chunk_type,chunk->num_cmds,&in_use,share); if (path == NULL) { @@ -756,8 +1244,11 @@ int cblk_chunk_attach_process_map_path(cflsh_chunk_t *chunk, int path_index,int return (-1); } + chunk->path[path_index] = path; + chunk->path[path_index]->fd = fd; + path->path_index = path_index; if (in_use == CFLASH_AFU_MPIO_INUSE) { @@ -789,9 +1280,28 @@ int cblk_chunk_attach_process_map_path(cflsh_chunk_t *chunk, int path_index,int } + } else if (!(chunk->flags & CFLSH_CHNK_VLUN) && + (chunk->path[path_index]->afu_share_type == CFLASH_AFU_SHARE_INUSE)) { + + /* + * If this after a fork, and a physical lun + * that is using shared AFU. then do + * a shared attach. + */ + + rc = cblk_chunk_attach_path(chunk,path_index,mode,adap_devno,cleanup_depth,assign_path,CFLASH_AFU_SHARE_INUSE); + + if (!rc) { + + chunk->path[path_index]->flags |= CFLSH_PATH_ACT; + } + + return rc; + } + rc = cblk_chunk_attach_path(chunk,path_index,mode,adap_devno,cleanup_depth,assign_path,in_use); if (rc) { @@ -837,12 +1347,45 @@ int cblk_chunk_attach_process_map_path(cflsh_chunk_t *chunk, int path_index,int *cleanup_depth = 40; - CBLK_TRACE_LOG_FILE(6,"mmio = 0x%llx",(uint64_t)chunk->path[path_index]->afu->mmio_mmap); + CBLK_TRACE_LOG_FILE(6,"mmio = 0x%llx, path_index = %d, afu = %p", + (uint64_t)chunk->path[path_index]->afu->mmio_mmap,path_index,chunk->path[path_index]->afu); /* * Set up response queue */ + + bzero((void *)chunk->path[path_index]->afu->p_hrrq_start ,chunk->path[path_index]->afu->size_rrq); + + chunk->path[path_index]->afu->p_hrrq_curr = chunk->path[path_index]->afu->p_hrrq_start; + + /* + * Since the host RRQ is + * bzeroed. The toggle bit in the host + * RRQ that initially indicates we + * have a new RRQ will need to be 1. + */ + + + chunk->path[path_index]->afu->toggle = 1; + + chunk->path[path_index]->afu->num_issued_cmds = 0; + + chunk->cmd_curr = chunk->cmd_start; + + if (chunk->path[path_index]->afu->flags & CFLSH_AFU_SQ) { + + /* + * If this AFU is using a Submission Queue (SQ) + * then, reinitialize the SQ. + */ + bzero((void *)chunk->path[path_index]->afu->p_sq_start ,chunk->path[path_index]->afu->size_sq); + + chunk->path[path_index]->afu->p_sq_curr = chunk->path[path_index]->afu->p_sq_start; + + + } + if (CBLK_ADAP_SETUP(chunk,path_index)) { @@ -859,7 +1402,7 @@ int cblk_chunk_attach_process_map_path(cflsh_chunk_t *chunk, int path_index,int return rc; } -#ifdef _AIX + /* ---------------------------------------------------------------------------- * * NAME: cblk_chunk_reuse_attach_mpio @@ -886,7 +1429,8 @@ int cblk_chunk_attach_process_map_path(cflsh_chunk_t *chunk, int path_index,int * * ---------------------------------------------------------------------------- */ -int cblk_chunk_reuse_attach_mpio(cflsh_chunk_t *chunk,int mode, dev64_t adap_devno,int arch_type, +int cblk_chunk_reuse_attach_mpio(cflsh_chunk_t *chunk,int mode, dev64_t adap_devno, + char *path_dev_name, int fd, int arch_type, int matching_path_index,int path_index, int *cleanup_depth) { int rc = 0; @@ -905,7 +1449,7 @@ int cblk_chunk_reuse_attach_mpio(cflsh_chunk_t *chunk,int mode, dev64_t adap_de share = TRUE; } - path = cblk_get_path(chunk,adap_devno,chunk_type,chunk->num_cmds,&in_use,share); + path = cblk_get_path(chunk,adap_devno,path_dev_name,chunk_type,chunk->num_cmds,&in_use,share); if (path == NULL) { @@ -918,6 +1462,8 @@ int cblk_chunk_reuse_attach_mpio(cflsh_chunk_t *chunk,int mode, dev64_t adap_de } chunk->path[path_index] = path; + + chunk->path[path_index]->fd = fd; path->path_index = path_index; @@ -967,7 +1513,7 @@ int cblk_chunk_reuse_attach_mpio(cflsh_chunk_t *chunk,int mode, dev64_t adap_de return rc; } -#endif /* AIX */ + /* ---------------------------------------------------------------------------- * @@ -997,9 +1543,10 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup { int path_index = 0; dev64_t adap_devno = 0; + int i,j; /* general counter */ + int previous_path_found; #ifdef _AIX struct devinfo iocinfo; - int i,j; /* general counter */ int first_reserve_path = -1; int first_good_path = -1; int second_good_path = -1; @@ -1013,8 +1560,13 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup uint32_t block_size = 0; dev64_t prim_path_devno; int prim_path_id = -1; - int previous_path_found; cflsh_block_chunk_type_t chunk_type; +#else + char *path_dev_list[CFLSH_BLK_MAX_NUM_PATHS]; + int num_paths = 0; + char *afu_name = NULL; + int open_flags; + int fd; #endif /* _AIX */ @@ -1035,6 +1587,11 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup #endif /* AIX */ + if (chunk->udid == NULL) { + chunk->udid = cblk_get_udid(chunk->dev_name); + } + + CBLK_TRACE_LOG_FILE(9,"chunk->udid = %s",chunk->udid); // Set cur_path to path 0 in the chunk; @@ -1272,9 +1829,13 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup * paths. First let the disk driver assign the primary path (with * the hope it will load balance these assignments) which will have * path_index of 0. Then attach to the remaining paths. + * + * NOTE: For AIX which uses MPIO, the path's file descriptor + * is identical to the chunk file descriptor, since the + * the disk driver (and device) are multi-path aware. */ - if (cblk_chunk_attach_process_map_path(chunk,0,mode,0,0,cleanup_depth,TRUE)) { + if (cblk_chunk_attach_process_map_path(chunk,0,mode,0,chunk->dev_name,chunk->fd,0,cleanup_depth,TRUE)) { return -1; } @@ -1289,7 +1850,10 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup /* * Find primary AFU selected for us. It * may be associated with multiple paths. - * Thus we need to find all of them for this AFU + * Thus we need to find all of them for this AFU. + * In the loop below, path_index indicates the current + * chunk->path[path_index] that is being processed/created. + * The primary path will will be path_index of 0. */ for (i = 0;ifd, + path_info[i].architecture, 0,path_index,cleanup_depth)) { @@ -1332,7 +1905,7 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup } else { /* - * Trace primary path_id for debug purposes + * Trace tentative primary path_id for debug purposes */ CBLK_TRACE_LOG_FILE(5,"Tentative Primary path_id = %d and path_index = %d", @@ -1363,12 +1936,13 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup chunk->path[path_index]->num_ports = 1; + CBLK_TRACE_LOG_FILE(9,"For same (primary) AFU, assigned path_id = %d and path_index = %d", + chunk->path[path_index]->path_id, path_index); + path_index++; - CBLK_TRACE_LOG_FILE(9,"Assigned path_id = %d and path_index = %d", - chunk->path[path_index]->path_id, path_index); } @@ -1376,7 +1950,7 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup if (path_index == 0) { - CBLK_TRACE_LOG_FILE(1,"Could not find select path, num_paths = %d", + CBLK_TRACE_LOG_FILE(1,"Could not find selected path, num_paths = %d", disk_paths.path.returned_path_count); return -1; } @@ -1400,8 +1974,17 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup if (chunk->path[j]->afu->adap_devno == path_info[i].devno) { + + /* + * + * NOTE: For AIX which uses MPIO, the path's file descriptor + * is identical to the chunk file descriptor, since the + * the disk driver (and device) are multi-path aware. + */ - if (cblk_chunk_reuse_attach_mpio(chunk,mode,path_info[i].devno,path_info[i].architecture, + if (cblk_chunk_reuse_attach_mpio(chunk,mode,path_info[i].devno,NULL, + chunk->fd, + path_info[i].architecture, j,path_index,cleanup_depth)) { @@ -1419,8 +2002,15 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup if (!previous_path_found) { - if (cblk_chunk_attach_process_map_path(chunk,path_index,mode,path_info[i].devno, - path_info[i].architecture, + + /* + * + * NOTE: For AIX which uses MPIO, the path's file descriptor + * is identical to the chunk file descriptor, since the + * the disk driver (and device) are multi-path aware. + */ + if (cblk_chunk_attach_process_map_path(chunk,path_index,mode,path_info[i].devno,NULL, + chunk->fd,path_info[i].architecture, cleanup_depth,FALSE)) { break; @@ -1551,6 +2141,158 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup } +#else + + if ((chunk->flags & CFLSH_CHNK_MPIO_FO) && + !(chunk->flags & CFLSH_CHNK_VLUN) && + (chunk->udid)) { + + + /* + * If we are using MPIO for physical luns, and we have a + * unique device ID (UDID), then attach and map all + * paths. First attach for the primary path. + * Then attach to the remaining paths. + * + * NOTE: For linux the primary path will have the same + * file descriptor as the chunk. All other paths will + * have different file descriptors. + */ + + if (cblk_chunk_attach_process_map_path(chunk,0,mode,0, + chunk->dev_name,chunk->fd,0, + cleanup_depth,TRUE)) { + + + return -1; + } + + /* + * Now find all remaining paths (special files) for this + * same lun. + */ + + bzero(path_dev_list,CFLSH_BLK_MAX_NUM_PATHS); + + num_paths = cblk_find_other_paths_to_device("scsi_generic",chunk->udid,chunk->dev_name, + path_dev_list,CFLSH_BLK_MAX_NUM_PATHS); + + /* + * Advance to the next potential path index + */ + + path_index = 1; + for (i = 0; i < num_paths; i++) { + + CBLK_TRACE_LOG_FILE(9,"path found %d = %s",i,path_dev_list[i]); + + + /* ?? Need support for other architectures */ + + /* + * ?? path file descriptor needs to be passed + * to the leaf routines to do the attach ioctls. + */ + + /* + * Attach to this path. First we need to see if + * we are already attached to this AFU before attempting + * an attach. + */ + + previous_path_found = FALSE; + for (j=0;j< path_index; j++) { + + afu_name = cblk_find_parent_dev(path_dev_list[i]); + + + if (!strcmp(chunk->path[j]->afu->master_name,afu_name)) { + + + CBLK_TRACE_LOG_FILE(9,"reusing attach for %s for path_index = %d", + path_dev_list[i],path_index); + + // ?? Need way to use same flags as original open + + open_flags = O_RDWR | O_NONBLOCK; + + fd = open(path_dev_list[i],open_flags); + + if (fd < 0) { + + CBLK_TRACE_LOG_FILE(1,"Unable to open device %s for path_index = %d errno = %d", + path_dev_list[i],path_index,errno); + + continue; + } + + if (cblk_chunk_reuse_attach_mpio(chunk,mode,0,path_dev_list[i],fd,CFLASH_BLK_CHUNK_SIS_LITE, + j,path_index,cleanup_depth)) { + + + break; + } + + + + previous_path_found = TRUE; + + break; + } + } + + if (!previous_path_found) { + + /* + * If no paths are using this same AFU, then attach now + */ + + + + CBLK_TRACE_LOG_FILE(9,"opening/attaching for new AFU %s for path_index = %d", + path_dev_list[i],path_index); + + // ?? Need way to use same flags as original open + + open_flags = O_RDWR | O_NONBLOCK; + + fd = open(path_dev_list[i],open_flags); + + if (fd < 0) { + + CBLK_TRACE_LOG_FILE(1,"Unable to open device %s for path_index = %d errno = %d", + path_dev_list[i],path_index,errno); + + continue; + } + + if (cblk_chunk_attach_process_map_path(chunk,path_index,mode,0,path_dev_list[i], + fd,0, + cleanup_depth,FALSE)) { + + break; + } + } + + + /* + * Only one port is associated with this path + */ + + chunk->path[path_index]->num_ports = 1; + + CBLK_TRACE_LOG_FILE(9,"Attached path dev_name = %s, path_index = %d", + chunk->path[path_index]->dev_name,path_index); + path_index++; + + } + + + + CBLK_TRACE_LOG_FILE(5,"rc = %d num_paths = %d", + rc, chunk->num_paths); + return rc; + } @@ -1562,7 +2304,7 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup #endif /* BLOCK_FILEMODE_ENABLED */ - if (cblk_chunk_attach_process_map_path(chunk,path_index,mode,adap_devno,0,cleanup_depth,assign_afu)) { + if (cblk_chunk_attach_process_map_path(chunk,path_index,mode,adap_devno,chunk->dev_name,chunk->fd,0,cleanup_depth,assign_afu)) { return -1; } @@ -1752,57 +2494,100 @@ void cblk_chunk_detach_path (cflsh_chunk_t *chunk, int path_index,int force) CFLASH_BLOCK_AFU_SHARE_LOCK(chunk->path[path_index]->afu); - if (chunk->path[path_index]->afu->flags & CFLSH_AFU_HALTED) { - + if ((chunk->path[path_index]->flags & CFLSH_PATH_ATTACH) || + (force)) { + + /* - * If path is in a halted state then fail - * from this routine + * Only detach on the paths, for which we did an attach */ - CBLK_TRACE_LOG_FILE(5,"afu halted, failing detach path afu->flags = 0x%x", - chunk->path[path_index]->afu->flags); + if (chunk->path[path_index]->afu->flags & CFLSH_AFU_HALTED) { + + /* + * If path is in a halted state then fail + * from this routine + */ + + CBLK_TRACE_LOG_FILE(5,"afu halted, failing detach path afu->flags = 0x%x", + chunk->path[path_index]->afu->flags); - CFLASH_BLOCK_AFU_SHARE_UNLOCK(chunk->path[path_index]->afu); - return; + CFLASH_BLOCK_AFU_SHARE_UNLOCK(chunk->path[path_index]->afu); + return; - } + } - /* - * Since we are reusing contexts, it is fine to detach - * even if this is not last path to uses this AFU. - */ + /* + * Since we are reusing contexts, it is fine to detach + * even if this is not last path to uses this AFU. + */ - bzero(&disk_detach,sizeof(disk_detach)); + bzero(&disk_detach,sizeof(disk_detach)); #ifdef _AIX - disk_detach.devno = chunk->path[path_index]->afu->adap_devno; + disk_detach.devno = chunk->path[path_index]->afu->adap_devno; #endif /* AIX */ #ifdef _AIX - disk_detach.ctx_token = chunk->path[path_index]->afu->contxt_id; + disk_detach.ctx_token = chunk->path[path_index]->afu->contxt_id; #else - disk_detach.context_id = chunk->path[path_index]->afu->contxt_id; + disk_detach.context_id = chunk->path[path_index]->afu->contxt_id; #endif /* !AIX */ - rc = ioctl(chunk->fd,DK_CAPI_DETACH,&disk_detach); + rc = ioctl(chunk->path[path_index]->fd,DK_CAPI_DETACH,&disk_detach); - if (rc) { + if (rc) { - CBLK_TRACE_LOG_FILE(1,"DK_CAPI_DETACH e failed with rc = %d, errno = %d, return_flags = 0x%llx", - rc,errno,disk_detach.return_flags); + CBLK_TRACE_LOG_FILE(1,"DK_CAPI_DETACH failed with rc = %d, errno = %d, return_flags = 0x%llx, chunk_id = %d", + rc,errno,disk_detach.return_flags,chunk->index, chunk->flags); - } + CBLK_TRACE_LOG_FILE(1,"DK_CAPI_DETACH failed (cont) chunk->in_use = %d, chunk->dev_name = %s, path_index = %d", + chunk->in_use, chunk->dev_name,path_index); + + + } + + if (chunk->path[path_index]->flags & CFLSH_PATH_CLOSE_POLL_FD) { + + close(chunk->path[path_index]->afu->poll_fd); + } + } CFLASH_BLOCK_AFU_SHARE_UNLOCK(chunk->path[path_index]->afu); +#ifndef _AIX + + + if ((path_index) && + (chunk->path[path_index]->fd >= 0)) { + + /* + * The primary path index's (0), + * file descriptor is same as + * the chunk's file descriptor, + * Thus we do not close the primary + * path index's file descriptor + * here. + */ + + CBLK_TRACE_LOG_FILE(9,"closing = %s, path_index = %d", + chunk->path[path_index]->dev_name,path_index); + + close(chunk->path[path_index]->fd); + + chunk->path[path_index]->fd = -1; + } +#endif /* !_AIX */ + + chunk->path[path_index]->flags &= ~CFLSH_PATH_ACT; return; @@ -2065,7 +2850,7 @@ int cblk_chunk_get_mc_phys_disk_resources_path(cflsh_chunk_t *chunk, */ if ((path_index == 0) && - (!(chunk->flags & CFLSH_CHNK_NO_RESRV))){ + (chunk->flags & CFLSH_CHNK_NO_RESRV)) { /* @@ -2093,7 +2878,7 @@ int cblk_chunk_get_mc_phys_disk_resources_path(cflsh_chunk_t *chunk, - rc = ioctl(chunk->fd,DK_CAPI_USER_DIRECT,&disk_physical); + rc = ioctl(chunk->path[path_index]->fd,DK_CAPI_USER_DIRECT,&disk_physical); if (rc) { @@ -2288,6 +3073,11 @@ int cblk_chunk_get_mc_device_resources(cflsh_chunk_t *chunk, disk_virtual.lun_size = 0; #endif /* !_AIX */ + /* + * Note: For virtual luns, we do not support multi-pathing. + * Thus we only use the chunk file descriptor for this ioctl. + */ + rc = ioctl(chunk->fd,DK_CAPI_USER_VIRTUAL,&disk_virtual); if (rc) { @@ -2442,6 +3232,10 @@ int cblk_chunk_set_mc_size(cflsh_chunk_t *chunk, size_t nblocks) + /* + * Note: For virtual luns, we do not support multi-pathing. + * Thus we only use the chunk file descriptor for this ioctl. + */ rc = ioctl(chunk->fd,DK_CAPI_VLUN_RESIZE,&disk_resize); @@ -2780,6 +3574,11 @@ int cblk_mc_clone(cflsh_chunk_t *chunk,int mode, int flags) #endif + /* + * Note: For virtual luns, we do not support multi-pathing. + * Thus we only use the chunk file descriptor for this ioctl. + */ + rc = ioctl(chunk->fd,DK_CAPI_VLUN_CLONE,&disk_clone); if (rc) { @@ -2818,6 +3617,13 @@ int cblk_mc_clone(cflsh_chunk_t *chunk,int mode, int flags) } + + if (chunk->path[chunk->cur_path]->flags & CFLSH_PATH_CLOSE_POLL_FD) { + + close(old_adap_fd); + } + + #else @@ -2945,7 +3751,7 @@ void cblk_chunk_free_mc_device_resources_path(cflsh_chunk_t *chunk, int path_in - rc = ioctl(chunk->fd,DK_CAPI_RELEASE,&disk_release); + rc = ioctl(chunk->path[path_index]->fd,DK_CAPI_RELEASE,&disk_release); if (rc) { @@ -3205,7 +4011,7 @@ int cblk_read_os_specific_intrpt_event(cflsh_chunk_t *chunk, int path_index,cfls dk_exceptions.rsrc_handle = chunk->path[path_index]->sisl.resrc_handle; dk_exceptions.flags = DK_QEF_ADAPTER; - rc = ioctl(chunk->fd,DK_CAPI_QUERY_EXCEPTIONS,&dk_exceptions); + rc = ioctl(chunk->path[path_index]->fd,DK_CAPI_QUERY_EXCEPTIONS,&dk_exceptions); if (rc) { @@ -3303,7 +4109,7 @@ int cblk_read_os_specific_intrpt_event(cflsh_chunk_t *chunk, int path_index,cfls dk_exceptions.rsrc_handle = chunk->path[path_index]->sisl.resrc_handle; - rc = ioctl(chunk->fd,DK_CAPI_QUERY_EXCEPTIONS,&dk_exceptions); + rc = ioctl(chunk->path[path_index]->fd,DK_CAPI_QUERY_EXCEPTIONS,&dk_exceptions); if (rc) { @@ -3778,7 +4584,7 @@ void cblk_check_os_adap_err_failure_cleanup(cflsh_chunk_t *chunk, cflsh_afu_t *a - afu->flags &= ~CFLSH_AFU_HALTED; + afu->flags &= ~(CFLSH_AFU_HALTED|CFLSH_AFU_RECOV); pthread_rc = pthread_cond_broadcast(&(afu->resume_event)); @@ -3873,26 +4679,30 @@ void cblk_check_os_adap_err_failure_cleanup(cflsh_chunk_t *chunk, cflsh_afu_t *a * chunk - Chunk the cmd is associated. * * RETURNS: - * None + * status on Recovery * * */ -void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) +cflash_block_check_os_status_t cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) { int rc = 0; + cflash_block_check_os_status_t status = CFLSH_BLK_CHK_OS_NO_RESET; cflsh_afu_t *afu; cflsh_path_t *path; int tmp_path_index; cflsh_chunk_t *tmp_chunk; dk_capi_recover_context_t disk_recover; int pthread_rc = 0; +#ifndef _AIX + int old_adap_fd; +#endif void *old_mmio; size_t old_mmio_size; if (CBLK_INVALID_CHUNK_PATH_AFU(chunk,path_index,__FUNCTION__)) { - return; + return status; } afu = chunk->path[path_index]->afu; @@ -3914,7 +4724,7 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) CBLK_TRACE_LOG_FILE(9,"AFU recovery is already active, for chunk->index = %d, chunk->dev_name = %s, path_index = %d,chunk->flags = 0x%x", chunk->index,chunk->dev_name,path_index,chunk->flags); - return; + return CFLSH_BLK_CHK_OS_RESET_PEND; } else { @@ -3987,7 +4797,7 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) } - return; + return CFLSH_BLK_CHK_OS_RESET; } @@ -3998,7 +4808,7 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) */ - afu->flags |= CFLSH_AFU_HALTED; + afu->flags |= CFLSH_AFU_HALTED|CFLSH_AFU_RECOV; path = afu->head_path; @@ -4056,7 +4866,7 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) CBLK_TRACE_LOG_FILE(5,"DK_CAPI_RECOVER initiated, for chunk->index = %d, chunk->dev_name = %s, path_index = %d,chunk->flags = 0x%x", chunk->index,chunk->dev_name,path_index,chunk->flags); - CBLK_TRACE_LOG_FILE(5,"DK_CAPI_RECOVER reattached new old afu->contxt_id = 0x%llx, old_adap_fd = %d", + CBLK_TRACE_LOG_FILE(5,"DK_CAPI_RECOVER old afu->contxt_id = 0x%llx, old_adap_fd = %d", chunk->path[path_index]->afu->contxt_id,chunk->path[path_index]->afu->poll_fd); @@ -4064,7 +4874,8 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) chunk->path[path_index]->afu->mmio,chunk->path[path_index]->afu->mmap_size); - CBLK_TRACE_LOG_FILE(5,"DK_CAPI_RECOVER num_active_cmds = %d",chunk->num_active_cmds); + CBLK_TRACE_LOG_FILE(5,"DK_CAPI_RECOVER num_active_cmds = %d, pid = 0x%llx", + chunk->num_active_cmds,(uint64_t)cflsh_blk.caller_pid); @@ -4079,7 +4890,7 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) - rc = ioctl(chunk->fd,DK_CAPI_RECOVER_CTX,&disk_recover); + rc = ioctl(chunk->path[path_index]->fd,DK_CAPI_RECOVER_CTX,&disk_recover); if (rc) { @@ -4107,7 +4918,7 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) cblk_check_os_adap_err_failure_cleanup(chunk,afu); CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); - return; + return status; } else { @@ -4119,6 +4930,9 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) * We need to process the updated information from this */ + + status = CFLSH_BLK_CHK_OS_RESET_SUCC; + path = afu->head_path; CFLASH_BLOCK_UNLOCK(afu->lock); @@ -4191,12 +5005,17 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) + CBLK_TRACE_LOG_FILE(5,"DK_CAPI_RECOVER reattached pid = 0x%llx", + (uint64_t)cflsh_blk.caller_pid); + #else old_mmio = chunk->path[path_index]->afu->mmio_mmap; + old_adap_fd = chunk->path[path_index]->afu->poll_fd; + old_mmio_size = chunk->path[path_index]->afu->mmap_size; chunk->path[path_index]->afu->contxt_id = disk_recover.context_id; @@ -4213,6 +5032,33 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) CBLK_TRACE_LOG_FILE(5,"DK_CAPI_RECOVER reattached new afu->contxt_id = 0x%llx, new_adap_fd = %d", chunk->path[path_index]->afu->contxt_id,chunk->path[path_index]->afu->poll_fd); +#ifdef DK_CXLFLASH_CONTEXT_SQ_CMD_MODE + if (disk_recover.return_flags & DK_CXLFLASH_CONTEXT_SQ_CMD_MODE) { + + + /* + * driver is indicating this AFU supports SQ. See if this matches + * our original setting. + */ + + if (!(chunk->path[path_index]->afu->flags & CFLSH_AFU_SQ)) { + + /* + * The driver is now indicating the AFU supports SQ, but + * we were not set up for that. So fail this. + */ + + + cblk_check_os_adap_err_failure_cleanup(chunk,afu); + + CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); + return CFLSH_BLK_CHK_OS_RESET_FAIL; + } + + } +#endif /*DK_CXLFLASH_CONTEXT_SQ_CMD_MODE */ + + /* * We do not need to unmap the old MMIO space, since the recover adapter ioctl did this @@ -4240,7 +5086,7 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) cblk_check_os_adap_err_failure_cleanup(chunk,afu); CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); - return; + return CFLSH_BLK_CHK_OS_RESET_FAIL; } @@ -4273,9 +5119,13 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) CBLK_TRACE_LOG_FILE(5,"DK_CAPI_RECOVER mmio = 0x%llx, mmio_size = 0x%llx", chunk->path[path_index]->afu->mmio,chunk->path[path_index]->afu->mmap_size); - - - + + + if (chunk->path[path_index]->flags & CFLSH_PATH_CLOSE_POLL_FD) { + + close(old_adap_fd); + } + #endif /* !AIX */ @@ -4313,7 +5163,7 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) cblk_check_os_adap_err_failure_cleanup(chunk,afu); CFLASH_BLOCK_RWUNLOCK(cflsh_blk.global_lock); - return; + return CFLSH_BLK_CHK_OS_RESET_FAIL; } @@ -4333,8 +5183,23 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) chunk->path[path_index]->afu->toggle = 1; - bzero((void *)chunk->path[path_index]->afu->p_hrrq_start , - (sizeof(*(chunk->path[path_index]->afu->p_hrrq_start)) * chunk->path[path_index]->afu->num_rrqs)); + + bzero((void *)chunk->path[path_index]->afu->p_hrrq_start ,chunk->path[path_index]->afu->size_rrq); + + + if (chunk->path[path_index]->afu->flags & CFLSH_AFU_SQ) { + + /* + * If this AFU is using a Submission Queue (SQ) + * then reinitialize the SQ. + */ + bzero((void *)chunk->path[path_index]->afu->p_sq_start ,chunk->path[path_index]->afu->size_sq); + + chunk->path[path_index]->afu->p_sq_curr = chunk->path[path_index]->afu->p_sq_start; + + + } + } @@ -4346,7 +5211,7 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) * Ensure the CFLSH_AFU_HALTED is cleared */ - afu->flags &= ~CFLSH_AFU_HALTED; + afu->flags &= ~(CFLSH_AFU_HALTED|CFLSH_AFU_RECOV); pthread_rc = pthread_cond_broadcast(&(afu->resume_event)); @@ -4429,7 +5294,7 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index) chunk->flags &= ~CFLSH_CHNK_RECOV_AFU; - return; + return status; } #ifdef _AIX @@ -4810,7 +5675,7 @@ void cblk_notify_mc_err(cflsh_chunk_t *chunk, int path_index,int error_num, chunk->dev_name,reason,cflsh_blk.process_name); - rc = ioctl(chunk->fd,DK_CAPI_LOG_EVENT,&disk_log); + rc = ioctl(chunk->path[path_index]->fd,DK_CAPI_LOG_EVENT,&disk_log); if (rc) { @@ -4897,7 +5762,15 @@ int cblk_verify_mc_lun(cflsh_chunk_t *chunk, cflash_block_notify_reason_t reaso CBLK_TRACE_LOG_FILE(5,"Issuing DK_CAPI_VERIFY for chunk index = %d\n", chunk->index); - rc = ioctl(chunk->fd,DK_CAPI_VERIFY,&disk_verify); + /* + * NOTE: For linux we only do the verify on the path + * that saw the error. It is assumed the other paths + * would either see the same error (such as Unit Attention) + * and a do a verify for that path at that time or the other paths + * would get an event notification. + */ + + rc = ioctl(chunk->path[chunk->cur_path]->fd,DK_CAPI_VERIFY,&disk_verify); if (rc) { diff --git a/src/block/cflash_block_linux.c b/src/block/cflash_block_linux.c index e69a054d..62c165b4 100644 --- a/src/block/cflash_block_linux.c +++ b/src/block/cflash_block_linux.c @@ -276,7 +276,7 @@ int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup } - path = cblk_get_path(chunk,0,chunk_type,chunk->num_cmds,&in_use, share); + path = cblk_get_path(chunk,0,NULL,chunk_type,chunk->num_cmds,&in_use, share); if (path == NULL) { @@ -1850,14 +1850,14 @@ int cblk_read_os_specific_intrpt_event(cflsh_chunk_t *chunk, int path_index,cfls * chunk - Chunk the cmd is associated. * * RETURNS: - * None + * status * * */ -void cblk_check_os_adap_err(cflsh_chunk_t *chunk,int path_index) +cflash_block_check_os_status_t cblk_check_os_adap_err(cflsh_chunk_t *chunk,int path_index) { int rc = 0; - + cflash_block_check_os_status_t status = CFLSH_BLK_CHK_OS_NO_RESET; chunk->stats.num_capi_adap_chck_err++; @@ -1876,7 +1876,7 @@ void cblk_check_os_adap_err(cflsh_chunk_t *chunk,int path_index) cblk_notify_mc_err(chunk,path_index,0x403,0,CFLSH_BLK_NOTIFY_AFU_FREEZE,NULL); - return; + return status; } /* diff --git a/src/block/cflash_block_protos.h b/src/block/cflash_block_protos.h index bee21c3c..45a1da28 100644 --- a/src/block/cflash_block_protos.h +++ b/src/block/cflash_block_protos.h @@ -36,7 +36,9 @@ void *cblk_async_recv_thread(void *data); #ifdef _REMOVE void cblk_wait_for_read_alarm(int signum, siginfo_t *siginfo, void *uctx); #endif /* _REMOVE */ +void cblk_get_host_type(void); void cblk_chunk_sigsev_handler (int signum, siginfo_t *siginfo, void *uctx); +void cblk_reopen_after_fork(cflsh_chunk_t *chunk, int flags); void cblk_prepare_fork (void); void cblk_parent_post_fork (void); void cblk_child_post_fork (void); @@ -46,6 +48,10 @@ void cblk_chunk_flush_cache (cflsh_chunk_t *chunk); void cblk_init_mc_interface(void); void cblk_cleanup_mc_interface(void); char *cblk_find_parent_dev(char *device_name); +char *cblk_get_udid(char *device_name); +int cblk_chunk_attach_process_map_path(cflsh_chunk_t *chunk, int path_index,int mode, + dev64_t adap_devno, char *path_dev_name, int fd, int arch_type, + int *cleanup_depth, int assign_path); int cblk_chunk_attach_process_map (cflsh_chunk_t *chunk, int mode, int *cleanup_depth); int cblk_chunk_get_mc_device_resources(cflsh_chunk_t *chunk, int *cleanup_depth); void cblk_chunk_free_mc_device_resources(cflsh_chunk_t *chunk); @@ -54,7 +60,7 @@ void cblk_chunk_detach (cflsh_chunk_t *chunk,int force); void cblk_setup_trace_files(int new_process); int cblk_valid_endianess(void); -cflsh_path_t *cblk_get_path(cflsh_chunk_t *chunk, dev64_t adap_devno,cflsh_block_chunk_type_t type,int num_cmds, +cflsh_path_t *cblk_get_path(cflsh_chunk_t *chunk, dev64_t adap_devno,char *path_name,cflsh_block_chunk_type_t type,int num_cmds, cflsh_afu_in_use_t *in_use, int share); int cblk_update_path_type(cflsh_chunk_t *chunk, cflsh_path_t *path, cflsh_block_chunk_type_t type); void cblk_release_path(cflsh_chunk_t *chunk, cflsh_path_t *path); @@ -92,7 +98,8 @@ void cblk_dump_debug_data(const char *reason, const char *reason_filename,const int, const char *reason_date); int cblk_setup_sigusr1_dump(void); int cblk_setup_sigsev_dump(void); - +int cblk_query_ue(void *buf, size_t len); +void cblk_check_cmd_data_buf_for_ue(cflsh_chunk_t *chunk,cflsh_cmd_mgm_t *cmd); void cblk_clear_poison_bits(void *buf, size_t len); void cblk_clear_poison_bits_chunk(cflsh_chunk_t *chunk, int path_index, int all_paths); @@ -105,7 +112,7 @@ int cblk_read_os_specific_intrpt_event(cflsh_chunk_t *chunk, int path_index,cfls size_t *transfer_size, struct pollfd poll_list[]); int cblk_chunk_set_mc_size(cflsh_chunk_t *chunk, size_t nblocks); int cblk_mc_clone(cflsh_chunk_t *chunk, int mode,int flags); -void cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index); +cflash_block_check_os_status_t cblk_check_os_adap_err(cflsh_chunk_t *chunk, int path_index); void cblk_notify_mc_err(cflsh_chunk_t *chunk, int path_index,int error_num, uint64_t out_rc,cflash_block_notify_reason_t reason, cflsh_cmd_mgm_t *cmd); @@ -116,4 +123,47 @@ int cblk_verify_mc_lun(cflsh_chunk_t *chunk, cflash_block_notify_reason_t reaso /* cflash_block_sisl.c protos */ int cblk_init_sisl_fcn_ptrs(cflsh_path_t *path); + + +/* cflash_block_raid0.c protos */ + +chunk_r0_id_t cblk_r0_open(const char *pdevs, + int max_num_requests, + int mode, + chunk_ext_arg_t ext, + int flags); +int cblk_r0_close(chunk_r0_id_t id, int flags); +int cblk_r0_get_stats(chunk_r0_id_t id, chunk_stats_t *stats, int flags); +int cblk_r0_get_lun_size(chunk_r0_id_t id, size_t *nblocks); +int cblk_r0_get_size(chunk_r0_id_t id, size_t *nblocks); +int cblk_r0_set_size(chunk_r0_id_t id, size_t nblocks); +int cblk_r0_read(chunk_r0_id_t id, + void *buf, + cflash_offset_t lba, + size_t nblocks, + int flags); +int cblk_r0_write(chunk_r0_id_t id, + void *buf, + cflash_offset_t lba, + size_t nblocks, + int flags); +int cblk_r0_aread(chunk_r0_id_t id, + void *buf, + cflash_offset_t lba, + size_t nblocks, + cflsh_cg_tag_t *tag, + cblk_arw_status_t *status, + int flags); +int cblk_r0_awrite(chunk_r0_id_t id, + void *buf, + cflash_offset_t lba, + size_t nblocks, + cflsh_cg_tag_t *tag, + cblk_arw_status_t *status, + int flags); +int cblk_r0_aresult(chunk_r0_id_t id, + cflsh_cg_tag_t *tag, + uint64_t *status, + int flags); + #endif /* _H_CFLASH_BLOCK_PROTO */ diff --git a/src/block/cflash_block_raid0.c b/src/block/cflash_block_raid0.c new file mode 100644 index 00000000..d7c4bf33 --- /dev/null +++ b/src/block/cflash_block_raid0.c @@ -0,0 +1,421 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/block/cflash_block_raid0.c $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ + +#include +#include + +#define CFLSH_BLK_FILENUM 0x0800 +#define CFLASH_R0_MAX_DEV 8 +#define CFLASH_R0_MAX_BUF 32 +#define CFLASH_R0_CMD_TO 50 + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +chunk_r0_id_t cblk_r0_open(const char *pdevs, + int max_num_requests, + int mode, + chunk_ext_arg_t ext, + int flags) +{ + char devstr[CFLASH_R0_MAX_DEV][CFLASH_R0_MAX_BUF]; + cflash_block_grp_fd_t *pgrp = NULL; + int devN = 0; + chunk_r0_id_t rid = 0; + char *pstr = NULL; + char *dbuf = NULL; + char buf[2048] = {0}; + int i,j; + + memset(devstr,0,sizeof(devstr)); + + if (pdevs) + { + CBLK_TRACE_LOG_FILE(2, "pdevs:%s QD:%d flags:%x", + pdevs, max_num_requests, flags); + } + + flags &= ~CBLK_GROUP_RAID0; + + /*-------------------------------------------------------------------------- + * parse the colon separated /dev/sgX:... + *------------------------------------------------------------------------*/ + if (pdevs && strncmp(pdevs,"/dev",4)==0) + { + + dbuf = strdup(pdevs); + sprintf(dbuf,"%s",pdevs); + while ((pstr=strsep((&dbuf),":"))) + { + strcat(buf, pstr); strcat(buf, " "); + sprintf(devstr[devN],"%s",pstr); + if (++devN == CFLASH_R0_MAX_DEV) {break;} + } + CBLK_TRACE_LOG_FILE(2, "parsed RAID0 (%s)", buf); + } + +#ifndef _AIX + /*-------------------------------------------------------------------------- + * RAID0 is passed in, use a script to gather a list of devices + * -parse RAID0:N, RAID0:* + *------------------------------------------------------------------------*/ + else if (pdevs && strncmp(pdevs,"RAID0",5)==0) + { + FILE *fp = NULL; + char device[CFLASH_R0_MAX_BUF] = {0}; + char devNStr[3] = {0}; + int devN_in = 0; + int wildcard = 1; + + if (strlen(pdevs) > 6 && pdevs[5] == ':' && pdevs[6] != '*') + { + wildcard = 0; + devNStr[0] = pdevs[6]; + devNStr[1] = pdevs[7]; + devN_in = atoi(devNStr); + } + fp = popen("/opt/ibm/capikv/afu/cflash_devices.pl", "r"); + if (!fp) + { + CBLK_TRACE_LOG_FILE(1, "cflash_devices.pl failed"); + errno = ENODEV; + goto error_devs; + } + /* read in devices, store in devstr */ + while (fgets(device, CFLASH_R0_MAX_BUF, fp)) + { + device[strlen(device)-1] = '\0'; + strcat(buf, device); strcat(buf, " "); + memset(devstr[devN],0,CFLASH_R0_MAX_BUF); + strcpy(devstr[devN], device); + memset(device,0,CFLASH_R0_MAX_BUF); + if (++devN == CFLASH_R0_MAX_DEV) {break;} + } + fclose(fp); + if (devN>1) {CBLK_TRACE_LOG_FILE(1, "RAID0: (%s)", buf);} + + /* RAID0:N is passed, ensure the number of devices found matches */ + if (!wildcard && devN != devN_in) + { + CBLK_TRACE_LOG_FILE(1, "RAID0 devs found(%d) does not match " + "the input(%s) num(%s<=>%d)", + devN, pdevs, devNStr, devN_in); + errno = ENODEV; + goto error_devs; + } + } +#endif /* !_AIX */ + + /*-------------------------------------------------------------------------- + * use whatever this is + *------------------------------------------------------------------------*/ + else if (pdevs) + { + CBLK_TRACE_LOG_FILE(2, "using device(%s)", pdevs); + strcpy(devstr[0], pdevs); + devN = 1; + } + /*-------------------------------------------------------------------------- + * use NULL + *------------------------------------------------------------------------*/ + else + { + CBLK_TRACE_LOG_FILE(2, "using devstr=NULL"); + devN = 1; + } + + if (devN == 0) + { + errno = ENODEV; + CBLK_TRACE_LOG_FILE(1,"ERROR: devN = 0, errno:%d", errno); + goto error_devs; + } + + if (cblk_cg_alloc(&rid, devN, &pgrp)) {goto error_devs;} + if (pgrp == NULL) {goto error_devs;} + + pgrp->flags |= CBLK_GROUP_RAID0; + + /*-------------------------------------------------------------------------- + * open devices + *------------------------------------------------------------------------*/ + for (i=0; inum_chunks; i++) + { + pgrp->chunk_list[i].id = cblk_open(devstr[i], max_num_requests, mode, + ext, flags); + if (pgrp->chunk_list[i].id == NULL_CHUNK_ID) + { + CBLK_TRACE_LOG_FILE(1, "error opening(%s)", devstr[i]); + for (j=0; j < i; j++) {cblk_close(pgrp->chunk_list[j].id,0);} + goto error_devs; + } + CBLK_TRACE_LOG_FILE(4, "new raid0 %d:%d", rid,pgrp->chunk_list[i].id); + } + + errno=0; + goto done; + +error_devs: + rid = NULL_CHUNK_CG_ID; + +done: + CBLK_TRACE_LOG_FILE(9,"rid:%d devN:%d", rid, devN); + return rid; +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +int cblk_r0_set_size(chunk_r0_id_t id, size_t blocksN) +{ + cflash_block_grp_fd_t *pgrp = NULL; + int rc = 0; + size_t blks = 0; + int i = 0; + + pgrp = CBLK_GET_CG_HASH(id); + if (!pgrp || !blocksN) {errno=EINVAL; rc=-1; goto done;} + + blks = blocksN / pgrp->num_chunks; + + /* if blocksN is not an even stripe, alloc an extra block per chunk */ + if (blocksN % pgrp->num_chunks) {++blks;} + + for (i=0; inum_chunks;i++) + { + rc = cblk_set_size(pgrp->chunk_list[i].id, blks, 0); + if (rc) {goto done;} + } + +done: + if (rc<0) {CBLK_TRACE_LOG_FILE(2,"FFDC id:%d rc:%d", id, rc);} + return rc; +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +int cblk_r0_read(chunk_r0_id_t id, + void *buf, + cflash_offset_t lba, + size_t nblocks, + int flags) +{ + cflash_block_grp_fd_t *pgrp = NULL; + chunk_id_t cid = 0; + uint32_t dev = 0; + cflash_offset_t clba = 0; + int rc = 0; + + if (NULL==(pgrp=CBLK_GET_CG_HASH(id))) + {errno=EINVAL; rc=-1; goto done;} + + flags &= ~CBLK_GROUP_RAID0; + dev = (uint32_t)lba % pgrp->num_chunks; + cid = pgrp->chunk_list[dev].id; + clba = lba/pgrp->num_chunks; + + rc = cblk_read(cid, buf, clba, nblocks, flags); + +done: + if (rc<0) {CBLK_TRACE_LOG_FILE(2,"FFDC id:%d rc:%d", id, rc);} + return rc; +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +int cblk_r0_write(chunk_r0_id_t id, + void *pbuf, + cflash_offset_t lba, + size_t nblocks, + int flags) +{ + cflash_block_grp_fd_t *pgrp = NULL; + chunk_id_t cid = 0; + uint32_t dev = 0; + cflash_offset_t clba = 0; + int rc = 0; + + if (NULL==(pgrp=CBLK_GET_CG_HASH(id))) + {errno=EINVAL; rc=-1; goto done;} + + flags &= ~CBLK_GROUP_RAID0; + dev = (uint32_t)lba % pgrp->num_chunks; + cid = pgrp->chunk_list[dev].id; + clba = lba/pgrp->num_chunks; + + rc = cblk_write(cid, pbuf, clba, nblocks, flags); + +done: + if (rc<0) {CBLK_TRACE_LOG_FILE(2,"FFDC id:%d rc:%d", id, rc);} + return rc; +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +int cblk_r0_aread(chunk_r0_id_t id, + void *buf, + cflash_offset_t lba, + size_t nblocks, + cflsh_cg_tag_t *tag, + cblk_arw_status_t *status, + int flags) +{ + cflash_block_grp_fd_t *pgrp = NULL; + chunk_id_t cid = 0; + uint32_t dev = 0; + cflash_offset_t clba = 0; + int ctag = 0; + int rc = 0; + + if (nblocks>1) {errno=EINVAL; rc=-1; goto done;} + + pgrp = CBLK_GET_CG_HASH(id); + if (!pgrp) {errno=EINVAL; rc=-1; goto done;} + if (!buf || !tag) {errno=EINVAL; rc=-1; goto done;} + + flags &= ~CBLK_GROUP_RAID0; + dev = (uint32_t)lba % pgrp->num_chunks; + cid = pgrp->chunk_list[dev].id; + clba = lba/pgrp->num_chunks; + tag->id = cid; + + rc = cblk_aread(cid, buf, clba, nblocks, &ctag, status, flags); + if (rc >= 0) {tag->tag = ctag;} + else {tag->tag = -1;} + +done: + if (rc<0) {CBLK_TRACE_LOG_FILE(2,"FFDC id:%d rc:%d", id, rc);} + return rc; +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +int cblk_r0_awrite(chunk_r0_id_t id, + void *buf, + cflash_offset_t lba, + size_t nblocks, + cflsh_cg_tag_t *tag, + cblk_arw_status_t *status, + int flags) +{ + cflash_block_grp_fd_t *pgrp = NULL; + chunk_id_t cid = 0; + uint32_t dev = 0; + cflash_offset_t clba = 0; + int ctag = 0; + int rc = 0; + + if (nblocks>1) {errno=EINVAL; rc=-1; goto done;} + + pgrp = CBLK_GET_CG_HASH(id); + if (!pgrp) {errno=EINVAL; rc=-1; goto done;} + if (!buf || !tag) {errno=EINVAL; rc=-1; goto done;} + + flags &= ~CBLK_GROUP_RAID0; + dev = (uint32_t)lba % pgrp->num_chunks; + cid = pgrp->chunk_list[dev].id; + clba = lba/pgrp->num_chunks; + tag->id = cid; + + rc = cblk_awrite(cid, buf, clba, nblocks, &ctag, status, flags); + if (rc >= 0) {tag->tag = ctag;} + else {tag->tag = -1;} + +done: + if (rc<0) {CBLK_TRACE_LOG_FILE(2,"FFDC id:%d rc:%d", id, rc);} + return rc; +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +int cblk_r0_aresult(chunk_r0_id_t id, + cflsh_cg_tag_t *ptag, + uint64_t *status, + int flags) +{ + cflash_block_grp_fd_t *pgrp = NULL; + int rc = 0; + int i = 0; + + flags &= ~CBLK_GROUP_RAID0; + + pgrp = CBLK_GET_CG_HASH(id); + if (!pgrp) {errno=EINVAL; rc=-1; goto done;} + if (!ptag) {errno=EINVAL; rc=-1; goto done;} + + if (flags & CBLK_ARESULT_NEXT_TAG) + { + int tflags = flags & ~CBLK_ARESULT_BLOCKING; + + /* if not blocking, check each device for a completed op */ + for (i=0; inum_chunks; i++) + { + ptag->id = cblk_cg_get_next_chunk_id(id); + rc = cblk_aresult(ptag->id, &ptag->tag,status,tflags); + if (rc>0) {goto done;} + if (rc<0 && errno==ENOENT) {rc=0;} + } + + /* if blocking, then loop until timeout or completed op */ + uint64_t start = getticks(); + while (flags & CBLK_ARESULT_BLOCKING) + { + if (SDELTA(start,cflsh_blk.nspt) >= CFLASH_R0_CMD_TO) + { + CBLK_TRACE_LOG_FILE(1,"TIMEOUT: id:%d", id); + rc=-1; errno=ETIME; goto done; + } + + /* wait for a cmd to complete */ + for (i=0; inum_chunks; i++) + { + ptag->id = cblk_cg_get_next_chunk_id(id); + rc = cblk_aresult(ptag->id, &ptag->tag, status, tflags); + if (rc>0) {goto done;} + if (rc<0 && errno==ENOENT) {rc=0;} + } + } + } + else + { + rc = cblk_aresult(ptag->id, &ptag->tag, status, flags); + } + +done: + if (rc<0) {CBLK_TRACE_LOG_FILE(2,"FFDC id:%d rc:%d", id, rc);} + return rc; +} diff --git a/src/block/cflash_block_sisl.c b/src/block/cflash_block_sisl.c index ebafddb0..6b45e5da 100644 --- a/src/block/cflash_block_sisl.c +++ b/src/block/cflash_block_sisl.c @@ -128,9 +128,30 @@ static inline void CBLK_OUT_MMIO_REG(volatile __u64 *addr, __u64 val) int cblk_get_sisl_num_interrupts(cflsh_chunk_t *chunk, int path_index) { +#ifndef _AIX + if (cflsh_blk.host_type == CFLASH_HOST_PHYP) { + + /* + * For linux pHyp the number of interrupts specified + * by user space should not include the page fault + * (PSL) handler interrupt). Thus we must substract 1. + * For AIX or BML, the define CFLSH_BLK_SISL_NUM_INTERRUPTS + * is the correct number of interrupts to use. + */ + + CBLK_TRACE_LOG_FILE(9,"This is a pHyp system, num interrupts = 0x%x", + (CFLSH_BLK_SISL_NUM_INTERRUPTS-1)); + + return (CFLSH_BLK_SISL_NUM_INTERRUPTS-1); + + + } + +#endif /* !_AIX */ return CFLSH_BLK_SISL_NUM_INTERRUPTS; + } @@ -162,6 +183,11 @@ uint64_t cblk_get_sisl_cmd_room(cflsh_chunk_t *chunk, int path_index) cmd_room = chunk->path[path_index]->afu->cmd_room; #else + if (chunk->path[path_index]->afu->flags & CFLSH_AFU_SQ) { + + CBLK_TRACE_LOG_FILE(1,"Attempt to read command room, but this AFU is using SQ"); + return 0; + } if (chunk->path[path_index]->afu->cmd_room) { @@ -174,7 +200,8 @@ uint64_t cblk_get_sisl_cmd_room(cflsh_chunk_t *chunk, int path_index) chunk->path[path_index]->afu->cmd_room = CBLK_IN_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_CMD_ROOM_OFFSET,FALSE); - CBLK_TRACE_LOG_FILE(9,"command room mmio is 0x%llx",chunk->path[path_index]->afu->cmd_room); + CBLK_TRACE_LOG_FILE(9,"command room mmio is 0x%llx for path_index %d", + chunk->path[path_index]->afu->cmd_room,path_index); cmd_room = chunk->path[path_index]->afu->cmd_room; @@ -308,6 +335,89 @@ void cblk_inc_sisl_rrq(cflsh_chunk_t *chunk, int path_index) return; } +/* + * NAME: cblk_inc_sisl_sq + * + * FUNCTION: This routine is called whenever an SQ entry has been issued + * + * + * NOTE; This routine assumes the caller is holding chunk->lock. + * + * RETURNS: None + * + * + */ + +void cblk_inc_sisl_sq(cflsh_chunk_t *chunk, int path_index) +{ + + + chunk->path[path_index]->afu->p_sq_curr++; + + + + if (chunk->path[path_index]->afu->p_sq_curr > chunk->path[path_index]->afu->p_sq_end) + { + + chunk->path[path_index]->afu->p_sq_curr = chunk->path[path_index]->afu->p_sq_start; + + + } + + + + return; +} + + +/* + * NAME: cblk_sisl_adap_sq_setup + * + * FUNCTION: This routine is called to set up the adapter to + * recognize an SQ + * + * + * NOTE; This routine assumes the caller is holding chunk->lock. + * + * RETURNS: 0 - success, otherwise failure + * + * + * + */ + +int cblk_sisl_adap_sq_setup(cflsh_chunk_t *chunk, int path_index) +{ + int rc = 0; + + + if (chunk->path[path_index]->afu->mmio == NULL) { + + CBLK_TRACE_LOG_FILE(1,"mmio is not valid for chunk index = %d, and path_index = %d", + chunk->index,path_index); + + return -1; + } + + if (!(chunk->path[path_index]->afu->flags & CFLSH_AFU_SQ)) { + + CBLK_TRACE_LOG_FILE(1,"this AFU does not have SQ enabled afu->flags 0x%x, chunk index = %d, and path_index = %d", + chunk->path[path_index]->afu->flags,chunk->index,path_index); + + return -1; + + } + + CBLK_OUT_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_SQ_START_EA_OFFSET, + (uint64_t)chunk->path[path_index]->afu->p_sq_start); + + + CBLK_OUT_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_SQ_END_EA_OFFSET, + (uint64_t)chunk->path[path_index]->afu->p_sq_end); + + return rc; +} + + /* * NAME: cblk_sisl_adap_setup * @@ -317,8 +427,8 @@ void cblk_inc_sisl_rrq(cflsh_chunk_t *chunk, int path_index) * * NOTE; This routine assumes the caller is holding chunk->lock. * - * RETURNS: The number of commands that can currently be issued to the AFU - * for this context. + * RETURNS: 0 - success, otherwise failure + * * * */ @@ -369,10 +479,21 @@ int cblk_sisl_adap_setup(cflsh_chunk_t *chunk, int path_index) - CBLK_OUT_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_RRQ0_START_EA_OFFSET, (uint64_t)chunk->path[path_index]->afu->p_hrrq_start); + CBLK_OUT_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_RRQ0_START_EA_OFFSET, + (uint64_t)chunk->path[path_index]->afu->p_hrrq_start); - CBLK_OUT_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_RRQ0_END_EA_OFFSET, (uint64_t)chunk->path[path_index]->afu->p_hrrq_end); + CBLK_OUT_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_RRQ0_END_EA_OFFSET, + (uint64_t)chunk->path[path_index]->afu->p_hrrq_end); + + + if (chunk->path[path_index]->afu->flags & CFLSH_AFU_SQ) { + + + + rc = cblk_sisl_adap_sq_setup(chunk,path_index); + + } /* @@ -458,14 +579,59 @@ scsi_cdb_t * cblk_get_sisl_cmd_cdb(cflsh_chunk_t *chunk, cflsh_cmd_mgm_t *cmd) */ cflsh_cmd_mgm_t *cblk_get_sisl_cmd_rsp(cflsh_chunk_t *chunk,int path_index) { + uint64_t value; #ifdef _AIX uint32_t tmp_val; #endif /* AIX */ cflsh_cmd_mgm_t *cmd = NULL; + + value = CBLK_READ_ADDR_CHK_UE(chunk->path[path_index]->afu->p_hrrq_curr); + + + if (value == 0xffffffffffffffffLL) { + + CBLK_TRACE_LOG_FILE(1,"Potential UE encountered for RRQ\n"); + + + cblk_check_os_adap_err(chunk,path_index); + + /* + * Tell caller there is no valid command + */ + + return NULL; + } + + if (chunk->path[path_index]->afu->hrrq_to_ioarcb_off) { + + /* + * If the afu's hrrq_to_ioarcb_off field is set + * then it indicates the value returned in the RRQ + * is not the IOARCB (and thus not the command, since + * the IOARCB is at the beginning of the command). Thus + * this hrrq_to_ioarcb_off field must be subtracted from + * the value returned to get that offset. + * + * NOTE: This code assumes hrrq_to_ioarcb_off has the low order + * bit (corresponding to the toggle bit) off. + */ + + if (value <= chunk->path[path_index]->afu->hrrq_to_ioarcb_off) { + + CBLK_TRACE_LOG_FILE(1,"hrrq_to_ioarcb_off 0x%llx is larger than rrq value 0x%llx\n", + chunk->path[path_index]->afu->hrrq_to_ioarcb_off, value); + return NULL; + } + + value -= chunk->path[path_index]->afu->hrrq_to_ioarcb_off; + } + + + #if !defined(__64BIT__) && defined(_AIX) - if ((*(chunk->path[path_index]->afu->p_hrrq_curr)) & 0xffffffff00000000LL) { + if ((value) & 0xffffffff00000000LL) { /* * This is not valid for 32-bit application. @@ -473,7 +639,7 @@ cflsh_cmd_mgm_t *cblk_get_sisl_cmd_rsp(cflsh_chunk_t *chunk,int path_index) CBLK_TRACE_LOG_FILE(1,"Invalid cmd pointer received by AFU = 0x%llx p_hrrq_curr = 0x%llx, chunk->index = %d", (uint64_t)chunk->path[path_index]->afu->p_hrrq_curr,chunk->index); - cblk_notify_mc_err(chunk,path_index,0x604,(*(chunk->path[path_index]->afu->p_hrrq_curr)), + cblk_notify_mc_err(chunk,path_index,0x604,value, CFLSH_BLK_NOTIFY_AFU_ERROR,NULL); CBLK_LIVE_DUMP_THRESHOLD(1,"0x604"); @@ -482,15 +648,17 @@ cflsh_cmd_mgm_t *cblk_get_sisl_cmd_rsp(cflsh_chunk_t *chunk,int path_index) * clear upper word only. */ + //?? What about UE check on store?? + *(chunk->path[path_index]->afu->p_hrrq_curr) &= *(chunk->path[path_index]->afu->p_hrrq_curr) & 0x00000000ffffffffLL; } - tmp_val = ((uint32_t)(*(chunk->path[path_index]->afu->p_hrrq_curr)) & 0xffffffff) & (~SISL_RESP_HANDLE_T_BIT); + tmp_val = ((uint32_t)(value) & 0xffffffff) & (~SISL_RESP_HANDLE_T_BIT); cmd = (cflsh_cmd_mgm_t *)tmp_val; #else - cmd = (cflsh_cmd_mgm_t *)((*(chunk->path[path_index]->afu->p_hrrq_curr)) & (~SISL_RESP_HANDLE_T_BIT)); + cmd = (cflsh_cmd_mgm_t *)((value) & (~SISL_RESP_HANDLE_T_BIT)); #endif @@ -621,6 +789,17 @@ int cblk_build_sisl_cmd(cflsh_chunk_t *chunk,int path_index, ioarcb->data_ea = (ulong)buf; ioarcb->data_len = buf_len; + + if (chunk->path[path_index]->afu->flags & CFLSH_AFU_SQ) { + + /* + * If we are using a submission queue (SQ), then + * we must fill in the address of the IOASA for + * this IOARCB. + */ + + ioarcb->sq_ioasa_ea = (uint64_t)&(cmd->sisl_cmd.sa); + } return rc; } @@ -723,53 +902,45 @@ int cblk_issue_sisl_cmd(cflsh_chunk_t *chunk, int path_index,cflsh_cmd_mgm_t *cm afu_mmio_write_dw(p_afu, 8, (uint64_t)ioarcb); #else - while ((CBLK_GET_CMD_ROOM(chunk,path_index) == 0) && - (wait_room_retry < CFLASH_BLOCK_MAX_WAIT_ROOM_RETRIES)) { + if (!(chunk->path[chunk->cur_path]->afu->flags & CFLSH_AFU_SQ)) { /* - * Wait a limited amount of time for the room on - * the AFU. Since we are waiting for the AFU - * to fetch some more commands, it is thought - * we can wait a little while here. It should also - * be noted we are not unlocking anything in this wait. - * Since the AFU is not waiting for us to process a command, - * this (not unlocking) may be alright. However it does mean - * other threads are being held off. If they are also trying - * to issue requests, then they would see this same issue. If - * these other threads are trying to process completions, then - * those will be delayed (perhaps unnecessarily). + * If we are not using a Submission Queue (SQ), then + * we need to check command room before attempting + * to issue IOARCBs. */ - CBLK_TRACE_LOG_FILE(5,"waiting for command room"); - usleep(CFLASH_BLOCK_DELAY_ROOM); - - if (chunk->flags & CFLSH_CHUNK_FAIL_IO) { - - errno = EIO; - - return -1; - } - - wait_room_retry++; - } + while ((CBLK_GET_CMD_ROOM(chunk,path_index) == 0) && + (wait_room_retry < CFLASH_BLOCK_MAX_WAIT_ROOM_RETRIES)) { + /* + * Wait a limited amount of time for the room on + * the AFU. Since we are waiting for the AFU + * to fetch some more commands, it is thought + * we can wait a little while here. It should also + * be noted we are not unlocking anything in this wait. + * Since the AFU is not waiting for us to process a command, + * this (not unlocking) may be alright. However it does mean + * other threads are being held off. If they are also trying + * to issue requests, then they would see this same issue. If + * these other threads are trying to process completions, then + * those will be delayed (perhaps unnecessarily). + */ - if (wait_room_retry >= CFLASH_BLOCK_MAX_WAIT_ROOM_RETRIES) { + CBLK_TRACE_LOG_FILE(5,"waiting for command room path_index = %d", + path_index); + usleep(CFLASH_BLOCK_DELAY_ROOM); + if (chunk->flags & CFLSH_CHUNK_FAIL_IO) { + errno = EIO; - /* - * We do not have any room to send this - * command. Fail this operation now. - */ + return -1; + } -#ifdef _FOR_DEBUG - CBLK_CLEANUP_BAD_MMIO_SIGNAL(chunk,path_index); -#endif /* _FOR_DEBUG */ - errno = EBUSY; + wait_room_retry++; + } - cblk_notify_mc_err(chunk,path_index,0x607,wait_room_retry,CFLSH_BLK_NOTIFY_ADAP_ERR,NULL); - return -1; } CFLASH_BLOCK_AFU_SHARE_LOCK(chunk->path[path_index]->afu); @@ -852,6 +1023,37 @@ int cblk_issue_sisl_cmd(cflsh_chunk_t *chunk, int path_index,cflsh_cmd_mgm_t *cm } + if (!(chunk->path[chunk->cur_path]->afu->flags & CFLSH_AFU_SQ)) { + + /* + * If we are not using a Submission Queue (SQ), then + * we need to check command room retries. + */ + + if (wait_room_retry >= CFLASH_BLOCK_MAX_WAIT_ROOM_RETRIES) { + + + + /* + * We do not have any room to send this + * command. Since it is possible that AFU + * may have been halted and we just woke up, + * check command room one more time, before failing + * this request. + */ + + if (CBLK_GET_CMD_ROOM(chunk,path_index) == 0) { +#ifdef _FOR_DEBUG + CBLK_CLEANUP_BAD_MMIO_SIGNAL(chunk,path_index); +#endif /* _FOR_DEBUG */ + errno = EBUSY; + + cblk_notify_mc_err(chunk,path_index,0x607,wait_room_retry,CFLSH_BLK_NOTIFY_ADAP_ERR,NULL); + return -1; + } + } + } + while ((chunk->path[path_index]->afu->num_issued_cmds >= chunk->path[path_index]->afu->num_rrqs) && (wait_rrq_retry < CFLASH_BLOCK_MAX_WAIT_RRQ_RETRIES)) { @@ -917,8 +1119,34 @@ int cblk_issue_sisl_cmd(cflsh_chunk_t *chunk, int path_index,cflsh_cmd_mgm_t *cm chunk->path[path_index]->afu->num_issued_cmds++; - CBLK_OUT_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_IOARRIN_OFFSET, (uint64_t)ioarcb); + if (chunk->path[path_index]->afu->flags & CFLSH_AFU_SQ) { + + /* + * Copy IOARCB to SQ and write. + * ?? TODO: Look at avoiding copy here and building + * directly in IOARCB. + */ + + bcopy(ioarcb,(void *)chunk->path[path_index]->afu->p_sq_curr, + sizeof(*ioarcb)); + + CBLK_INC_SQ(chunk,path_index); + CBLK_OUT_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_SQ_TAIL_EA_OFFSET, + (uint64_t)chunk->path[path_index]->afu->p_sq_curr); + + CBLK_TRACE_LOG_FILE(9,"SQ mmio = 0x%llx, issued command path_index = %d for ioarcb = %p, afu = %p", + (uint64_t)chunk->path[path_index]->afu->mmio_mmap,path_index, + ioarcb,chunk->path[path_index]->afu); + + } else { + CBLK_OUT_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_IOARRIN_OFFSET, (uint64_t)ioarcb); + + CBLK_TRACE_LOG_FILE(9,"mmio = 0x%llx, issued command path_index = %d for ioarcb = %p, afu = %p", + (uint64_t)chunk->path[path_index]->afu->mmio_mmap,path_index, + ioarcb,chunk->path[path_index]->afu); + + } #endif /* !_USE_LIB_AFU */ @@ -1139,6 +1367,7 @@ int cblk_complete_status_sisl_cmd(cflsh_chunk_t *chunk,cflsh_cmd_mgm_t *cmd) int cblk_process_sisl_cmd_intrpt(cflsh_chunk_t *chunk,int path_index,cflsh_cmd_mgm_t **cmd,int *cmd_complete,size_t *transfer_size) { int rc = 0; + int cmd_rc = 0; cflsh_cmd_mgm_t *p_cmd = NULL; @@ -1185,11 +1414,11 @@ int cblk_process_sisl_cmd_intrpt(cflsh_chunk_t *chunk,int path_index,cflsh_cmd_m - CBLK_TRACE_LOG_FILE(7,"*(chunk->path[path_index]->afu->p_hrrq_curr) = 0x%llx, chunk->path[path_index]->afu->toggle = 0x%llx, p_hrrq_curr = 0x%llx, chunk->index = %d", - *(chunk->path[path_index]->afu->p_hrrq_curr),(uint64_t)chunk->path[path_index]->afu->toggle,(uint64_t)chunk->path[path_index]->afu->p_hrrq_curr,chunk->index); + CBLK_TRACE_LOG_FILE(7,"*(chunk->path[%d]->afu->p_hrrq_curr) = 0x%llx, chunk->path[path_index]->afu->toggle = 0x%llx, p_hrrq_curr = 0x%llx, chunk->index = %d",path_index, + CBLK_READ_ADDR_CHK_UE(chunk->path[path_index]->afu->p_hrrq_curr),(uint64_t)chunk->path[path_index]->afu->toggle,(uint64_t)chunk->path[path_index]->afu->p_hrrq_curr,chunk->index); - while (((*(chunk->path[path_index]->afu->p_hrrq_curr)) & (SISL_RESP_HANDLE_T_BIT)) == chunk->path[path_index]->afu->toggle) { + while (((CBLK_READ_ADDR_CHK_UE(chunk->path[path_index]->afu->p_hrrq_curr)) & (SISL_RESP_HANDLE_T_BIT)) == chunk->path[path_index]->afu->toggle) { /* * Process all RRQs that have been posted via this interrupt @@ -1197,8 +1426,9 @@ int cblk_process_sisl_cmd_intrpt(cflsh_chunk_t *chunk,int path_index,cflsh_cmd_m p_cmd = CBLK_GET_CMD_RSP(chunk,path_index); - CBLK_TRACE_LOG_FILE(8,"*(chunk->path[path_index].p_hrrq_curr) = 0x%llx, chunk->path[path_index].toggle = 0x%llx, p_hrrq_curr = 0x%llx, chunk->index = %d", - *(chunk->path[path_index]->afu->p_hrrq_curr),(uint64_t)chunk->path[path_index]->afu->toggle,(uint64_t)chunk->path[path_index]->afu->p_hrrq_curr,chunk->index); + CBLK_TRACE_LOG_FILE(8,"*(chunk->path[%d].p_hrrq_curr) = 0x%llx, chunk->path[path_index].toggle = 0x%llx, p_hrrq_curr = 0x%llx, chunk->index = %d", + path_index, + CBLK_READ_ADDR_CHK_UE(chunk->path[path_index]->afu->p_hrrq_curr),(uint64_t)chunk->path[path_index]->afu->toggle,(uint64_t)chunk->path[path_index]->afu->p_hrrq_curr,chunk->index); @@ -1254,7 +1484,7 @@ int cblk_process_sisl_cmd_intrpt(cflsh_chunk_t *chunk,int path_index,cflsh_cmd_m CBLK_TRACE_LOG_FILE(7,"*(chunk->path[path_index].p_hrrq_curr) = 0x%llx, chunk->path[path_index].toggle = %d, p_hrrq_curr = 0x%llx, chunk->index = %d", - *(chunk->path[path_index]->afu->p_hrrq_curr),chunk->path[path_index]->afu->toggle,(uint64_t)chunk->path[path_index]->afu->p_hrrq_curr,chunk->index); + CBLK_READ_ADDR_CHK_UE(chunk->path[path_index]->afu->p_hrrq_curr),chunk->path[path_index]->afu->toggle,(uint64_t)chunk->path[path_index]->afu->p_hrrq_curr,chunk->index); continue; } @@ -1270,7 +1500,7 @@ int cblk_process_sisl_cmd_intrpt(cflsh_chunk_t *chunk,int path_index,cflsh_cmd_m p_cmd->cmdi->state = CFLSH_MGM_CMP; - rc = cblk_process_cmd(chunk,path_index,p_cmd); + cmd_rc = cblk_process_cmd(chunk,path_index,p_cmd); if ((*cmd == NULL) && (!(*cmd_complete))) { @@ -1296,7 +1526,7 @@ int cblk_process_sisl_cmd_intrpt(cflsh_chunk_t *chunk,int path_index,cflsh_cmd_m CBLK_TRACE_LOG_FILE(9,"command for other chunk *(chunk->path[path_index].p_hrrq_curr) = 0x%llx, chunk->path[path_index].toggle = %d, p_hrrq_curr = 0x%llx, chunk->path[path_index].index = %d", - *(chunk->path[path_index]->afu->p_hrrq_curr),chunk->path[path_index]->afu->toggle,(uint64_t)chunk->path[path_index]->afu->p_hrrq_curr,chunk->index); + CBLK_READ_ADDR_CHK_UE(chunk->path[path_index]->afu->p_hrrq_curr),chunk->path[path_index]->afu->toggle,(uint64_t)chunk->path[path_index]->afu->p_hrrq_curr,chunk->index); if ((chunk->path[path_index] == NULL) || @@ -1328,29 +1558,38 @@ int cblk_process_sisl_cmd_intrpt(cflsh_chunk_t *chunk,int path_index,cflsh_cmd_m */ - if ((*cmd) && - (rc != CFLASH_CMD_RETRY_ERR) && - (rc != CFLASH_CMD_DLY_RETRY_ERR)) { + if (*cmd) { - /* - * Since we found our command completed and - * we are not retrying it, lets - * set the flag so we can avoid polling for any - * more interrupts. However we need to process - * all responses posted to the RRQ for this - * interrupt before exiting. - */ + if ((cmd_rc != CFLASH_CMD_RETRY_ERR) && + (cmd_rc != CFLASH_CMD_DLY_RETRY_ERR)) { + + /* + * Since we found our command completed and + * we are not retrying it, lets + * set the flag so we can avoid polling for any + * more interrupts. However we need to process + * all responses posted to the RRQ for this + * interrupt before exiting. + */ #ifndef _COMMON_INTRPT_THREAD - - CBLK_COMPLETE_CMD(chunk,*cmd,transfer_size); + + CBLK_COMPLETE_CMD(chunk,*cmd,transfer_size); #else + + if (chunk->flags & CFLSH_CHNK_NO_BG_TD) { + CBLK_COMPLETE_CMD(chunk,*cmd,transfer_size); + } + +#endif + *cmd_complete = TRUE; + + if (cmd_rc == CFLASH_CMD_FATAL_ERR) { + + rc = -1; + } + } - if (chunk->flags & CFLSH_CHNK_NO_BG_TD) { - CBLK_COMPLETE_CMD(chunk,*cmd,transfer_size); - } -#endif - *cmd_complete = TRUE; } @@ -1373,7 +1612,7 @@ int cblk_process_sisl_cmd_intrpt(cflsh_chunk_t *chunk,int path_index,cflsh_cmd_m } CBLK_TRACE_LOG_FILE(7,"*(chunk->path[path_index].p_hrrq_curr) = 0x%llx, chunk->path[path_index].toggle = 0x%llx, chunk->index = %d", - *(chunk->path[path_index]->afu->p_hrrq_curr),chunk->path[path_index]->afu->toggle,chunk->index); + CBLK_READ_ADDR_CHK_UE(chunk->path[path_index]->afu->p_hrrq_curr),chunk->path[path_index]->afu->toggle,chunk->index); } /* Inner while loop on RRQ */ @@ -1828,6 +2067,7 @@ cflash_cmd_err_t cblk_process_sisl_cmd_err(cflsh_chunk_t *chunk,int path_index,c CBLK_TRACE_LOG_FILE(6,"mmap_size = 0x%llx",(uint64_t)chunk->path[path_index]->afu->mmap_size); CBLK_TRACE_LOG_FILE(6,"hrrq_start = 0x%llx",(uint64_t)chunk->path[path_index]->afu->p_hrrq_start); CBLK_TRACE_LOG_FILE(6,"hrrq_end = 0x%llx",(uint64_t)chunk->path[path_index]->afu->p_hrrq_end); + CBLK_TRACE_LOG_FILE(6,"chunk->flags = 0x%x, afu_flags = 0x%x",chunk->flags,chunk->path[path_index]->afu->flags); CBLK_TRACE_LOG_FILE(6,"cmd_start = 0x%llx",(uint64_t)chunk->cmd_start); CBLK_TRACE_LOG_FILE(6,"cmd_end = 0x%llx",(uint64_t)chunk->cmd_end); @@ -1951,9 +2191,6 @@ cflash_cmd_err_t cblk_process_sisl_cmd_err(cflsh_chunk_t *chunk,int path_index,c * The AFU is not reset and new requests can be issued. * This routine assumes the caller has the afu->lock. * - * NOTE: AFU does not properly support this yet. So it is not currently - * used. - * * INPUTS: * chunk - Chunk associated with this error * @@ -1966,6 +2203,25 @@ int cblk_reset_context_sisl(cflsh_chunk_t *chunk, int path_index) { int rc = 0; int wait_reset_context_retry = 0; + volatile __u64 *reg_offset = 0; + + + + if (chunk->path[path_index]->afu->flags & CFLSH_AFU_SQ) { + + /* + * This AFU is using SQ, so it does context reset + * via the SQ context reset register. + */ + + reg_offset = chunk->path[path_index]->afu->mmio + CAPI_SQ_CTXTRST_EA_OFFSET; + } else { + /* + * This AFU is not using SQ, so it does context reset + * via the IOARRIN register. + */ + reg_offset = chunk->path[path_index]->afu->mmio + CAPI_IOARRIN_OFFSET; + } #ifdef _FOR_DEBUG if (CBLK_SETUP_BAD_MMIO_SIGNAL(chunk,path_index,CAPI_IOARRIN_OFFSET+0x20)) { @@ -1981,23 +2237,39 @@ int cblk_reset_context_sisl(cflsh_chunk_t *chunk, int path_index) #endif /* _FOR_DEBUG */ /* - * Writing 1 to the IOARRIN, will cause all active commands + * Writing 1 to the IOARRIN/SQ_CONTEXT_RESET, will cause all active commands * to ultimately be dropped by the AFU. Then the AFU can * be issued commands again. */ - CBLK_OUT_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_IOARRIN_OFFSET, (uint64_t)1); + + CBLK_OUT_MMIO_REG(reg_offset, (uint64_t)1); - while ((CBLK_IN_MMIO_REG(chunk->path[path_index]->afu->mmio + CAPI_IOARRIN_OFFSET,FALSE)) && + while ((CBLK_IN_MMIO_REG(reg_offset,FALSE)) && (wait_reset_context_retry < CFLASH_BLOCK_MAX_WAIT_RST_CTX_RETRIES)) { /* * Wait a limited amount of time for the reset * context to complete. We are notified of this when * a read of IOARRIN returns 0. + * + * We also need to detect possible EEH here. + * */ CBLK_TRACE_LOG_FILE(5,"waiting for context reset to complete"); + + if ((CBLK_IN_MMIO_REG(reg_offset,FALSE)) == 0xffffffffffffffffLL) { + + CBLK_TRACE_LOG_FILE(5,"POssible UE detected"); + + /* + * Treat this a good completion of context reset, since the + * adapter is about to be reset. + */ + + return -1; + } usleep(CFLASH_BLOCK_DELAY_RST_CTX); wait_reset_context_retry++; @@ -2034,7 +2306,7 @@ int cblk_reset_context_sisl(cflsh_chunk_t *chunk, int path_index) * caller of this routine doing the retries. */ - while (((*(chunk->path[path_index]->afu->p_hrrq_curr)) & (SISL_RESP_HANDLE_T_BIT)) == chunk->path[path_index]->afu->toggle) { + while (((CBLK_READ_ADDR_CHK_UE(chunk->path[path_index]->afu->p_hrrq_curr)) & (SISL_RESP_HANDLE_T_BIT)) == chunk->path[path_index]->afu->toggle) { /* @@ -2083,8 +2355,10 @@ int cblk_init_sisl_fcn_ptrs(cflsh_path_t *path) path->fcn_ptrs.get_num_interrupts = cblk_get_sisl_num_interrupts; path->fcn_ptrs.get_cmd_room = cblk_get_sisl_cmd_room; path->fcn_ptrs.adap_setup = cblk_sisl_adap_setup; + path->fcn_ptrs.adap_sq_setup = cblk_sisl_adap_sq_setup; path->fcn_ptrs.get_intrpt_status = cblk_get_sisl_intrpt_status; path->fcn_ptrs.inc_rrq = cblk_inc_sisl_rrq; + path->fcn_ptrs.inc_sq = cblk_inc_sisl_sq; path->fcn_ptrs.get_cmd_data_length = cblk_get_sisl_cmd_data_length; path->fcn_ptrs.get_cmd_cdb = cblk_get_sisl_cmd_cdb; path->fcn_ptrs.get_cmd_rsp = cblk_get_sisl_cmd_rsp; diff --git a/src/block/exportfile b/src/block/exportfile index 44e7c2f2..aa24fcb4 100644 --- a/src/block/exportfile +++ b/src/block/exportfile @@ -1,36 +1,31 @@ -/* -* %Z%%M% %I% %W% %G% %U% -*/ -/* IBM_PROLOG_BEGIN_TAG */ -/* This is an automatically generated prolog. */ -/* */ -/* $Source: src/block/exportfile $ */ -/* */ -/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ -/* */ -/* Contributors Listed Below - COPYRIGHT 2014,2015 */ -/* [+] International Business Machines Corp. */ -/* */ -/* */ -/* Licensed under the Apache License, Version 2.0 (the "License"); */ -/* you may not use this file except in compliance with the License. */ -/* You may obtain a copy of the License at */ -/* */ -/* http://www.apache.org/licenses/LICENSE-2.0 */ -/* */ -/* Unless required by applicable law or agreed to in writing, software */ -/* distributed under the License is distributed on an "AS IS" BASIS, */ -/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ -/* implied. See the License for the specific language governing */ -/* permissions and limitations under the License. */ -/* */ -/* IBM_PROLOG_END_TAG */ - -/* +* IBM_PROLOG_BEGIN_TAG +* This is an automatically generated prolog. +* +* $Source: src/block/exportfile $ +* +* IBM Data Engine for NoSQL - Power Systems Edition User Library Project +* +* Contributors Listed Below - COPYRIGHT 2014,2015 +* [+] International Business Machines Corp. +* +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +* implied. See the License for the specific language governing +* permissions and limitations under the License. +* +* IBM_PROLOG_END_TAG +* * shared library export symbols * data * text -*/ cblk_init cblk_term @@ -47,3 +42,15 @@ cblk_awrite cblk_aresult cblk_listio cblk_clone_after_fork +cblk_cg_open +cblk_cg_close +cblk_cg_get_lun_size +cblk_cg_get_size +cblk_cg_set_size +cblk_cg_read +cblk_cg_write +cblk_cg_aread +cblk_cg_awrite +cblk_cg_aresult +cblk_cg_clone_after_fork +cblk_cg_get_num_chunks \ No newline at end of file diff --git a/src/block/libcflash_exportmap b/src/block/libcflash_exportmap index 2f27d81c..efc8d16b 100644 --- a/src/block/libcflash_exportmap +++ b/src/block/libcflash_exportmap @@ -47,5 +47,17 @@ global: cblk_init; cblk_term; cblk_aresult; cblk_listio; cblk_clone_after_fork; + cblk_cg_open; + cblk_cg_close; + cblk_cg_get_lun_size; + cblk_cg_get_size; + cblk_cg_set_size; + cblk_cg_read; + cblk_cg_write; + cblk_cg_aread; + cblk_cg_awrite; + cblk_cg_aresult; + cblk_cg_clone_after_fork; + cblk_cg_get_num_chunks; local: *; }; diff --git a/src/block/libcflsh_block.exp b/src/block/libcflsh_block.exp index 8928a824..8c8f7475 100644 --- a/src/block/libcflsh_block.exp +++ b/src/block/libcflsh_block.exp @@ -42,3 +42,15 @@ cblk_awrite cblk_aresult cblk_listio cblk_clone_after_fork +cblk_cg_open +cblk_cg_close +cblk_cg_get_lun_size +cblk_cg_get_size +cblk_cg_set_size +cblk_cg_read +cblk_cg_write +cblk_cg_aread +cblk_cg_awrite +cblk_cg_aresult +cblk_cg_clone_after_fork +cblk_cg_get_num_chunks \ No newline at end of file diff --git a/src/block/makefile b/src/block/makefile index a0b3a5cc..9d619979 100644 --- a/src/block/makefile +++ b/src/block/makefile @@ -24,7 +24,8 @@ # IBM_PROLOG_END_TAG ROOTPATH = ../.. -CFLAGS += +CFLAGS += -I$(ROOTPATH)/src/include -Wno-format-security +CFLAGS64 += -I$(ROOTPATH)/src/include #enable file mode by default, if the env var is not set. #to disable file mode, set BLOCK_FILEMODE_ENABLED=0 in surelock-sw/customrc @@ -51,22 +52,22 @@ endif MODULE = cflsh_block -ifeq ($(BLOCK_KERNEL_MC_ENABLED),1) - -CUSTOMFLAGS += -D_KERNEL_MASTER_CONTXT +OBJS = cflash_scsi_user.o cflash_tools_user.o cflash_block.o cflash_block_int.o\ + cflash_block_sisl.o cflash_block_cgroup.o cflash_block_raid0.o -OBJS = cflash_scsi_user.o cflash_tools_user.o cflash_block.o cflash_block_int.o cflash_block_kern_mc.o cflash_block_sisl.o +ifeq ($(BLOCK_KERNEL_MC_ENABLED),1) -OBJS64 = cflash_scsi_user.64o cflash_tools_user.64o cflash_block.64o cflash_block_int.64o cflash_block_kern_mc.64o cflash_block_sisl.64o +CUSTOMFLAGS += -D_KERNEL_MASTER_CONTXT +OBJS += cflash_block_kern_mc.o else -OBJS = cflash_scsi_user.o cflash_tools_user.o cflash_block.o cflash_block_int.o cflash_block_linux.o cflash_block_sisl.o - -OBJS64 = cflash_scsi_user.64o cflash_tools_user.64o cflash_block.64o cflash_block_int.64o cflash_block_linux.64o cflash_block_sisl.64o +OBJS += cflash_block_linux.o endif +OBJS64 = $(subst .o,.64o,$(OBJS)) + UNAME=$(shell uname) ifeq ($(UNAME),AIX) MODLIBS = -lpthreads diff --git a/src/block/test/blk_api_tst.c b/src/block/test/blk_api_tst.c index 50a33668..f6a113f6 100755 --- a/src/block/test/blk_api_tst.c +++ b/src/block/test/blk_api_tst.c @@ -30,13 +30,15 @@ #include #endif +#define CHUNK_SIZE 4096 +#define NBUFS 1000 char test_filename[256]; pthread_t blk_thread[MAX_NUM_THREADS]; - pthread_mutex_t completion_lock; +extern int mode; extern int num_opens; extern uint32_t thread_count; extern uint64_t block_number; @@ -47,6 +49,7 @@ extern uint64_t max_xfer;; extern int virt_lun_flags; extern int share_cntxt_flags; extern int test_max_cntx; +extern int host_type; extern char* env_filemode; extern char* env_max_xfer; extern char* env_num_cntx; @@ -56,7 +59,9 @@ extern int num_listio; char def_capidev[50] = "/dev/cxl/afu0.0s"; -char *dev_paths[MAX_LUNS]; +char dev_paths[MAX_LUNS][128]; + +chunk_ext_arg_t ext=1; char *env_blk_verbosity = NULL; @@ -96,16 +101,32 @@ void teminate_blk_tests() } } -int blk_fvt_setup(int size) - +/** + ******************************************************************************* + * \brief + * set device to RAID0 if cflash_devices.pl finds GT luns, else use FVT_DEV + ******************************************************************************/ +void blk_fvt_RAID0_setup(void) { + char buf[1024] = {0}; +#ifdef _AIX + FILE *fp = popen("cflash_devices.pl 2>/dev/null", "r"); +#else + FILE *fp = popen("/opt/ibm/capikv/afu/cflash_devices.pl 2>/dev/null", "r"); +#endif + if (fp) {fgets(buf, 1024, fp);} + if (strlen(buf) != 0) {strcpy(dev_paths[0],"RAID0");} + return; +} +int blk_fvt_setup(int size) +{ char *env_user = getenv("USER"); int fd; - char *p; - int i,ret; + FILE *fp=NULL; + char platform[1024]; paths = getenv("FVT_DEV"); @@ -164,6 +185,17 @@ int blk_fvt_setup(int size) } } +#ifndef _AIX + fp = popen("cat /proc/cpuinfo|grep platform", "r"); + if (fp) + { + fgets(platform, 1024, fp); + pclose(fp); + if (strlen(platform)>0 && strstr(platform,"PowerNV")==NULL) + {host_type=HOST_TYPE_VM;} + } +#endif + /* strcpy instead */ strcpy (&tpaths[0], paths); @@ -176,10 +208,9 @@ int blk_fvt_setup(int size) parents[i] = find_parent(p); DEBUG_2("\nparent of %s p = %s\n", p, parents[i]); } - dev_paths[i] = p; + strcpy(dev_paths[i],p); DEBUG_2("\npath %d : %s\n", i, p); p = strtok((char *)NULL,","); - } num_devs = i; DEBUG_1("blk_fvt_setup: num_devs = %d\n",num_devs); @@ -190,11 +221,15 @@ int blk_fvt_setup(int size) ret = blk_fvt_alloc_bufs(size); DEBUG_1("blk_fvt_alloc_bufs: ret = 0x%x\n", ret); + num_opens = 0; + ext = 0; + mode = O_RDWR; + DEBUG_2("blk_fvt_setupi_exit: dev1 = %s, dev2 = %s\n", dev_paths[0], dev_paths[1]); return (ret); - } + /* check parents of all devs, and see if the context is sharable */ int validate_share_context() { @@ -219,7 +254,6 @@ int validate_share_context() void blk_open_tst_inv_path (const char* path, int *id, int max_reqs, int *er_no, int opn_cnt, int flags, int mode) { chunk_id_t j = NULL_CHUNK_ID; - chunk_ext_arg_t ext = 0; errno = 0; j = cblk_open (path, max_reqs, mode, ext, flags); @@ -230,29 +264,48 @@ void blk_open_tst_inv_path (const char* path, int *id, int max_reqs, int *er_no, void blk_open_tst (int *id, int max_reqs, int *er_no, int opn_cnt, int flags, int mode) { - - // chunk_id_t handle[MAX_OPENS+15]; - int i = 0; chunk_id_t j = NULL_CHUNK_ID; - chunk_ext_arg_t ext = 0; errno = 0; - DEBUG_2("blk_open_tst : open %s %d times\n",dev_paths[0],opn_cnt); + DEBUG_4("blk_open_tst : path:%s cnt:%d max_reqs:%d flags:%x\n", + dev_paths[0], opn_cnt, max_reqs, flags); + DEBUG_3(" ext:%d num_opens:%d mode:%d\n", + (uint32_t)ext, num_opens, mode); - if (opn_cnt > 0) { - for (i = 0; i!= opn_cnt; i++) { + if (opn_cnt > 0) + { + for (i = 0; i!= opn_cnt; i++) + { DEBUG_1("Opening %s\n",dev_paths[0]); - j = cblk_open (dev_paths[0], max_reqs, mode, ext, flags); - if (j != NULL_CHUNK_ID) { + if (flags & CBLK_GROUP_ID) + { + DEBUG_1("Opening CG ext:%d\n", (uint32_t)ext); + j = cblk_cg_open(dev_paths[0], max_reqs, mode, ext, 0, flags); + } + else if (flags & CBLK_GROUP_RAID0) + { + DEBUG_1("Opening RAID0 ext:%d\n", (uint32_t)ext); + j = cblk_cg_open(dev_paths[0], max_reqs, mode, 1, ext, flags); + } + else + { + DEBUG_0("Opening dev\n"); + j = cblk_open(dev_paths[0], max_reqs, mode, ext, flags); + } + if (j != NULL_CHUNK_ID) + { // handle[i] = j; chunks[i] = j; num_opens += 1; - DEBUG_2("blk_open_tst: OPENED %d, chunk_id=%d\n", (num_opens), j); - } else { - *id = j; + DEBUG_3("blk_open_tst: OPENED %d, ext:%ld chunk_id=%d\n", + (num_opens), ext, j); + } + else + { + *id = j; *er_no = errno; - DEBUG_2("blk_open_tst: Failed: open i = %d, errno = %d\n", + DEBUG_2("blk_open_tst: Failed: open i = %d, errno = %d\n", i, errno); return; } @@ -267,7 +320,7 @@ void blk_close_tst(int id, int *rc, int *err, int close_flag) errno = 0; - *rc = cblk_close (id, close_flag); + *rc = cblk_close (id, close_flag); *err = errno; // it should be done in cleanup if ( !(*rc) && !(*err)) @@ -275,26 +328,34 @@ void blk_close_tst(int id, int *rc, int *err, int close_flag) DEBUG_3("blk_close_tst: id = %d, erno = %d, rc = %d\n", id, errno, *rc); } -void blk_open_tst_cleanup () +void blk_open_tst_cleanup (int flags, int *p_rc, int *p_err) { - int i = 0; int rc = 0; - errno = 0; + *p_rc = 0; + *p_err = 0; + errno = 0; - DEBUG_1("blk_open_tst_cleanup: closing num_opens = %d\n", num_opens); - for (i=0; num_opens |= 0; i++) { - if (chunks[i] != NULL_CHUNK_ID) { - rc = cblk_close (chunks[i],0); - if (!rc) { - DEBUG_1("blk_open_tst_cleanup: closed %d\n", num_opens); + DEBUG_2("blk_open_tst_cleanup: closing num_opens:%d flags:%x\n", + num_opens, flags); + for (i=0; i 0) { if (blk_verbosity) { DEBUG_0("Async read data completed ...\n"); @@ -554,11 +571,14 @@ void blk_fvt_io (chunk_id_t id, int cmd, uint64_t lba_no, size_t nblocks, int *r } } } else if (rc == 0) { - DEBUG_3("cblk_aresult completed: command still active for tag = 0x%x, rc = %d, errno = %d\n",tag,rc,errno); + DEBUG_3("cblk_aresult completed: command still active " + "for tag = 0x%x, rc = %d, errno = %d\n", + tag,rc,errno); usleep(1000); continue; } else { - DEBUG_3("cblk_aresult completed for for tag = 0x%d, rc = %d, errno = %d\n",tag,rc,errno); + DEBUG_3("cblk_aresult completed for for tag = 0x%d, " + "rc = %d, errno = %d\n",tag,rc,errno); } break; @@ -568,31 +588,64 @@ void blk_fvt_io (chunk_id_t id, int cmd, uint64_t lba_no, size_t nblocks, int *r break; case FV_WRITE: - rc = cblk_write(id,blk_fvt_comp_data_buf+align,lba_no,nblocks,0); - DEBUG_4("cblk_write complete at lba = 0x%lx, size = %lx, rc = %d, errno = %d\n",lba_no,nblocks,rc,errno); + rc = cblk_write(id,blk_fvt_comp_data_buf+align,lba_no, + nblocks,io_flags); + DEBUG_4("cblk_write complete at lba = 0x%lx, size = %lx, rc = %d, " + "errno = %d\n",lba_no,nblocks,rc,errno); if (rc <= 0) { - DEBUG_3("cblk_write failed at lba = 0x%lx, rc = %d, errno = %d\n",lba_no,rc,errno); + DEBUG_3("cblk_write failed at lba = 0x%lx, rc = %d, errno=%d\n", + lba_no,rc,errno); } break; case FV_AWRITE: - if ((open_flag & FV_NO_INRPT) && - (io_flags & CBLK_ARW_USER_STATUS_FLAG)) { - rc = cblk_awrite(id, blk_fvt_comp_data_buf+align, lba_no, nblocks, &tag, &awrt_status, io_flags); - } else { - rc = cblk_awrite(id, blk_fvt_comp_data_buf+align, lba_no, nblocks, - &tag, NULL, io_flags); + if (io_flags & CBLK_ARW_USER_STATUS_FLAG) + { + if (io_flags & CBLK_GROUP_RAID0) + { + rc = cblk_cg_awrite(id, blk_fvt_comp_data_buf+align, lba_no, + nblocks, &cgtag, &awrt_status, io_flags); + } + else + { + rc = cblk_awrite(id, blk_fvt_comp_data_buf+align, lba_no, + nblocks, &tag, &awrt_status, io_flags); + } + } + else + { + if (io_flags & CBLK_GROUP_RAID0) + { + rc = cblk_cg_awrite(id, blk_fvt_comp_data_buf+align, lba_no, + nblocks, &cgtag, NULL, io_flags); + } + else + { + rc = cblk_awrite(id, blk_fvt_comp_data_buf+align, lba_no, + nblocks, &tag, NULL, io_flags); + } } if (rc < 0) { - DEBUG_3("cblk_awrite error lba = 0x%lx, rc = %d, errno = %d\n", lba_no, rc, errno); + DEBUG_3("cblk_awrite error lba = 0x%lx, rc = %d, errno = %d\n", + lba_no, rc, errno); *ret = rc; *err = errno; return; - } else { - while (TRUE) { - - rc = cblk_aresult(id,&tag, &ar_status,arflag); - if (rc > 0) { + } + else + { + while (TRUE) + { + if (io_flags & CBLK_GROUP_RAID0) + { + rc = cblk_cg_aresult(id,&cgtag, &ar_status,io_flags); + } + else + { + rc = cblk_aresult(id, &tag, &ar_status, io_flags); + } + if (rc > 0) + { if (blk_verbosity) { DEBUG_0("Async write data completed ...\n"); if (blk_verbosity == 9) { @@ -600,14 +653,20 @@ void blk_fvt_io (chunk_id_t id, int cmd, uint64_t lba_no, size_t nblocks, int *r hexdump(blk_fvt_comp_data_buf,100,NULL); } } - } else if (rc == 0) { - DEBUG_3("cblk_aresult completed: command still active for tag = 0x%x, rc = %d, errno = %d\n",tag,rc,errno); + } + else if (rc == 0) + { + DEBUG_3("cblk_aresult completed: command still active " + "for tag = 0x%x, rc = %d, errno = %d\n", + tag,rc,errno); usleep(1000); continue; - } else { - DEBUG_3("cblk_aresult completed for for tag = 0x%d, rc = %d, errno = %d\n",tag,rc,errno); } - + else + { + DEBUG_3("cblk_aresult completed for for tag = 0x%d, " + "rc = %d, errno = %d\n",tag,rc,errno); + } break; } /* while */ @@ -624,19 +683,18 @@ void blk_fvt_io (chunk_id_t id, int cmd, uint64_t lba_no, size_t nblocks, int *r } -void -blk_fvt_intrp_io_tst(chunk_id_t id, - int testflag, - int open_flag, - int *ret, - int *err) +void blk_fvt_intrp_io_tst(chunk_id_t id, + int testflag, + int open_flag, + int *ret, + int *err) { int rc = 0; int err_no = 0; errno = 0; int tag = 0; - uint64_t lba_no = 1; - int nblocks = 1; + uint64_t lba_no = 1; + int nblocks = 1; cblk_arw_status_t arw_status; int fd; int arflag = 0; @@ -704,14 +762,12 @@ blk_fvt_intrp_io_tst(chunk_id_t id, void check_astatus(chunk_id_t id, int *tag, int arflag, int open_flag, int io_flags, cblk_arw_status_t *arw_status, int *rc, int *err) { - uint64_t ar_status = 0; int ret = 0; - while (TRUE) { - if ((open_flag & ~FV_NO_INRPT) && - (io_flags & CBLK_ARW_USER_STATUS_FLAG)) { + if (io_flags & CBLK_ARW_USER_STATUS_FLAG) + { switch (arw_status->status) { case CBLK_ARW_STATUS_SUCCESS: ret = arw_status->blocks_transferred; @@ -747,7 +803,10 @@ check_astatus(chunk_id_t id, int *tag, int arflag, int open_flag, int io_flags, } - +/** +******************************************************************************** +** \brief +*******************************************************************************/ void blk_fvt_dump_stats(chunk_stats_t *stats) { fprintf(stderr,"chunk_statistics:\n"); @@ -796,6 +855,10 @@ void blk_fvt_dump_stats(chunk_stats_t *stats) } +/** +******************************************************************************** +** \brief +*******************************************************************************/ void blk_get_statistics (chunk_id_t id, int flags, int *ret, int *err) { @@ -817,370 +880,352 @@ void blk_get_statistics (chunk_id_t id, int flags, int *ret, int *err) } } - - -void *blk_io_loop(void *data) +/** +******************************************************************************** +** \brief +*******************************************************************************/ +void* blk_io_loop(void *data) { - int rc = 0; - int tag; - int i; - blk_thread_data_t *blk_data = data; - uint32_t local_thread_count = 0; - void *data_buf = NULL; - void *comp_data_buf = NULL; - uint64_t blk_number; - uint64_t ar_status; - int cmd_type; - int fd; - int arflag = 0; - int x, num_luns; - - - - pthread_mutex_lock(&completion_lock); - local_thread_count = thread_count++; - - /* - * Each thread is using a different - * block number range. - */ - - - blk_number = block_number + (num_loops * thread_count); - - num_luns = num_devs ; + cflsh_cg_tag_t cgtag = {0}; + int tag = 0; + blk_thread_data_t *blk_data = data; + int rc = 0; + void *data_buf = NULL; + void *comp_data_buf = NULL; + uint64_t blk_number = 0; + uint64_t ar_status = 0; + int cmd_type = 0; + int fd = 0; + int arflag = 0; + int x = 0; + int loop = 0; + int lba = 0; + int max_lba = 0; + uint8_t rand = 0; - pthread_mutex_unlock(&completion_lock); + fd = open ("/dev/urandom", O_RDONLY); + read (fd, &rand, 1); + close(fd); + /* Each thread is using a different block number range */ + blk_number = blk_data->tid * CHUNK_SIZE; + max_lba = blk_number + CHUNK_SIZE; - /* - * Align data buffer on page boundary. - */ - if ( posix_memalign((void *)&data_buf,4096,BLK_FVT_BUFSIZE)) { - + if (posix_memalign((void *)&data_buf,4096,BLK_FVT_BUFSIZE)) + { perror("posix_memalign failed for data buffer"); - - blk_data->status.ret = -1; - blk_data->status.errcode = errno; - pthread_exit(&(blk_data->status)); + blk_data->ret = -1; + blk_data->errcode = errno; + goto exit; + } + if (posix_memalign((void*)&comp_data_buf,4096,BLK_FVT_BUFSIZE)) + { + perror("posix_memalign failed for data buffer"); + blk_data->ret = -1; + blk_data->errcode = errno; + goto exit; } - errno = 0; - if (local_thread_count % 2) { - cmd_type = FV_RW_AWAR; - } else { + + if (blk_data->tid % 2 && !(blk_data->flags & SYNC_IO_ONLY)) + { + cmd_type=FV_RW_AWAR; + } + else + { cmd_type = FV_RW_COMP; } - for (x =0; xstatus.ret = -1; - blk_data->status.errcode = errno; - pthread_exit(&(blk_data->status)); + for (x=0; xnum_devs; x++) + { + lba = blk_number; + loop = 0; - } + while (loop < num_loops) + { + memset(comp_data_buf, ++rand, BLK_FVT_BUFSIZE); + + if (cmd_type == FV_RW_COMP) + { + rc = cblk_write(blk_data->chunk_id[x],comp_data_buf, + lba,1,blk_data->flags); + if (rc != 1) + { + DEBUG_3("write failed tid:%d rc:%d errno:%d\n", + blk_data->tid, rc, errno); + goto exit; + } - fd = open ("/dev/urandom", O_RDONLY); - read (fd, comp_data_buf, BLK_FVT_BUFSIZE); - close (fd); + rc = cblk_read(blk_data->chunk_id[x],data_buf, + lba,1,blk_data->flags); + if (rc != 1) + { + DEBUG_3("read failed tid:%d rc:%d errno:%d\n", + blk_data->tid, rc, errno); + goto exit; + } - rc = cblk_write(blk_data->chunk_id[x],comp_data_buf,blk_number,1,0); - - if (rc != 1) { - blk_data->status.ret = rc; - blk_data->status.errcode = errno; - free(comp_data_buf); - free(data_buf); - pthread_mutex_lock(&completion_lock); - DEBUG_2("Write failed rc = %d, errno = %d\n",rc, errno); - pthread_mutex_unlock(&completion_lock); - pthread_exit(&(blk_data->status)); + if ((rc=memcmp(data_buf,comp_data_buf,BLK_FVT_BUFSIZE)) != 0) + { + if(blk_verbosity == 9) + { + fprintf(stderr,"Data compare failure\n"); + /* + fprintf(stderr,"Written data:\n"); + dumppage(data_buf,BLK_FVT_BUFSIZE); + fprintf(stderr,"******************************\n\n"); + fprintf(stderr,"read data:\n"); + dumppage(comp_data_buf,BLK_FVT_BUFSIZE); + fprintf(stderr,"******************************\n\n"); + */ } - rc = cblk_read(blk_data->chunk_id[x],data_buf,blk_number,1,0); - - if (rc != 1) { - - blk_data->status.ret = rc; - blk_data->status.errcode = errno; - free(comp_data_buf); - free(data_buf); - pthread_mutex_lock(&completion_lock); - DEBUG_2("Read failed rc = %d, errno = %d\n",rc, errno); - pthread_mutex_unlock(&completion_lock); - pthread_exit(&(blk_data->status)); - } - - rc = memcmp(data_buf,comp_data_buf,BLK_FVT_BUFSIZE); - - if (rc) { - blk_data->status.ret = rc; - blk_data->status.errcode = errno; - if(blk_verbosity == 9) { - pthread_mutex_lock(&completion_lock); - fprintf(stderr,"Data compare failure\n"); - /* - fprintf(stderr,"Written data:\n"); - dumppage(data_buf,BLK_FVT_BUFSIZE); - fprintf(stderr,"**********************************************************\n\n"); - fprintf(stderr,"read data:\n"); - dumppage(comp_data_buf,BLK_FVT_BUFSIZE); - fprintf(stderr,"**********************************************************\n\n"); - */ - pthread_mutex_unlock(&completion_lock); + rc = cblk_read(blk_data->chunk_id[x],data_buf, + lba,1,blk_data->flags); + if (rc == 1) + { + if (blk_verbosity == 9) + { + fprintf(stderr,"Dump of re-read\n"); + dumppage(data_buf,BLK_FVT_BUFSIZE); } + } + goto exit; + } + } + else if (cmd_type == FV_RW_AWAR) + { + memset(comp_data_buf, ++rand, BLK_FVT_BUFSIZE); - rc = cblk_read(blk_data->chunk_id[x],data_buf,blk_number,1,0); - if (rc == 1) { - if(blk_verbosity == 9) { - pthread_mutex_lock(&completion_lock); - fprintf(stderr,"Dump of re-read\n"); - - dumppage(data_buf,BLK_FVT_BUFSIZE); - pthread_mutex_unlock(&completion_lock); - } - } + arflag = blk_data->flags; + if (blk_data->flags & CBLK_GROUP_RAID0) + { + rc = cblk_cg_awrite(blk_data->chunk_id[x], comp_data_buf, + lba, 1, &cgtag, NULL, blk_data->flags); + } + else + { + rc = cblk_awrite(blk_data->chunk_id[x], comp_data_buf, + lba, 1, &tag, NULL,0); + } + if (rc < 0) + { + DEBUG_3("awrite failed tid:%d rc:%d errno:%d\n", + blk_data->tid, rc, errno); + goto exit; + } + while (TRUE) + { + if (blk_data->flags & CBLK_GROUP_RAID0) + { + rc = cblk_cg_aresult(blk_data->chunk_id[x], &cgtag, + &ar_status, arflag); } - free(comp_data_buf); - break; - case FV_RW_AWAR: - - /* - * Perform write then read comparision test - */ - - /* - * Align data buffer on page boundary. - */ - if ( posix_memalign((void *)&comp_data_buf,4096,BLK_FVT_BUFSIZE)) { - perror("posix_memalign failed for data buffer"); - perror("posix_memalign failed for data buffer"); - blk_data->status.ret = 0; - blk_data->status.errcode = errno; - pthread_exit(&(blk_data->status)); + else + { + rc = cblk_aresult(blk_data->chunk_id[x], &tag, + &ar_status, arflag); } - - fd = open ("/dev/urandom", O_RDONLY); - read (fd, comp_data_buf, BLK_FVT_BUFSIZE); - close (fd); - - rc = cblk_awrite(blk_data->chunk_id[x],comp_data_buf,blk_number,1,&tag,NULL,0); - - if (rc < 0) { - blk_data->status.ret = rc; - blk_data->status.errcode = errno; - free(comp_data_buf); - free(data_buf); - pthread_mutex_lock(&completion_lock); - DEBUG_2("Awrite failed rc = %d, errno = %d\n",rc, errno); - pthread_mutex_unlock(&completion_lock); - pthread_exit(&(blk_data->status)); - } - arflag = CBLK_ARESULT_BLOCKING; - while (TRUE) { - rc = cblk_aresult(blk_data->chunk_id[x],&tag, &ar_status,arflag); - if (rc > 0) { - pthread_mutex_lock(&completion_lock); - DEBUG_2("wrc=%d, tag = %x\n",rc, tag); - DEBUG_0("Async write data completed ...\n"); - pthread_mutex_unlock(&completion_lock); - } else if (rc == 0) { - pthread_mutex_lock(&completion_lock); - DEBUG_2("Waiting for command to complete wrc=%d, tag = %x\n",rc, tag); - pthread_mutex_unlock(&completion_lock); - usleep(1000); - continue; - } else { - pthread_mutex_lock(&completion_lock); - DEBUG_3("cblk_aresult completed (failed write) for for tag = 0x%x, rc = %d, errno = %d\n",tag,rc,errno); - pthread_mutex_unlock(&completion_lock); - } + if (rc > 0) + { + DEBUG_3("awrite cmp tid:%3d lba:%6d rc=%d \n", + blk_data->tid, lba, rc); break; - } /* while */ - - - rc = cblk_aread(blk_data->chunk_id[x],data_buf,blk_number,1,&tag,NULL,0); - - if (rc < 0) { - blk_data->status.ret = rc; - blk_data->status.errcode = errno; - free(comp_data_buf); - free(data_buf); - pthread_mutex_lock(&completion_lock); - DEBUG_2("Aread failed rc = %d, errno = %d\n",rc, errno); - pthread_mutex_unlock(&completion_lock); - pthread_exit(&(blk_data->status)); - } - - arflag = CBLK_ARESULT_BLOCKING; - while (TRUE) { + } + else if (rc == 0) + { + DEBUG_3("awrite wait tid:%3d lba:%6d rc=%d\n", + blk_data->tid, lba, rc); + usleep(10); + } + else + { + DEBUG_4("awrite err tid:%3d lba:%6d rc:%d errno:%d\n", + blk_data->tid, lba,rc,errno); + goto exit; + } + } - rc = cblk_aresult(blk_data->chunk_id[x],&tag, &ar_status,arflag); - if (rc > 0) { - pthread_mutex_lock(&completion_lock); - DEBUG_2("rc=%d, tag = %x\n",rc, tag); - DEBUG_0("Async read data completed ...\n"); - pthread_mutex_unlock(&completion_lock); - } else if (rc == 0) { - pthread_mutex_lock(&completion_lock); - DEBUG_2("rc=%d, tag = %x\n",rc, tag); - DEBUG_3("Waiting for command to complete for tag = 0x%x, rc = %d, errno = %d\n",tag,rc,errno); - pthread_mutex_unlock(&completion_lock); - usleep(1000); - continue; - } else { - pthread_mutex_lock(&completion_lock); - DEBUG_3("cblk_aresult completed (failed read) for for tag = 0x%x, rc = %d, errno = %d\n",tag,rc,errno); - pthread_mutex_unlock(&completion_lock); - } + if (blk_data->flags & CBLK_GROUP_RAID0) + { + rc = cblk_cg_aread(blk_data->chunk_id[x],data_buf,lba,1, + &cgtag,NULL,arflag); + } + else + { + rc = cblk_aread(blk_data->chunk_id[x],data_buf,lba,1, + &tag,NULL,arflag); + } + if (rc < 0) + { + DEBUG_2("Aread failed rc = %d, errno = %d\n",rc, errno); + goto exit; + } + while (TRUE) + { + if (blk_data->flags & CBLK_GROUP_RAID0) + { + rc = cblk_cg_aresult(blk_data->chunk_id[x], &cgtag, + &ar_status, arflag); + } + else + { + rc = cblk_aresult(blk_data->chunk_id[x], &tag, + &ar_status, arflag); + } + if (rc > 0) + { + DEBUG_3("aread cmp tid:%3d lba:%6d rc=%d \n", + blk_data->tid, lba, rc); break; + } + else if (rc == 0) + { + DEBUG_3("aread wait tid:%3d lba:%6d rc=%d\n", + blk_data->tid, lba, rc); + usleep(10); + } + else + { + DEBUG_4("aread err tid:%3d lba:%6d rc:%d errno:%d\n", + blk_data->tid, lba,rc,errno); + goto exit; + } + } - } /* while */ - - pthread_mutex_lock(&completion_lock); - DEBUG_2("Read ******** RC = %d tag %x\n",rc, tag); - - DEBUG_1("Read completed with rc = %d\n",rc); - pthread_mutex_unlock(&completion_lock); - - rc = memcmp(data_buf,comp_data_buf,BLK_FVT_BUFSIZE); - - if (rc) { - blk_data->status.ret = rc; - blk_data->status.errcode = errno; - if (blk_verbosity==9) { - pthread_mutex_lock(&completion_lock); - fprintf(stderr,"Data compare failure\n"); + if ((rc=memcmp(data_buf,comp_data_buf,BLK_FVT_BUFSIZE)) != 0) + { + if (blk_verbosity==9) + { + fprintf(stderr,"Data compare failure\n"); + /* + fprintf(stderr,"Written data:\n"); + dumppage(data_buf,BLK_FVT_BUFSIZE); + fprintf(stderr,"******************************\n\n"); + fprintf(stderr,"read data:\n"); + dumppage(comp_data_buf,BLK_FVT_BUFSIZE); + fprintf(stderr,"******************************\n\n"); + */ + } + rc = cblk_read(blk_data->chunk_id[x],data_buf, + lba,1,blk_data->flags); + if (rc == 1) + { + if (blk_verbosity==9) + { + DEBUG_0("Dump re-read OK\n"); /* - fprintf(stderr,"Written data:\n"); dumppage(data_buf,BLK_FVT_BUFSIZE); - fprintf(stderr,"**********************************************************\n\n"); - fprintf(stderr,"read data:\n"); - dumppage(comp_data_buf,BLK_FVT_BUFSIZE); - fprintf(stderr,"**********************************************************\n\n"); */ - pthread_mutex_unlock(&completion_lock); - } - rc = cblk_read(blk_data->chunk_id[x],data_buf,blk_number,1,0); - - if (rc == 1) { - if (blk_verbosity==9) { - DEBUG_0("Dump re-read OK\n"); - /* - dumppage(data_buf,BLK_FVT_BUFSIZE); - */ - } } - - } else if (!thread_flag) { - DEBUG_0("Memcmp succeeded\n"); } - free(comp_data_buf); - break; - default: - - fprintf(stderr,"Invalid cmd_type = %d\n",cmd_type); - i = num_loops; - } /* switch */ - - - blk_number++; - - pthread_mutex_lock(&completion_lock); - DEBUG_3("Dev = %d. Loop count = %d of %d\n",x, i,num_loops); - pthread_mutex_unlock(&completion_lock); - } /* for num_loops */ + goto exit; + } + } + //DEBUG_4("tid:%d dev:%d loop:%d lba:%d\n", + // blk_data->tid, x, loop, lba); + + if (++lba == max_lba) + { + loop += 1; + lba = blk_number; + DEBUG_3("tid:%d dev:%d loop:%d \n", blk_data->tid, x, loop); + } + } /* while num_loops */ } /* for num_luns */ - free(data_buf); +exit: + blk_data->ret = rc; + blk_data->errcode = errno; + + if (data_buf) {free(data_buf);} + if (comp_data_buf) {free(comp_data_buf);} - pthread_exit(&(blk_data->status)); + return NULL; } -void blk_thread_tst(int *ret, int *err) +/** +******************************************************************************** +** \brief +*******************************************************************************/ +void blk_thread_tst(int *ret, int *err, int open_flags, int io_flags) { + int rc = 0; + int ret_code = 0; + int i = 0; + int x = 0; + blk_thread_data_t *tdata = NULL; + chunk_id_t id[MAX_LUNS]= {0}; + + if (env_filemode && atoi(env_filemode)==1) + { + num_loops = 1; + num_threads = 4; + num_devs = 1; + } + tdata = malloc(num_threads*sizeof(blk_thread_data_t)); + memset(tdata,0,num_threads*sizeof(blk_thread_data_t)); - int rc; /* Return code */ - int ret_code=0; - int i,x; - void *status; - blk_thread_status_t thread_stat; - chunk_ext_arg_t ext = 0; - int flags = 0; - - - for (x=0; x < num_devs; x++) { - blk_data.size = 64; - if (virt_lun_flags) - flags = CBLK_OPN_VIRT_LUN; - if (share_cntxt_flags) - flags |= CBLK_OPN_SHARE_CTXT; - blk_data.chunk_id[x] = cblk_open(dev_paths[x],64,O_RDWR,ext,flags); - - if (blk_data.chunk_id[x] == NULL_CHUNK_ID) { + for (x=0; x= 1) { - - /* - * Create all threads here - */ - - for (i=0; i< num_threads; i++) { - - /* - rc = pthread_create(&blk_thread[i].thread,NULL,blk_io_loop,(void *)&blk_data); - */ - - rc = pthread_create(&blk_thread[i],NULL,blk_io_loop,(void *)&blk_data); - if (rc) { - + if (num_threads > 0) + { + /* Create all threads here */ + for (i=0; i 0) { + if (rc > 0) + { DEBUG_2("aresult rc = %d, tag = %d\n",rc,rtag); t++; - if (t>1000) - break; + if (t>num_cmds) {break;} } - if (rc == 0) { - printf("Z"); - usleep(1000); - continue; - } else if (rc < 0) { + else + { DEBUG_1("aresult error = %d\n",errno); break; } } - *ret = rc; *err = errno; return; } -#define NBUFS 10 - -void user_tag_io_tst (chunk_id_t id, int *ret, int *err) +/** +******************************************************************************** +** \brief +*******************************************************************************/ +void user_tag_io_tst (chunk_id_t id, int *ret, int *err, int flags) { int rc = -1; uint64_t ar_status = 0; @@ -1301,7 +1339,7 @@ void user_tag_io_tst (chunk_id_t id, int *ret, int *err) size_t nblocks = 1; int i = 0; int iter = 0; - int tag = 0; + int tag[NBUFS]= {0}; int arflg = 0; char *data_buf[NBUFS]; @@ -1330,16 +1368,16 @@ void user_tag_io_tst (chunk_id_t id, int *ret, int *err) } bzero(blk_fvt_comp_data_buf, BLK_FVT_BUFSIZE); - arflg = CBLK_ARW_WAIT_CMD_FLAGS | CBLK_ARW_USER_TAG_FLAG; + arflg = CBLK_ARW_WAIT_CMD_FLAGS | CBLK_ARW_USER_TAG_FLAG | flags; /* do writes */ - lba = 1; + lba = 10; for (i=0; i0; i--) + for (i=NBUFS-1; i>=0; i--) { - tag = i; - rc = cblk_aresult(id, &tag, &ar_status, arflg); + rc = cblk_aresult(id, &tag[i], &ar_status, arflg); if (rc != 1) { DEBUG_1("aresult error = %d\n",errno); rc=-1; goto exit; } - DEBUG_2("\n***** cblk_aresult rc = 0x%d, tag = %d\n", rc, i); + DEBUG_2("\n***** cblk_aresult rc = 0x%d, tag = %d\n", rc, tag[i]); } /* do read-compares */ - lba = 1; + lba = 10; for (i=0; i0) + { num_cmds = atoi(env_num_cmds); - /** limit 4K cmds **/ - if (num_cmds > 4096) - num_cmds = 4096; - } - - if (env_num_loop && atoi(env_num_loop )) { - loops = atoi(env_num_loop); - } - - if (env_io_comp && (atoi(env_io_comp)==1)) { - validate = atoi(env_io_comp); + if (num_cmds > 4096) {num_cmds = 4096;} } + if (env_num_loop && atoi(env_num_loop)>0) {loops = atoi(env_num_loop);} + if (env_io_comp && atoi(env_io_comp)==1) {validate = 1;} + if (env_filemode && atoi(env_filemode)==1) {loops=2; num_cmds=64;} + ctag = malloc(num_cmds*sizeof(int)); + cgtag = malloc(num_cmds*sizeof(cflsh_cg_tag_t)); /* fill compare buffer with pattern */ fd = open ("/dev/urandom", O_RDONLY); read (fd, blk_fvt_comp_data_buf, BLK_FVT_BUFSIZE*num_cmds); close (fd); - for (x=0; x 0) { - t++; - if (t>num_cmds) + i=0; + while (TRUE) + { + errno=0; + if (flags & CBLK_GROUP_RAID0) + { + if (cgtag[i].tag == -1) {++i; continue;} + rc = cblk_cg_aresult(id, &cgtag[i], &ar_status, flags); + } + else + { + if (ctag[i] == -1) {++i; continue;} + rc = cblk_aresult(id, &ctag[i], &ar_status, flags); + } + if (rc > 0) + { + cgtag[i++].tag = -1; + if (i>=num_cmds) + { + DEBUG_3("awrite->aresult DONE rc:%d t:%d num_cmds:%d\n", + rc, t, num_cmds); break; + } } - if (rc == 0) - continue; - if (rc < 0) { - fprintf(stderr,"aresult cmdno = %d, error = %d, rc = 0x%x, \n", + else if (rc == 0) {DEBUG_0("awrite->aresult rc=0\n"); continue;} + else + { + fprintf(stderr,"awrite->aresult cmdno:%d errno:%d rc:%d\n", t, errno,rc); *ret = rc; *err = errno; @@ -1503,69 +1559,94 @@ void io_perf_tst (chunk_id_t id, int *ret, int *err) } } - /* read wrote buffer */ - - for (t=1, lba=1, i=0; i < num_cmds; i++,lba++) { - rc = cblk_aread(id, - (char*)(blk_fvt_data_buf + (i*size)), - lba,nblocks,&cmdtag[i],NULL,arflg); - if (rc < 0) { - fprintf(stderr,"awrite failed for lba = 0x%lx, rc = %d, errno = %d\n",lba,rc,errno); + /* read the buffer we wrote */ + for (lba=0, i=0; i 0) { - t++; - if (t>num_cmds) - break; + i=0; + while (TRUE) + { + if (flags & CBLK_GROUP_RAID0) + { + if (cgtag[i].tag == -1) {++i; continue;} + rc = cblk_cg_aresult(id, &cgtag[i], &ar_status, flags); } - if (rc == 0) - continue; - if (rc < 0) { - fprintf(stderr,"aresult error = %d\n",errno); + else + { + if (ctag[i] == -1) {++i; continue;} + rc = cblk_aresult(id, &ctag[i], &ar_status, flags); + } + if (rc > 0) + { + cgtag[i++].tag = -1; + if (i>=num_cmds) {break;} + } + else if (rc == 0) {continue;} + else + { + fprintf(stderr,"aread->aresult error = %d\n",errno); *ret = rc; *err = errno; return; } } - if (!validate) { + if (validate) + { ret_code = memcmp((char*)(blk_fvt_data_buf), - (char*)(blk_fvt_comp_data_buf), - BLK_FVT_BUFSIZE*size); - - if (ret_code) { + (char*)(blk_fvt_comp_data_buf), + num_cmds*size); + if (ret_code) + { fprintf(stderr,"\n memcmp failed rc = 0x%x\n",ret_code); *ret = ret_code; *err = errno; return; } + /* clear the read buf for the next pass */ + memset(blk_fvt_data_buf,0,num_cmds*size); } } + if (ctag) {free(ctag);} + if (cgtag) {free(cgtag);} DEBUG_2("Perf Test existing i = %d, x = %d\n", i, x ); return; } - +/** +******************************************************************************** +** \brief +*******************************************************************************/ int max_context(int *ret, int *err, int reqs, int cntx, int flags, int mode) { int i = 0; int t = 0; chunk_id_t j = NULL_CHUNK_ID; - chunk_ext_arg_t ext = 0; errno = 0; int status ; - char *path = dev_paths[0]; + char *path = (char*)dev_paths[0]; pid_t child_pid [700]; errno = 0; @@ -1630,12 +1711,13 @@ int max_context(int *ret, int *err, int reqs, int cntx, int flags, int mode) read(pipefds[i*2], &close_now, sizeof(int)); DEBUG_2("\nChild %d, Received %d \n",pid, close_now); } - DEBUG_2("\nChild %d, Received Parent's OK =%d \n",pid, close_now); + DEBUG_3("\nChild %d, Received Parent's OK =%d close id:%d\n", + pid, close_now, j); cblk_close(j, 0); /* exit success */ exit(0); } else { - fprintf(stderr,"\nmax_context: child =%d ret = 0x%x,open error = %d\n",i+1, j, errno); + DEBUG_3("\nmax_context: child =%d ret = 0x%x,open error = %d\n",i+1, j, errno); child_ret = j; child_err = errno; /* Send errcode thru ouput side */ @@ -1693,7 +1775,7 @@ int max_context(int *ret, int *err, int reqs, int cntx, int flags, int mode) } /* give max 5 sec */ /* end test on any error */ if (ret_code || errcode) { - fprintf(stderr,"\nmax_context: Child = %d, ret = 0x%x, err = %d\n", + DEBUG_3("\nmax_context: Child = %d, ret = 0x%x, err = %d\n", i+1, ret_code,errcode); break; } @@ -1701,6 +1783,7 @@ int max_context(int *ret, int *err, int reqs, int cntx, int flags, int mode) *ret = ret_code; *err = errcode; + DEBUG_2("ret%d, err:%d\n", *ret, *err); // Close all pipes fds @@ -1716,15 +1799,18 @@ int max_context(int *ret, int *err, int reqs, int cntx, int flags, int mode) return(0); } +/** +******************************************************************************** +** \brief +*******************************************************************************/ int child_open(int c, int max_reqs, int flags, int mode) { chunk_id_t j = NULL_CHUNK_ID; - chunk_ext_arg_t ext = 0; errno = 0; DEBUG_1 ("\nchild_open: opening for child %d\n", c); - char *path = dev_paths[0]; + char *path = (char*)dev_paths[0]; j = cblk_open (path, max_reqs, mode, ext, flags); if (j != NULL_CHUNK_ID) { @@ -1740,6 +1826,11 @@ int child_open(int c, int max_reqs, int flags, int mode) return (-1); } } + +/** +******************************************************************************** +** \brief +*******************************************************************************/ int fork_and_clone_mode_test(int *ret, int *err, int pmode, int cmode) { chunk_id_t id = 0; @@ -1866,51 +1957,46 @@ int fork_and_clone_mode_test(int *ret, int *err, int pmode, int cmode) return(0); } -int fork_and_clone(int *ret, int *err, int mode) +/** +******************************************************************************** +** \brief +*******************************************************************************/ +int fork_and_clone(int *ret, int *err, int mode, int open_flags, int io_flags) { chunk_id_t id = 0; - int flags = CBLK_OPN_VIRT_LUN; - int sz_flags= 0; int max_reqs= 64; int open_cnt= 1; - int get_set_size_flag = 0; // 0 = get phys lun sz - // 1 = get chunk sz - // 2 = set chunk sz - - int rc; - int er; - + int rc; + int er; uint64_t lba; - int io_flags = 0; - int open_flag = 0; size_t temp_sz,nblks; int cmd; - pid_t child_pid; - int child_status; - pid_t w_ret; - + pid_t child_pid; + int child_status; + pid_t w_ret; int child_ret; int child_err; int ret_code=0; int ret_err=0; int fd[2]; - + int get_set_size_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz // create pipe to be used by child to pass back status - pipe(fd); if (blk_fvt_setup(1) < 0) return (-1); // open virtual lun - blk_open_tst( &id, max_reqs, &er, open_cnt, flags, mode); + blk_open_tst(&id, max_reqs, &er, open_cnt, open_flags, mode); if (id == NULL_CHUNK_ID) return (-1); temp_sz = 64; get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &rc, &er); + blk_fvt_get_set_lun_size(id, &temp_sz, io_flags, get_set_size_flag, &rc, &er); if (rc | er) { DEBUG_0("fork_and_clone: set_size failed\n"); *ret = rc; @@ -1921,7 +2007,7 @@ int fork_and_clone(int *ret, int *err, int mode) cmd = FV_WRITE; lba = 1; nblks = 1; - blk_fvt_io(id, cmd, lba, nblks, &rc, &er, io_flags, open_flag); + blk_fvt_io(id, cmd, lba, nblks, &rc, &er, io_flags, open_flags); if (rc != 1) { DEBUG_0("fork_and_clone: blk_fvt_io failed\n"); @@ -1962,7 +2048,7 @@ int fork_and_clone(int *ret, int *err, int mode) cmd = FV_READ; lba = 1; nblks = 1; - blk_fvt_io(id, cmd, lba, nblks, &rc, &er, io_flags, open_flag); + blk_fvt_io(id, cmd, lba, nblks, &rc, &er, io_flags, open_flags); if (rc != 1) { // error DEBUG_0("fork_and_clone: child I/O failed\n"); @@ -2031,6 +2117,10 @@ int fork_and_clone(int *ret, int *err, int mode) return(0); } +/** +******************************************************************************** +** \brief +*******************************************************************************/ void blk_list_io_arg_test(chunk_id_t id, int arg_tst, int *err, int *ret) { int rc = 0; @@ -2185,6 +2275,10 @@ void blk_list_io_arg_test(chunk_id_t id, int arg_tst, int *err, int *ret) return; } +/** +******************************************************************************** +** \brief +*******************************************************************************/ void blk_list_io_test(chunk_id_t id, int cmd, int t_type, int uflags, uint64_t timeout, int *err, int *ret, int num_listio) { int rc = 0; @@ -2368,6 +2462,10 @@ void blk_list_io_test(chunk_id_t id, int cmd, int t_type, int uflags, uint64_t t return; } +/** +******************************************************************************** +** \brief +*******************************************************************************/ int poll_arw_stat(cblk_io_t *io, int num_listio) { @@ -2443,6 +2541,10 @@ int poll_arw_stat(cblk_io_t *io, int num_listio) } +/** +******************************************************************************** +** \brief +*******************************************************************************/ int check_completions(cblk_io_t *io, int num_listio ) { int i; @@ -2500,7 +2602,10 @@ int check_completions(cblk_io_t *io, int num_listio ) } - +/** +******************************************************************************** +** \brief +*******************************************************************************/ #ifdef _AIX char *find_parent(char *device_name) { diff --git a/src/block/test/blk_tst.h b/src/block/test/blk_tst.h index e1bc84d6..f67ddc3c 100755 --- a/src/block/test/blk_tst.h +++ b/src/block/test/blk_tst.h @@ -59,7 +59,7 @@ extern void *blk_fvt_comp_data_buf; #define MAX_NUM_THREADS 4096 #define BLK_FVT_BUFSIZE 4096 #define NUM_LIST_IO 500 - +#define SYNC_IO_ONLY 0x0800000 #define FILESZ 4096*4096*64 @@ -70,15 +70,22 @@ extern void *blk_fvt_comp_data_buf; #define FV_RW_COMP 4 #define FV_RW_AWAR 5 -/* io_flags used in blk_fvt_io, to force conditions */ -#define FV_ALIGN_BUFS 0x1 -#define FV_ARESULT_BLOCKING 0x2 -#define FV_ARESULT_NEXT_TAG 0x4 -#define FV_NO_INRPT 0x8 +#define HOST_TYPE_BMC 0 +#define HOST_TYPE_VM 1 #define TESTCASE_SKIP(_reason) \ printf("[ SKIPPED ] %s\n", _reason); +#define TESTCASE_SKIP_IF_FILEMODE \ + do \ + { \ + if ((env_filemode) && (atoi(env_filemode) == 1)) \ + { \ + TESTCASE_SKIP("skip if BLOCK_FILEMODE_ENABLED==1"); \ + return; \ + } \ +} while (0) + #define DEBUG_0(A) \ do \ { \ @@ -117,16 +124,15 @@ extern void *blk_fvt_comp_data_buf; {fprintf(stderr,A,B,C,D,E);fflush(stderr);} \ } while (0) -typedef struct blk_thread_status { - int ret; - int errcode; -} blk_thread_status_t; - -typedef struct blk_thread_data { - chunk_id_t chunk_id[MAX_LUNS]; - blk_thread_status_t status; - int flags; - size_t size; +typedef struct blk_thread_data +{ + chunk_id_t chunk_id[MAX_LUNS]; + int num_devs; + int flags; + int tid; + size_t size; + int ret; + int errcode; } blk_thread_data_t; @@ -138,7 +144,7 @@ void blk_open_tst_inv_path(const char* path,int *id, int max_reqs, int *er_no, i void blk_open_tst_close ( int id); -void blk_open_tst_cleanup (); +void blk_open_tst_cleanup (int flags, int *p_ret, int *p_err); void blk_close_tst(int id, int *ret, int *er_no, int close_flag); @@ -147,16 +153,17 @@ void blk_fvt_get_set_lun_size(chunk_id_t id, size_t *size, int sz_flags, int get void blk_fvt_io (chunk_id_t id, int cmd, uint64_t lba, size_t nblocks, int *ret, int *err, int io_flag, int open_flag); int blk_fvt_setup(int size); +void blk_fvt_RAID0_setup(void); int multi_lun_setup(); void blk_fvt_cmp_buf(int size, int *ret); void blk_get_statistics (chunk_id_t id, int flags, int *ret, int *err); -void blk_thread_tst(int *ret, int *err); +void blk_thread_tst(int *ret, int *err, int open_flags, int io_flags); void blk_multi_luns_thread_tst(int *ret, int *err); void blocking_io_tst (chunk_id_t id, int *ret, int *err); -void user_tag_io_tst (chunk_id_t id, int *ret, int *err); -void io_perf_tst (chunk_id_t id, int *ret, int *err); -int fork_and_clone(int *ret, int *err, int mode); +void user_tag_io_tst (chunk_id_t id, int *ret, int *err, int flags); +void io_perf_tst (chunk_id_t id, int *ret, int *err, int flags); +int fork_and_clone(int *ret, int *err, int mode, int open_flags, int io_flags); int fork_and_clone_mode_test(int *ret, int *err, int pmode, int cmode); int fork_and_clone_my_test(int *ret, int *err, int pmode, int cmode); void blk_fvt_intrp_io_tst(chunk_id_t id, int testflag, int open_flag, int *ret, int *err); diff --git a/src/block/test/blockio.c b/src/block/test/blockio.c index cc1148bd..7faf1903 100644 --- a/src/block/test/blockio.c +++ b/src/block/test/blockio.c @@ -50,23 +50,33 @@ #include #include #include -#include #include #include #include #include #include #include -#include #include -#include +#include +#include + +#ifdef _OS_INTERNAL +#include +#else +#include +#endif -#define _8K 8*1024 -#define _4K 4*1024 -#define NBUFS _8K +#include -#define TIME_INTERVAL 1024 -#define SET_NBLKS 260*1024 +#define _8K 8*1024 +#define _4K 4*1024 +#define NBUFS _8K +#define _1MSEC 1000 +#define _1SEC 1000000 +#define _10SEC 10000000 +#define READ 1 +#define WRITE 0 +#define MAX_SGDEVS 8 /** ******************************************************************************** @@ -74,28 +84,139 @@ *******************************************************************************/ typedef struct { + int id; uint64_t start; - uint8_t iss; + uint64_t lba; + uint32_t cnt; + uint32_t miss; + uint32_t min_to; + uint32_t max_to; + uint32_t iss; + uint32_t rd; } OP_t; /** ******************************************************************************** -** \brief call to begin an op, set issued flag, set start ticks +** \brief struct for each dev *******************************************************************************/ -#define OP_BEG(_tag) op[_tag].iss=1; op[_tag].start=getticks(); +typedef struct +{ + int id; + int htag; + OP_t *op; + uint32_t RD; + uint32_t WR; + size_t lba; + size_t last_lba; + uint32_t nblocks; + uint32_t issN; + uint8_t *rbuf; + uint8_t *wbuf; +} SGDEV_t; /** ******************************************************************************** -** \brief call to end an op, reset issued flag, save tick delta +** \brief print debug msg if DEBUG (-D) *******************************************************************************/ -#define OP_END(_tag) op[_tag].iss=0; tlat+=getticks() - op[_tag].start; +#define debug(fmt, ...) \ + do {if (DEBUG) {fprintf(stdout,fmt,##__VA_ARGS__); fflush(stdout);}} while(0) /** ******************************************************************************** ** \brief bump lba, reset lba if it wraps *******************************************************************************/ -#define BMP_LBA() lba+=nblocks+1; \ - if (lba+nblocks >= nblks) {lba=lrand48()%nblks;} +#define BMP_LBA(_dev) \ + if (seq) \ + { \ + _dev->lba += _dev->nblocks; \ + if (_dev->lba+_dev->nblocks >= _dev->last_lba) {_dev->lba=0;} \ + } \ + else {_dev->lba = lrand48() % _dev->last_lba;} + +/** +******************************************************************************** +** \brief call to begin an op, init data for the op +*******************************************************************************/ +#define OP_BEG(_dev,_op,_rd) _op->start = getticks(); ++N; \ + _op->iss = 1; \ + _op->id = _dev->id; \ + _op->lba = _dev->lba; \ + _op->rd = _rd; \ + _op->cnt = cnt; \ + _op->miss = 0; \ + _op->min_to = 0; \ + _op->max_to = 0; \ + _dev->issN += 1; \ + if (_rd==READ) {--_dev->RD;} else {--_dev->WR;} \ + debug("OP_BEG: id:%d rd:%d lba:%9ld len:%d, issN:%4d RD:%d cnt:%d\n", \ + _op->id, (_op->rd==READ), _op->lba, _dev->nblocks, _dev->issN, \ + _dev->RD,cnt); \ + BMP_LBA(_dev); + +/** +******************************************************************************** +** \brief call to end an op, reset issued flag, save tick delta +*******************************************************************************/ +#define OP_END(_dev, _op, _cticks) \ + lat = _cticks - _op->start; --N; \ + cnt += 1; \ + _dev->issN -= 1; \ + _op->iss = 0; \ + tlat += lat; \ + if (verbose) \ + { if (_op->rd) {++cnt_rd; tlat_rd+=lat;} \ + else {++cnt_wr; tlat_wr+=lat;} \ + } \ + debug("OP_END: id:%d rd:%d lba:%9ld issN:%4d\n", \ + _op->id, (_op->rd==READ), _op->lba, _dev->issN); + +/** +******************************************************************************** +** \brief call to check timeout for an op(cmd), print msg for warning TO & TO +*******************************************************************************/ +#define OP_CHK_WARN_TO(_op, _cticks, _min, _max) \ + if (verbose) \ + { \ + lat = ((_cticks - _op->start) * ns_per_tick) / 1000; \ + if (lat > _min && !_op->min_to++) \ + {printf("ETO: pid:%d cnt:%7d lba:%16ld rd:%d miss:%d TIME:%d\n", \ + getpid(), cnt, _op->lba, _op->rd, \ + _op->miss, lat);} \ + else if (lat > _max && !_op->max_to++) \ + {printf("TO: pid:%d cnt:%7d lba:%16ld rd:%d miss:%d TIME:%d\n", \ + getpid(), cnt, _op->lba, _op->rd, \ + _op->miss, lat);} \ + } + +/** +******************************************************************************** +** \brief call to check latency for each op, print msg for LONG cmds +*******************************************************************************/ +#define OP_CHK_LATENCY(_op, _lto, _cticks) \ + if (verbose) \ + { \ + lat = ((_cticks - _op->start) * ns_per_tick) / 1000; \ + if (lat > _lto) \ + {printf("LTO: pid:%d cnt:%7d lba:%16ld rd:%d miss:%d TIME:%d\n", \ + getpid(), cnt, _op->lba, _op->rd, \ + _op->miss, lat);} \ + } + +/** +******************************************************************************** +** \brief print errno and exit +** \details +** An IO has failed, print the errno and exit +*******************************************************************************/ +#define IO_ISS_ERR(_type) \ + fprintf(stderr, "io_iss_error: errno:%d pid:%d cnt:%7d rd:%d\n", \ + errno, getpid(), cnt, (READ==_type)); \ + goto exit; + +#define IO_CMP_ERR(_op) \ + fprintf(stderr, "io_cmp_error: errno:%d pid:%d cnt:%7d lba:%16ld rd:%d\n", \ + errno, getpid(), _op->cnt, _op->lba, (READ==_op->rd)); \ + goto exit; /** ******************************************************************************** @@ -106,31 +227,23 @@ typedef struct ** -q queuedepth *the number of outstanding ops to maintain \n ** -n nblocks *the number of 4k blocks in each I/O request \n ** -s secs *the number of seconds to run the I/O \n +** -e eto *early timeout: microseconds, print warning for each op \n +** -l lto *long timeout, microseconds, print elapse time for each op \n +** -S vlunsize *size in gb for the vlun \n ** -p *run in physical lun mode \n -** -i *run using interrupts, not polling +** -i *run using interrupts, not polling \n +** -R *randomize lbas, default is on, use -R0 to run sequential \n +** -v *print cmd timeout warnings \n *******************************************************************************/ void usage(void) { printf("Usage:\n"); - printf(" \ -[-d device] [-r %%rd] [-q queuedepth] [-n nblocks] [-s secs] [-p] [-i]\n"); + printf(" \ +[-d device:device] [-r %%rd] [-q queuedepth] [-n nblocks] [-s secs] [-e eto] \ +[-l lto] [-S vlunsize] [-p] [-i] [-R0] [-v]\n"); exit(0); } -/** -******************************************************************************** -** \brief print errno and exit -** \details -** An IO has failed, print the errno and exit -*******************************************************************************/ -void io_error(int id, int err) -{ - fprintf(stderr, "io_error: errno:%d\n", err); - cblk_close(id,0); - cblk_term(NULL,0); - exit(err); -} - /** ******************************************************************************** ** \brief main @@ -144,84 +257,163 @@ void io_error(int id, int err) *******************************************************************************/ int main(int argc, char **argv) { - struct timeval start, delta; - long int mil = 1000000; - float esecs = 0; - uint8_t *rbuf = NULL; - uint8_t *wbuf = NULL; + char devStrs[MAX_SGDEVS][64]; + char *devStr = NULL; + char *pstr = NULL; + SGDEV_t *devs = NULL; + SGDEV_t *dev = NULL; + uint32_t devN = 0; OP_t *op = NULL; - char *dev = NULL; char FF = 0xFF; char c = '\0'; chunk_ext_arg_t ext = 0; int flags = 0; - int i, rc = 0; - int id = 0; + int i=0, rc = 0; char *_secs = NULL; char *_QD = NULL; char *_RD = NULL; char *_nblocks = NULL; + char *_vlunsize = NULL; + char *_eto = NULL; + char *_lto = NULL; + char *_R = NULL; uint32_t plun = 0; uint32_t nsecs = 4; - uint32_t QD = 256; + uint32_t QD = 128; uint32_t nRD = 100; - uint32_t RD = 0; - uint32_t WR = 0; + uint32_t eto = 1000000; + uint32_t lto = 10000; uint32_t intrp_thds = 0; int rtag = 0; - int htag = 0; - uint32_t lba = 0; - size_t nblks = 0; + int pflag = 0; + size_t lun_size = 0; uint32_t nblocks = 1; + uint32_t vlunsize = 1; uint32_t cnt = 0; + uint32_t cnt_rd = 0; + uint32_t cnt_wr = 0; uint32_t tmiss = 0; + uint32_t hits = 1; + uint32_t hbar = 0; uint64_t status = 0; - uint32_t TI = TIME_INTERVAL; uint32_t N = 0; uint32_t TIME = 1; uint32_t COMP = 0; - uint32_t miss = 0; + uint32_t lat = 0; + uint32_t esecs = 0; + uint32_t seq = 0; + uint32_t aresN = 0; + uint32_t verbose = 0; + uint32_t DEBUG = 0; + uint32_t MAX_POLL = 100; + uint32_t issI = 0; + uint32_t dev_i = 0; uint64_t tlat = 0; + uint64_t tlat_rd = 0; + uint64_t tlat_wr = 0; + uint64_t sticks = 0; + uint64_t cticks = 0; double ns_per_tick = 0; /*-------------------------------------------------------------------------- * process and verify input parms *------------------------------------------------------------------------*/ - while (FF != (c=getopt(argc, argv, "d:r:q:n:s:phi"))) + while (FF != (c=getopt(argc, argv, "d:r:q:n:s:e:S:R:l:phivD"))) { switch (c) { - case 'd': dev = optarg; break; + case 'd': devStr = optarg; break; case 'r': _RD = optarg; break; case 'q': _QD = optarg; break; case 'n': _nblocks = optarg; break; + case 'S': _vlunsize = optarg; break; case 's': _secs = optarg; break; + case 'e': _eto = optarg; break; + case 'l': _lto = optarg; break; + case 'R': _R = optarg; break; case 'p': plun = 1; break; case 'i': intrp_thds = 1; break; + case 'D': DEBUG = 1; break; + case 'v': verbose = 1; break; case 'h': case '?': usage(); break; } } - if (_secs) nsecs = atoi(_secs); - if (_QD) QD = atoi(_QD); - if (_nblocks) nblocks = atoi(_nblocks); - if (_RD) nRD = atoi(_RD); + if (_secs) nsecs = atoi(_secs); + if (_QD) QD = atoi(_QD); + if (_nblocks) nblocks = atoi(_nblocks); + if (_vlunsize) vlunsize = atoi(_vlunsize); + if (_RD) nRD = atoi(_RD); + if (_eto) eto = atoi(_eto); + if (_lto) lto = atoi(_lto); + else if (QD>1) lto = 20000; + if (_R && _R[0]=='0') seq= 1; - if (QD > _8K) QD = _8K; - if (nRD > 100) nRD = 100; + QD = (QD < _8K) ? QD : _8K; + nRD = (nRD < 100) ? nRD : 100; + if (plun && vlunsize > 1) + { + printf("error: <-S %d> can only be used with a virtual lun\n",vlunsize); + usage(); + } if (!plun && nblocks > 1) { printf("error: <-n %d> can only be used with a physical lun\n",nblocks); usage(); } - if (dev == NULL) usage(); + if (devStr == NULL) usage(); + + while ((pstr=strsep(&devStr,":"))) + { + if (devN == MAX_SGDEVS) {debug("too many devs\n"); break;}; + debug("%s\n",pstr); + sprintf(devStrs[devN++],"%s",pstr); + } + + vlunsize = vlunsize*256*1024; srand48(time(0)); ns_per_tick = time_per_tick(1000, 100); - N = QD; - COMP = QD < 8 ? 1 : QD/8; + hbar = QD<10 ? 1 : QD/10; + COMP = QD<6 ? 1 : QD/6; + + /*-------------------------------------------------------------------------- + * alloc data + *------------------------------------------------------------------------*/ + debug("malloc devs: %d\n", devN); + if ((rc=posix_memalign((void**)&devs, 128, devN*sizeof(SGDEV_t)))) + { + fprintf(stderr,"posix_memalign failed, size=%ld, rc=%d\n", + devN*sizeof(SGDEV_t), rc); + exit(0); + } + memset(devs,0,devN*sizeof(SGDEV_t)); + for (i=0; iop QD:%d\n", i, QD); + if ((rc=posix_memalign((void**)&devs[i].op, 128, QD*sizeof(OP_t)))) + { + fprintf(stderr,"posix_memalign failed, size=%ld, rc=%d\n", + QD*sizeof(OP_t), rc); + exit(0); + } + memset(devs[i].op,0,QD*sizeof(OP_t)); + if ((rc=posix_memalign((void**)&devs[i].rbuf, _4K, _4K*nblocks))) + { + fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n", + _4K*nblocks, rc); + exit(0); + } + if ((rc=posix_memalign((void**)&devs[i].wbuf, _4K, _4K*nblocks))) + { + fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n", + _4K*nblocks, rc); + exit(0); + } + memset(devs[i].wbuf,0x79,_4K*nblocks); + } /*-------------------------------------------------------------------------- * open device and set lun size @@ -235,154 +427,167 @@ int main(int argc, char **argv) } if (!plun) flags = CBLK_OPN_VIRT_LUN; if (!intrp_thds) flags |= CBLK_OPN_NO_INTRP_THREADS; - id = cblk_open(dev, QD, O_RDWR, ext, flags); - if (id == NULL_CHUNK_ID) - { - if (ENOSPC == errno) fprintf(stderr,"cblk_open: ENOSPC\n"); - else if (ENODEV == errno) fprintf(stderr,"cblk_open: ENODEV\n"); - else fprintf(stderr,"cblk_open: errno:%d\n",errno); - cblk_term(NULL,0); - exit(errno); - } - rc = cblk_get_lun_size(id, &nblks, 0); - if (rc) - { - fprintf(stderr, "cblk_get_lun_size failed: errno: %d\n", errno); - exit(errno); - } - if (!plun) + for (i=0; i SET_NBLKS ? SET_NBLKS : nblks; - rc = cblk_set_size(id, nblks, 0); + dev=devs+i; + dev->id = cblk_open(devStrs[i], QD, O_RDWR, ext, flags); + + if (dev->id == NULL_CHUNK_ID) + { + if (ENOSPC == errno) fprintf(stderr,"cblk_open: ENOSPC\n"); + else if (ENODEV == errno) fprintf(stderr,"cblk_open: ENODEV\n"); + else fprintf(stderr,"cblk_open: errno:%d\n",errno); + cblk_term(NULL,0); + exit(errno); + } + + rc = cblk_get_lun_size(dev->id, &lun_size, 0); if (rc) { - fprintf(stderr, "cblk_set_size failed, errno: %d\n", errno); + fprintf(stderr, "cblk_get_lun_size failed: errno: %d\n", errno); exit(errno); } - } - lba = lrand48() % nblks; + if (plun) {dev->last_lba = lun_size-1;} + else + { + dev->last_lba = vlunsize > lun_size ? lun_size-1 : vlunsize-1; + rc = cblk_set_size(dev->id, dev->last_lba+1, 0); + if (rc) + { + fprintf(stderr, "cblk_set_size failed, errno: %d\n", errno); + exit(errno); + } + } + debug("open: %s id:%d nblks:%ld\n", devStrs[i],dev->id,dev->last_lba+1); - /*-------------------------------------------------------------------------- - * alloc data for IO - *------------------------------------------------------------------------*/ - op = malloc(QD*sizeof(OP_t)); - if ((rc=posix_memalign((void**)&rbuf, _4K, _4K*nblocks))) - { - fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n", - _4K*nblocks, rc); - cblk_close(id,0); - cblk_term(NULL,0); - exit(0); + dev->nblocks = nblocks; + BMP_LBA(dev); } - if ((rc=posix_memalign((void**)&wbuf, _4K, _4K*nblocks))) - { - fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n", - _4K*nblocks, rc); - cblk_close(id,0); - cblk_term(NULL,0); - exit(0); - } - memset(wbuf,0x79,_4K*nblocks); - memset(op, 0, QD*sizeof(OP_t)); + + if (devN <= 4) {MAX_POLL = QD<2 ? 1 : QD/2;} + else {MAX_POLL = QD<4 ? 1 : QD/4;} + + sticks = cticks = getticks(); /*-------------------------------------------------------------------------- - * loop running IO until secs expire + * loop running IO for nsecs *------------------------------------------------------------------------*/ - gettimeofday(&start, NULL); - do { - /* setup #read ops and #write ops to send before completing ops */ - if (!RD && !WR) {RD=nRD; WR=100-RD;} - - /*---------------------------------------------------------------------- - * send up to RD reads, as long as the queuedepth N is not max - *--------------------------------------------------------------------*/ - while (TIME && RD && N) - { - rc = cblk_aread(id, rbuf, lba, nblocks, &rtag, NULL, - CBLK_ARW_WAIT_CMD_FLAGS); - if (0 == rc) {OP_BEG(rtag); --RD; --N; BMP_LBA();} - else if (EBUSY == errno) {break;} - else {io_error(id,errno);} - } - /*---------------------------------------------------------------------- - * send up to WR writes, as long as the queuedepth N is not max - *--------------------------------------------------------------------*/ - while (TIME && WR && N) + for (issI=0; issIRD && !dev->WR) {dev->RD=nRD; dev->WR=100-nRD;} + + /*------------------------------------------------------------------ + * send up to RD reads, as long as the queuedepth N is not max + *----------------------------------------------------------------*/ + for (i=0; iRD && dev->issNid, dev->rbuf, dev->lba, dev->nblocks, + &rtag, NULL, CBLK_ARW_WAIT_CMD_FLAGS); + if (0 == rc) {OP_BEG(dev,(dev->op+rtag),READ);} + else if (EBUSY == errno) {break;} + else {IO_ISS_ERR(READ);} + + if (devN==1 && QD==1) {debug("USLEEP_RD\n"); usleep(10);} + } + /*------------------------------------------------------------------ + * send up to WR writes, as long as the queuedepth N is not max + *----------------------------------------------------------------*/ + for (i=0; iWR && dev->issNid, dev->wbuf, dev->lba, dev->nblocks, + &rtag, NULL, CBLK_ARW_WAIT_CMD_FLAGS); + if (0 == rc) {OP_BEG(dev,(dev->op+rtag),WRITE);} + else if (EBUSY == errno) {break;} + else {IO_ISS_ERR(WRITE);} + } } + /* if polling, usleep if #hits are below bar */ + if (QD>1 && !intrp_thds && hits<=hbar) + {debug("USLEEP N:%d\n",N); usleep(10);} + /*---------------------------------------------------------------------- * complete cmds *--------------------------------------------------------------------*/ - for (i=0; iissN; i++, dev->htag++) { if (intrp_thds) { - rc = cblk_aresult(id, &htag, &status, CBLK_ARESULT_BLOCKING | - CBLK_ARESULT_NEXT_TAG); - if (rc != nblocks) {io_error(id,errno);} - OP_END(htag); ++cnt; ++N; - continue; - } + rc = cblk_aresult(dev->id, &dev->htag, &status, + CBLK_ARESULT_BLOCKING | + CBLK_ARESULT_NEXT_TAG); + cticks = getticks(); - if (htag>=QD) htag=0; - if (!op[htag].iss) {continue;} - rc = cblk_aresult(id, &htag, &status, 0); - if (rc == 0) - { - if (QD==1 && ++miss==1) {usleep(80);} - ++tmiss; continue; + op=dev->op+dev->htag; + if (rc != dev->nblocks) {IO_CMP_ERR(op);} + OP_END(dev,op,cticks); + if (++aresN >= COMP) {break;} } - else if (rc < 0) - {io_error(id,errno);} + else + { + dev->htag %= QD; + op = dev->op + dev->htag; + if (!op->iss) {continue;} + if (++aresN > MAX_POLL) {break;} - OP_END(htag); ++cnt; ++N; miss=0; - } + if (pflag==0) {debug("HARVEST id:%d\n", dev->id);} + rc = cblk_aresult(dev->id, &dev->htag, &status, pflag); + cticks = getticks(); - /*---------------------------------------------------------------------- - * at an interval which does not impact performance, check if secs - * have expired, and randomize lba - *--------------------------------------------------------------------*/ - if (cnt > TI) - { - TI += TIME_INTERVAL; - gettimeofday(&delta, NULL); - if (delta.tv_sec - start.tv_sec >= nsecs) {TIME=0; COMP=QD;} - lba = lrand48() % nblks; + OP_CHK_WARN_TO(op, cticks, eto, _10SEC); + if (!pflag) {pflag=CBLK_ARESULT_NO_HARVEST;} + if (rc == 0) {++op->miss; ++tmiss; continue;} + else if (rc < 0) {IO_CMP_ERR(op);} + + OP_CHK_LATENCY(op,lto,cticks); + OP_END(dev,op,cticks); ++hits; + } } + dev_i = (dev_i+1) % devN; + + if (TIME && SDELTA(sticks,ns_per_tick) >= nsecs) {TIME=0;} } - while (TIME || QD-N); + while (TIME || N); + + esecs = SDELTA(sticks,ns_per_tick); /*-------------------------------------------------------------------------- * print IO stats *------------------------------------------------------------------------*/ - gettimeofday(&delta, NULL); - esecs = ((float)((delta.tv_sec*mil + delta.tv_usec) - - (start.tv_sec*mil + start.tv_usec))) / (float)mil; - printf("d:%s r:%d q:%d s:%d p:%d n:%d i:%d miss:%d lat:%d mbps:%d iops:%d", - dev, nRD, QD, nsecs, plun, nblocks, intrp_thds, tmiss, + printf("r:%d q:%d s:%d p:%d n:%d i:%d v:%d eto:%d miss:%d/%d \ +lat:%d mbps:%d iops:%d", + nRD, QD, nsecs, plun, nblocks, + intrp_thds, verbose, eto, tmiss/cnt, tmiss, (uint32_t)((tlat*ns_per_tick)/cnt/1000), - (uint32_t)((float)((cnt*nblocks*4)/1024)/esecs), - (uint32_t)((float)(cnt/esecs))); - if (plun && nblocks > 1) - printf(" 4k-iops:%d", (uint32_t)((float)(cnt*nblocks)/esecs)); - printf("\n"); + ((cnt*nblocks*4)/1024)/esecs, + cnt/esecs); + if (plun && nblocks > 1) {printf(" 4k-iops:%d", (cnt*nblocks)/esecs);} + if (verbose) {printf(" rlat:%d wlat:%d", + (uint32_t)((tlat_rd*ns_per_tick)/cnt_rd/1000), + (uint32_t)((tlat_wr*ns_per_tick)/cnt_wr/1000));} + printf("\n"); fflush(stdout); +exit: /*-------------------------------------------------------------------------- * cleanup *------------------------------------------------------------------------*/ - free(op); - free(rbuf); - free(wbuf); - cblk_close(id,0); + for (i=0; istat.status == -1) {io_error(id, "status==-1", -1);} else if (lio[tag].ioP[i]->stat.status ==CBLK_ARW_STATUS_SUCCESS) - {++cnt; ++lio[tag].ncmp; miss=0;} + {++cnt; ++lio[tag].ncmp;} else if (lio[tag].ioP[i]->stat.status == CBLK_ARW_STATUS_FAIL) {io_error(id, "CMD FAIL",lio[tag].ioP[i]->stat.fail_errno);} else if (lio[tag].ioP[i]->stat.status ==CBLK_ARW_STATUS_PENDING) @@ -348,7 +341,7 @@ int main(int argc, char **argv) if (lio[tag].ioI[i]->stat.status == -1) {io_error(id, "status==-1", -1);} else if (lio[tag].ioI[i]->stat.status ==CBLK_ARW_STATUS_SUCCESS) - {++cnt; ++lio[tag].ncmp; miss=0;} + {++cnt; ++lio[tag].ncmp;} else if (lio[tag].ioI[i]->stat.status == CBLK_ARW_STATUS_FAIL) {io_error(id, "CMD FAIL",lio[tag].ioI[i]->stat.fail_errno);} else if (lio[tag].ioI[i]->stat.status ==CBLK_ARW_STATUS_PENDING) @@ -358,10 +351,8 @@ int main(int argc, char **argv) } lio[tag].nP = p_i; - if (lio[tag].ncmp == 0) {++miss; ++tmiss;} - else if (lio[tag].ncmp == LD) {miss=0;} - - if (NL==1 && LD==1 && !lio[tag].ncmp && miss==1) {usleep(80);} + if (lio[tag].ncmp == 0) {++tmiss;} + if (NL==1 && lio[tag].ncmp == 0) {usleep(20);} N -= lio[tag].ncmp; if (++tag >= NL) tag = 0; @@ -386,8 +377,8 @@ int main(int argc, char **argv) gettimeofday(&delta, NULL); esecs = ((float)((delta.tv_sec*mil + delta.tv_usec) - (start.tv_sec*mil + start.tv_usec))) / (float)mil; - printf("d:%s l:%d c:%d p:%d s:%d i:%d tmiss:%d mbps:%d iops:%d\n", - dev, NL, LD, plun, nsecs, intrp_thds, tmiss, + printf("d:%s l:%d c:%d p:%d s:%d i:%d miss:%d/%d mbps:%d iops:%d\n", + dev, NL, LD, plun, nsecs, intrp_thds, tmiss/cnt, tmiss, (uint32_t)((float)(cnt*4)/1024/esecs), (uint32_t)((float)cnt/esecs)); diff --git a/src/block/test/blockr0io.c b/src/block/test/blockr0io.c new file mode 100644 index 00000000..e8f5c180 --- /dev/null +++ b/src/block/test/blockr0io.c @@ -0,0 +1,614 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/block/test/blockr0io.c $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/** + ******************************************************************************* + * \file + * \brief Block Interface I/O Driver + * \details + * This runs I/O to the capi Block Interface using aread/awrite/aresult. The + * expected iops are 300k-400k per capi card. \n + * Using the queuedepth (-q) option affects iops, as there are less cmds. \n + * Using the physical lun (-p) option affects iops, as it uses only 1 port \n + * Using the -p with blocksize (-n) option also affects iops. \n + * \n + * Examples: \n + * \n + * blockr0io -d RAID0 \n + * r:100 q:300 s:4 p:0 n:1 i:o err:0 mbps:1401 iops:358676 \n + * \n + * blockr0io -d RAID0 -s 20 -q 1 -r 70 -p \n + * r:70 q:1 s:20 p:1 n:1 i:0 err:0 mbps:26 iops:6905 \n + * \n + * blockr0io -d RAID0 -q 10 -n 20 -p \n + * r:100 q:10 s:4 p:1 n:20 i:0 mbps:784 iops:10050 4k-iops:201008 \n + * + ******************************************************************************* + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _OS_INTERNAL +#include +#else +#include +#endif + +#include + +#define _8K 8*1024 +#define _4K 4*1024 +#define NBUFS _8K +#define _1MSEC 1000 +#define _1SEC 1000000 +#define _10SEC 10000000 +#define READ 1 +#define WRITE 0 +#define MAX_WIDTH 8 + +/** +******************************************************************************** +** \brief struct for each issued op +*******************************************************************************/ +typedef struct +{ + int id; + cflsh_cg_tag_t tag; + uint64_t start; + uint64_t lba; + uint32_t cnt; + uint32_t miss; + uint32_t min_to; + uint32_t max_to; + uint32_t iss; + uint32_t rd; +} OP_t; + +uint8_t *rbuf; +uint8_t *wbuf; +OP_t *ops; +int width=0; +int id; +int opI[MAX_WIDTH]={0}; +uint32_t RD; +uint32_t WR; +size_t lba; +size_t last_lba; +uint32_t nblocks; +uint32_t issN[MAX_WIDTH]={0}; + +/** +******************************************************************************** +** \brief print debug msg if DEBUG (-D) +*******************************************************************************/ +#define debug(fmt, ...) \ + do {if (DEBUG) {fprintf(stdout,fmt,##__VA_ARGS__); fflush(stdout);}} while(0) + +/** +******************************************************************************** +** \brief bump lba, reset lba if it wraps +*******************************************************************************/ +#define BMP_LBA() \ + if (seq) \ + { \ + lba += nblocks; \ + if (lba+nblocks >= last_lba) {lba=0;} \ + } \ + else {lba = lrand48() % last_lba;} + +/** +******************************************************************************** +** \brief call to begin an op, init data for the op +*******************************************************************************/ +#define OP_BEG(_tag,_rd) \ + do \ + { \ + int _i = _tag.id % width; \ + OP_t *_op = ops + (_i*QD) + _tag.tag; \ + _op->start = getticks(); \ + _op->iss = 1; \ + _op->id = _tag.id; \ + _op->tag.id = _tag.id; \ + _op->tag.tag = _tag.tag; \ + _op->lba = lba; \ + _op->rd = _rd; \ + _op->cnt = cnt; \ + _op->miss = 0; \ + _op->min_to = 0; \ + _op->max_to = 0; \ + N += 1; \ + issN[_i] += 1; \ + if (_rd==READ) {--RD;} else {--WR;} \ + debug("OP_BEG: ops[%4ld]={%d:%3d} rd:%d lba:%9ld issN[%d]:%4d " \ + "N:%3d cnt:%d RD:%d len:%d\n", \ + (_op)-ops, _tag.id, _tag.tag, (_op->rd==READ), \ + _op->lba, _i, issN[_i], N, cnt, RD, nblocks); \ + BMP_LBA(); \ + } while (0) + +/** +******************************************************************************** +** \brief call to end an op, reset issued flag, save tick delta +*******************************************************************************/ +#define OP_END(_tag, _cticks) \ + do \ + { \ + int _i = _tag.id % width; \ + OP_t *_op = ops + (_i*QD) + _tag.tag; \ + lat = _cticks - _op->start; \ + cnt += 1; \ + _op->iss = 0; \ + tlat += lat; \ + N -= 1; \ + issN[_i] -= 1; \ + if (verbose) \ + { if (_op->rd) {++cnt_rd; tlat_rd+=lat;} \ + else {++cnt_wr; tlat_wr+=lat;} \ + } \ + debug("OP_END: ops[%4ld]={%d:%3d} rd:%d lba:%9ld issN[%d]:%4d N:%3d\n", \ + (_op)-ops, _op->tag.id, _op->tag.tag, \ + (_op->rd==READ), _op->lba, _i, issN[_i], N); \ + } while (0); + +/** +******************************************************************************** +** \brief call to calc delta us +*******************************************************************************/ +#define USTAMP(_usecs, _sticks, _cticks) \ + _usecs=((_cticks-_sticks)*ns_per_tick) / 1000 + +/** +******************************************************************************** +** \brief call to calc delta secs +*******************************************************************************/ +#define TSTAMP(_secs, _sticks, _cticks) \ + _secs=((_cticks-_sticks)*ns_per_tick) / 1000000000 + +/** +******************************************************************************** +** \brief call to check timeout for an op(cmd), print msg for warning TO & TO +*******************************************************************************/ +#define OP_CHK_WARN_TO(_op, _cticks, _min, _max) \ + if (verbose) \ + { \ + lat = ((_cticks - _op->start) * ns_per_tick) / 1000; \ + if (lat > _min && !_op->min_to++) \ + {printf("ETO: pid:%d cnt:%7d lba:%16ld rd:%d miss:%d TIME:%d\n", \ + getpid(), cnt, _op->lba, _op->rd, \ + _op->miss, lat);} \ + else if (lat > _max && !_op->max_to++) \ + {printf("TO: pid:%d cnt:%7d lba:%16ld rd:%d miss:%d TIME:%d\n", \ + getpid(), cnt, _op->lba, _op->rd, \ + _op->miss, lat);} \ + } + +/** +******************************************************************************** +** \brief call to check latency for each op, print msg for LONG cmds +*******************************************************************************/ +#define OP_CHK_LATENCY(_op, _lto, _cticks) \ + if (verbose) \ + { \ + lat = ((_cticks - _op->start) * ns_per_tick) / 1000; \ + if (lat > _lto) \ + {printf("LTO: pid:%d cnt:%7d lba:%16ld rd:%d miss:%d TIME:%d\n", \ + getpid(), cnt, _op->lba, _op->rd, \ + _op->miss, lat);} \ + } + +/** +******************************************************************************** +** \brief print errno and exit +** \details +** An IO has failed, print the errno and exit +*******************************************************************************/ +#define IO_ISS_ERR(_type) \ + fprintf(stderr, "io_iss_error: errno:%d pid:%d cnt:%7d rd:%d\n", \ + errno, getpid(), cnt, (READ==_type)); \ + goto exit; + +#define IO_CMP_ERR(_tag) \ + do \ + { \ + int _i = _tag.id % width; \ + OP_t *_op = ops + (_i*QD) + _tag.tag; \ + fprintf(stderr, "io_cmp_error: errno:%d pid:%d cnt:%7d lba:%16ld " \ + "rd:%d\n", \ + errno, getpid(), _op->cnt, _op->lba, (READ==_op->rd)); \ + goto exit; \ + } while (0); + +/** +******************************************************************************** +** \brief print the usage +** \details +** -d device *the device name to run Block IO \n +** -r %rd *the percentage of reads to issue (0..100) \n +** -q queuedepth *the number of outstanding ops to maintain \n +** -n nblocks *the number of 4k blocks in each I/O request \n +** -s secs *the number of seconds to run the I/O \n +** -e eto *early timeout: microseconds, print warning for each op \n +** -l lto *long timeout, microseconds, print elapse time for each op \n +** -S vlunsize *size in gb for the vlun \n +** -p *run in physical lun mode \n +** -i *run using interrupts, not polling \n +** -R *randomize lbas, default is on, use -R0 to run sequential \n +** -v *print cmd timeout warnings \n +*******************************************************************************/ +void usage(void) +{ + printf("Usage:\n"); + printf(" \ +[-d device:device] [-r %%rd] [-q queuedepth] [-n nblocks] [-s secs] [-e eto] \ +[-l lto] [-S vlunsize] [-p] [-i] [-R0] [-v]\n"); + exit(0); +} + +/** +******************************************************************************** +** \brief main +** \details +** process input parms \n +** open device \n +** alloc memory \n +** loop running IO until secs expire \n +** print IO stats \n +** cleanup +*******************************************************************************/ +int main(int argc, char **argv) +{ + char *devStr = NULL; + OP_t *op = NULL; + char FF = 0xFF; + char c = '\0'; + int flags = CBLK_GROUP_RAID0; + int i=0,j=0, rc= 0; + char *_secs = NULL; + char *_QD = NULL; + char *_RD = NULL; + char *_nblocks = NULL; + char *_vlunsize = NULL; + char *_eto = NULL; + char *_lto = NULL; + char *_R = NULL; + uint32_t plun = 0; + uint32_t nsecs = 4; + uint32_t QD = 240; + uint32_t QDT = 0; + uint32_t nRD = 100; + uint32_t eto = 1000000; + uint32_t lto = 10000; + uint32_t intrp_thds = 0; + cflsh_cg_tag_t rtag = {-1,-1}; + int pflag = 0; + size_t lun_size = 0; + uint32_t nblocks = 1; + uint32_t vlunsize = 1; + uint32_t cnt = 0; + uint32_t cnt_rd = 0; + uint32_t cnt_wr = 0; + uint32_t tmiss = 0; + uint32_t hits = 1; + uint32_t hbar = 0; + uint64_t status = 0; + uint32_t N = 0; + uint32_t TIME = 1; + uint32_t COMP = 0; + uint32_t lat = 0; + uint32_t usecs = 0; + uint32_t esecs = 0; + uint32_t isecs = 0; + uint32_t seq = 0; + uint32_t aresN = 0; + uint32_t verbose = 0; + uint32_t DEBUG = 0; + uint32_t MAX_POLL = 100; + uint64_t tlat = 0; + uint64_t tlat_rd = 0; + uint64_t tlat_wr = 0; + uint64_t sticks = 0; + uint64_t cticks = 0; + double ns_per_tick = 0; + uint32_t cur = 0; + uint32_t wI = 0; + + /*-------------------------------------------------------------------------- + * process and verify input parms + *------------------------------------------------------------------------*/ + while (FF != (c=getopt(argc, argv, "d:r:q:n:s:e:S:R:l:phivD"))) + { + switch (c) + { + case 'd': devStr = optarg; break; + case 'r': _RD = optarg; break; + case 'q': _QD = optarg; break; + case 'n': _nblocks = optarg; break; + case 'S': _vlunsize = optarg; break; + case 's': _secs = optarg; break; + case 'e': _eto = optarg; break; + case 'l': _lto = optarg; break; + case 'R': _R = optarg; break; + case 'p': plun = 1; break; + case 'i': intrp_thds = 1; break; + case 'D': DEBUG = 1; break; + case 'v': verbose = 1; break; + case 'h': + case '?': usage(); break; + } + } + if (_secs) nsecs = atoi(_secs); + if (_QD) QD = atoi(_QD); + if (_nblocks) nblocks = atoi(_nblocks); + if (_vlunsize) vlunsize = atoi(_vlunsize); + if (_RD) nRD = atoi(_RD); + if (_eto) eto = atoi(_eto); + if (_lto) lto = atoi(_lto); + else if (QD>1) lto = 20000; + if (_R && _R[0]=='0') seq= 1; + + QD = (QD < _8K) ? QD : _8K; + nRD = (nRD < 100) ? nRD : 100; + + if (nblocks > 1) + { + printf("nblocks > 1 is not supported yet\n"); + usage(); + } + if (plun && vlunsize > 1) + { + printf("error: <-S %d> can only be used with a virtual lun\n",vlunsize); + usage(); + } + if (!plun && nblocks > 1) + { + printf("error: <-n %d> can only be used with a physical lun\n",nblocks); + usage(); + } + if (devStr == NULL) usage(); + + vlunsize = vlunsize*256*1024; + + srand48(time(0)); + ns_per_tick = time_per_tick(1000, 100); + + /*-------------------------------------------------------------------------- + * open device and set lun size + *------------------------------------------------------------------------*/ + rc = cblk_init(NULL,0); + if (rc) + { + fprintf(stderr,"cblk_init failed rc = %d and errno = %d\n", rc,errno); + exit(1); + } + if (!plun) flags |= CBLK_OPN_VIRT_LUN; + if (!intrp_thds) flags |= CBLK_OPN_NO_INTRP_THREADS; + + debug("start open: %s flags:0x%08x\n", devStr, flags); + id = cblk_cg_open(devStr, QD, O_RDWR, 1, 0, flags); + + if (id == NULL_CHUNK_ID) + { + if (ENOSPC == errno) fprintf(stderr,"cblk_open: ENOSPC\n"); + else if (ENODEV == errno) fprintf(stderr,"cblk_open: ENODEV\n"); + else fprintf(stderr,"cblk_open: errno:%d\n",errno); + cblk_term(NULL,0); + exit(errno); + } + + rc = cblk_cg_get_lun_size(id, &lun_size, CBLK_GROUP_RAID0); + if (rc) + { + fprintf(stderr, "cblk_get_lun_size failed: errno: %d\n", errno); + exit(errno); + } + if (plun) {last_lba = lun_size-1;} + else + { + last_lba = vlunsize > lun_size ? lun_size-1 : vlunsize-1; + rc = cblk_cg_set_size(id, last_lba+1, CBLK_GROUP_RAID0); + if (rc) + { + fprintf(stderr, "cblk_set_size failed, errno: %d\n", errno); + exit(errno); + } + } + + /*-------------------------------------------------------------------------- + * alloc data + *------------------------------------------------------------------------*/ + width = cblk_cg_get_num_chunks(id, 0); + QDT = QD*width; + if ((rc=posix_memalign((void**)&ops, 128, QDT*sizeof(OP_t)))) + { + fprintf(stderr,"posix_memalign failed, size=%ld, rc=%d\n", + QDT*sizeof(OP_t), rc); + exit(0); + } + memset(ops, 0, QDT*sizeof(OP_t)); + + if ((rc=posix_memalign((void**)&rbuf, _4K, _4K*nblocks))) + { + fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n", + _4K*nblocks, rc); + exit(0); + } + if ((rc=posix_memalign((void**)&wbuf, _4K, _4K*nblocks))) + { + fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n", + _4K*nblocks, rc); + exit(0); + } + memset(wbuf,0x79,_4K*nblocks); + if (!seq) {BMP_LBA();} + + hbar = QD<20 ? 1 : 2; + COMP = QD<10 ? 1 : QD/10; + MAX_POLL = QD<48 ? 4 : (QD/width)*3/4; + + sticks = cticks = getticks(); + + debug("open: %s id:%d width:%d nblks:%ld\n", devStr, id, width, last_lba+1); + + /*-------------------------------------------------------------------------- + * loop running IO for nsecs + *------------------------------------------------------------------------*/ + do + { + /* setup #read ops and #write ops to send */ + if (!RD && !WR) {RD=nRD; WR=100-nRD;} + + /*------------------------------------------------------------------ + * send up to RD reads, as long as the queuedepth N is not max + *----------------------------------------------------------------*/ + for (i=0; i1 && !intrp_thds && hitsiss) {continue;} + + if (++aresN > MAX_POLL) {break;} + + if (!(pflag&CBLK_ARESULT_NO_HARVEST)) + {debug("HARVEST %d\n", op->id);} + rc = cblk_cg_aresult(id, &op->tag, &status, pflag); + cticks = getticks(); + + OP_CHK_WARN_TO(op, cticks, eto, _10SEC); + if (!(pflag&CBLK_ARESULT_NO_HARVEST)) + {pflag|=CBLK_ARESULT_NO_HARVEST;} + if (rc == 0) {++op->miss; ++tmiss; continue;} + else if (rc < 0) {IO_CMP_ERR(op->tag);} + + OP_CHK_LATENCY(op,lto,cticks); + OP_END(op->tag,cticks); ++hits; + } + } + wI=(wI+1)%width; + } + + USTAMP(usecs, sticks, cticks); + + /* check time expiration every second */ + if (usecs > isecs) + { + isecs += _1SEC; + TSTAMP(esecs, sticks, cticks); + if (esecs >= nsecs) {TIME=0; debug("TIME'S UP: N:%d\n", N);} + } + } + while (TIME || N); + + cticks = getticks(); + TSTAMP(esecs, sticks, cticks); + + /*-------------------------------------------------------------------------- + * print IO stats + *------------------------------------------------------------------------*/ + printf("r:%d q:%d s:%d p:%d n:%d i:%d v:%d eto:%d amiss:%d tmiss:%d \ +lat:%d mbps:%d iops:%d", + nRD, QD, nsecs, plun, nblocks, + intrp_thds, verbose, eto, tmiss/cnt, tmiss, + (uint32_t)((tlat*ns_per_tick)/cnt/1000), + ((cnt*nblocks*4)/1024)/esecs, + cnt/esecs); + if (plun && nblocks > 1) {printf(" 4k-iops:%d", (cnt*nblocks)/esecs);} + if (verbose) {printf(" rlat:%d wlat:%d", + (uint32_t)((tlat_rd*ns_per_tick)/cnt_rd/1000), + (uint32_t)((tlat_wr*ns_per_tick)/cnt_wr/1000));} + printf("\n"); fflush(stdout); + +exit: + /*-------------------------------------------------------------------------- + * cleanup + *------------------------------------------------------------------------*/ + free(ops); + free(rbuf); + free(wbuf); + debug("cblk_close id:%d\n", id); + cblk_cg_close(id,CBLK_GROUP_RAID0); + cblk_term(NULL,0); + return rc; +} diff --git a/src/block/test/blockr0sio.c b/src/block/test/blockr0sio.c new file mode 100644 index 00000000..a7c2374d --- /dev/null +++ b/src/block/test/blockr0sio.c @@ -0,0 +1,397 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/block/test/blocksio.c $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/** + ******************************************************************************* + * \file + * \brief Block Interface Sync I/O Driver + * \details + * This runs I/O to the capi Block Interface using read/write. The \n + * expected iops are 300k-400k per card. \n + * Using the queuedepth (-q) option affects iops, as there are less cmds. \n + * Using the -p with blocksize (-n) option also affects iops. \n + * \n + * Examples: \n + * \n + * blocksio -d /dev/sg10 \n + * d:/dev/sg10 r:100 q:300 s:4 p:0 n:1 i:o err:0 mbps:1401 iops:358676 \n + * \n + * blocksio -d /dev/sg10 -s 20 -q 1 -r 70 -p \n + * d:/dev/sg10 r:70 q:1 s:20 p:1 n:1 i:0 err:0 mbps:26 iops:6905 \n + * \n + * blocksio -d /dev/sg34 -q 10 -n 20 -p \n + * d:/dev/sg34 r:100 q:10 s:4 p:1 n:20 i:0 \n + * mbps:784 iops:10050 4k-iops:201008 \n + * + ******************************************************************************* + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _OS_INTERNAL +#include +#else +#include +#endif + +#include + +#define _8K 8*1024 +#define _4K 4*1024 +#define NBUFS _8K +#define _1MSEC 1000 +#define _1SEC 1000000 +#define _10SEC 10000000 +#define READ 1 +#define WRITE 0 +#define MAX_SGDEVS 8 +#define CFLASH_R0_MAX_QD 2*4096 + +int id = 0; +uint64_t tlat = 0; +uint32_t seq = 0; +size_t nblks = 0; +size_t nblocks = 1; +uint32_t tcnt = 0; +uint32_t nRD = 100; +uint64_t start_ticks = 0; +double ns_per_tick = 0; +uint32_t nsecs = 4; +char *devStr = NULL; +uint32_t DEBUG = 0; +uint32_t plun = 0; + +pthread_mutex_t lock; + +/** +******************************************************************************** +** \brief print errno and exit +** \details +** An IO has failed, print the errno and exit +*******************************************************************************/ +#define IO_ERR(_rd, _lba, _errno) \ + fprintf(stderr, "io_error: errno:%d pid:%d lba:%8d rd:%d\n", \ + _errno, getpid(), _lba, _rd); \ + return; + +/** +******************************************************************************** +** \brief bump lba, reset lba if it wraps +*******************************************************************************/ +#define BMP_LBA() if (seq) \ + { \ + lba+=nblocks; \ + if (lba+nblocks >= nblks) {lba=0;} \ + } \ + else {lba=lrand48()%nblks;} + +#define debug(fmt, ...) \ + do { if (DEBUG) fprintf(stdout,fmt,##__VA_ARGS__); fflush(stdout);} while (0) + +/** +******************************************************************************** +** \brief print the usage +** \details +** -d device *the device name to run Block IO \n +** -r %rd *the percentage of reads to issue (0..100) \n +** -q queuedepth *the number of outstanding ops to maintain \n +** -n nblocks *the number of 4k blocks in each I/O request \n +** -s secs *the number of seconds to run the I/O \n +** -S vlunsize *size in gb for the vlun \n +** -p *run in physical lun mode \n +** -i *run using interrupts, not polling \n +** -R *randomize lbas, default is on, use -R0 to run sequential \n +*******************************************************************************/ +void usage(void) +{ + printf("Usage:\n"); + printf(" [-d device] [-r %%rd] [-q queuedepth] [-n nblocks] [-s secs] \ +[-S vlunsize] [-p] [-i] [-R0]\n"); + exit(0); +} + +/** +******************************************************************************** +** \brief run sync IO in a thread +*******************************************************************************/ +void run_sync(void *p) +{ + int tid = (uint32_t)(uint64_t)p; + int rc = 0; + uint8_t *rbuf = NULL; + uint8_t *wbuf = NULL; + uint32_t lba = 0; + uint64_t stime = 0; + uint64_t sticks = 0; + uint64_t cticks = 0; + uint32_t RD = 0; + uint32_t WR = 0; + uint32_t cnt = 0; + uint64_t lat = 0; + int flags = 0; + + if (!seq) {lba = lrand48() % nblks;} + flags = CBLK_GROUP_RAID0; + + /*-------------------------------------------------------------------------- + * alloc data for IO + *------------------------------------------------------------------------*/ + if ((rc=posix_memalign((void**)&rbuf, _4K, _4K*nblocks))) + { + fprintf(stderr,"posix_memalign failed, size=%ld, rc=%d\n", + _4K*nblocks, rc); + cblk_term(NULL,0); + exit(0); + } + if ((rc=posix_memalign((void**)&wbuf, _4K, _4K*nblocks))) + { + fprintf(stderr,"posix_memalign failed, size=%ld, rc=%d\n", + _4K*nblocks, rc); + cblk_term(NULL,0); + exit(0); + } + memset(wbuf,0x79,_4K*nblocks); + + debug("start tid:%d\n", tid); + + /*-------------------------------------------------------------------------- + * loop running IO for nsecs + *------------------------------------------------------------------------*/ + stime = getticks(); + do + { + /* setup #read ops and #write ops to send */ + if (!RD && !WR) {RD=nRD; WR=100-RD;} + + if (RD) + { + debug(" read: id:%d lba:%d\n", id, lba); + sticks=getticks(); + rc = cblk_cg_read(id, rbuf, lba, nblocks, flags); + if (nblocks == rc) + { + cticks=getticks()-sticks; + --RD; BMP_LBA(); + lat +=cticks; + ++cnt; + } + else if (EBUSY != errno) {IO_ERR(READ,lba,errno);} + } + if (WR) + { + debug(" write: id:%d lba:%d\n", id, lba); + sticks=getticks(); + rc = cblk_cg_write(id, wbuf, lba, nblocks, flags); + if (nblocks == rc) + { + cticks=getticks()-sticks; + --WR; BMP_LBA(); + lat +=cticks; + ++cnt; + } + else if (EBUSY != errno) {IO_ERR(WRITE,lba,errno);} + } + } while (SDELTA(stime,ns_per_tick) < nsecs); + + pthread_mutex_lock(&lock); + tlat += lat; + tcnt += cnt; + pthread_mutex_unlock(&lock); + + free(rbuf); + free(wbuf); + debug("exiting id:%d\n", id); + return; +} + +/** +******************************************************************************** +** \brief main +** \details +** process input parms \n +** open device \n +** alloc memory \n +** loop running IO until secs expire \n +** print IO stats \n +** cleanup +*******************************************************************************/ +int main(int argc, char **argv) +{ + char *devStr = NULL; + char FF = 0xFF; + char c = '\0'; + int flags = CBLK_GROUP_RAID0; + int i,rc = 0; + char *_secs = NULL; + char *_QD = NULL; + char *_RD = NULL; + char *_nblocks = NULL; + char *_vlunsize = NULL; + char *_R = NULL; + uint32_t intrp_thds = 0; + uint32_t pths = 0; + uint32_t QD = 128; + uint32_t vlunsize = 1; + uint32_t esecs = 0; + chunk_ext_arg_t ext = 0; + + /*-------------------------------------------------------------------------- + * process and verify input parms + *------------------------------------------------------------------------*/ + while (FF != (c=getopt(argc, argv, "d:r:q:n:s:S:R:phiD"))) + { + switch (c) + { + case 'd': devStr = optarg; break; + case 'r': _RD = optarg; break; + case 'q': _QD = optarg; break; + case 'n': _nblocks = optarg; break; + case 'S': _vlunsize = optarg; break; + case 's': _secs = optarg; break; + case 'R': _R = optarg; break; + case 'i': intrp_thds = 1; break; + case 'p': plun = 1; break; + case 'D': DEBUG = 1; break; + case 'h': + case '?': usage(); break; + } + } + if (_secs) nsecs = atoi(_secs); + if (_QD) QD = atoi(_QD); + if (_nblocks) nblocks = atoi(_nblocks); + if (_vlunsize) vlunsize = atoi(_vlunsize); + if (_RD) nRD = atoi(_RD); + if (_R && _R[0]=='0') seq= 1; + + if (QD > _8K) QD = _8K; + if (nRD > 100) nRD = 100; + + if (plun && vlunsize > 1) + { + printf("error: <-S %d> can only be used with a vlun\n", vlunsize); + usage(); + } + if (!plun && nblocks > 1) + { + printf("error: <-n %ld> can only be used with a plun\n", nblocks); + usage(); + } + if (devStr == NULL) usage(); + + vlunsize = vlunsize*256*1024; + + srand48(time(0)); + ns_per_tick = time_per_tick(1000, 100); + + pths = (QD > _8K) ? _8K : QD; + + /*-------------------------------------------------------------------------- + * open device and set lun size + *------------------------------------------------------------------------*/ + rc = cblk_init(NULL,0); + if (rc) + { + fprintf(stderr,"cblk_init failed with rc = %d and errno = %d\n", + rc,errno); + exit(1); + } + if (!plun) {flags |= CBLK_OPN_VIRT_LUN;} + if (!intrp_thds) {flags |= CBLK_OPN_NO_INTRP_THREADS;} + + id = cblk_cg_open(devStr, CFLASH_R0_MAX_QD, O_RDWR, 1, ext, flags); + if (id == NULL_CHUNK_ID) + { + if (ENOSPC == errno) fprintf(stderr,"cblk_open: ENOSPC\n"); + else if (ENODEV == errno) fprintf(stderr,"cblk_open: ENODEV\n"); + else fprintf(stderr,"cblk_open: errno:%d\n",errno); + cblk_term(NULL,0); + exit(errno); + } + + rc = cblk_cg_get_lun_size(id, &nblks, CBLK_GROUP_RAID0); + if (rc) + { + fprintf(stderr, "cblk_cg_get_lun_size failed: errno: %d\n", errno); + exit(errno); + } + if (!plun) + { + nblks = vlunsize > nblks ? nblks : vlunsize; + rc = cblk_cg_set_size(id, nblks, CBLK_GROUP_RAID0); + if (rc) + { + fprintf(stderr, "cblk_cg_set_size failed, errno: %d\n", errno); + exit(errno); + } + } + debug("open: %s id:%d nblks:%ld\n", devStr, id, nblks); + pthread_mutex_init(&lock,NULL); + + /*-------------------------------------------------------------------------- + * create threads to run sync IO + *------------------------------------------------------------------------*/ + start_ticks = getticks(); + + pthread_t pth[_8K]; + void* (*fp)(void*) = (void*(*)(void*))run_sync; + + for (i=0; i 1) {printf(" 4k-iops:%ld", (tcnt*nblocks)/esecs);} + printf("\n"); fflush(stdout); + + debug("close %d\n", id); + cblk_cg_close(id,CBLK_GROUP_RAID0); + cblk_term(NULL,0); + return 0; +} + diff --git a/src/block/test/blocksio.c b/src/block/test/blocksio.c new file mode 100644 index 00000000..ab682fd2 --- /dev/null +++ b/src/block/test/blocksio.c @@ -0,0 +1,415 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/block/test/blocksio.c $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/** + ******************************************************************************* + * \file + * \brief Block Interface Sync I/O Driver + * \details + * This runs I/O to the capi Block Interface using read/write. The \n + * expected iops are 300k-400k per card. \n + * Using the queuedepth (-q) option affects iops, as there are less cmds. \n + * Using the -p with blocksize (-n) option also affects iops. \n + * \n + * Examples: \n + * \n + * blocksio -d /dev/sg10 \n + * d:/dev/sg10 r:100 q:300 s:4 p:0 n:1 i:o err:0 mbps:1401 iops:358676 \n + * \n + * blocksio -d /dev/sg10 -s 20 -q 1 -r 70 -p \n + * d:/dev/sg10 r:70 q:1 s:20 p:1 n:1 i:0 err:0 mbps:26 iops:6905 \n + * \n + * blocksio -d /dev/sg34 -q 10 -n 20 -p \n + * d:/dev/sg34 r:100 q:10 s:4 p:1 n:20 i:0 \n + * mbps:784 iops:10050 4k-iops:201008 \n + * + ******************************************************************************* + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _OS_INTERNAL +#include +#else +#include +#endif + +#include + +#define _8K 8*1024 +#define _4K 4*1024 +#define NBUFS _8K +#define _1MSEC 1000 +#define _1SEC 1000000 +#define _10SEC 10000000 +#define READ 1 +#define WRITE 0 +#define MAX_SGDEVS 8 + +int id = 0; +uint64_t tlat = 0; +uint32_t seq = 0; +size_t nblks = 0; +uint32_t nblocks = 1; +uint32_t tcnt = 0; +uint32_t nRD = 100; +uint64_t start_ticks = 0; +double ns_per_tick = 0; +uint32_t nsecs = 4; +char devStrs[MAX_SGDEVS][64]; +char *devStr = NULL; +char *pstr = NULL; +uint32_t devN = 0; +uint32_t DEBUG = 0; +uint32_t plun = 0; +int ids[_4K]; + +pthread_mutex_t lock; + +/** +******************************************************************************** +** \brief print errno and exit +** \details +** An IO has failed, print the errno and exit +*******************************************************************************/ +#define IO_ERR(_rd, _lba, _errno) \ + fprintf(stderr, "io_error: errno:%d pid:%d lba:%8d rd:%d\n", \ + _errno, getpid(), _lba, _rd); \ + return; + +/** +******************************************************************************** +** \brief bump lba, reset lba if it wraps +*******************************************************************************/ +#define BMP_LBA() if (seq) \ + { \ + lba+=nblocks; \ + if (lba+nblocks >= nblks) {lba=0;} \ + } \ + else {lba=lrand48()%nblks;} + +#define debug(fmt, ...) \ + do {if (DEBUG) {fprintf(stdout,fmt,##__VA_ARGS__); fflush(stdout);}} while(0) + +/** +******************************************************************************** +** \brief print the usage +** \details +** -d device *the device name to run Block IO \n +** -r %rd *the percentage of reads to issue (0..100) \n +** -q queuedepth *the number of outstanding ops to maintain \n +** -n nblocks *the number of 4k blocks in each I/O request \n +** -s secs *the number of seconds to run the I/O \n +** -S vlunsize *size in gb for the vlun \n +** -p *run in physical lun mode \n +** -i *run using interrupts, not polling \n +** -R *randomize lbas, default is on, use -R0 to run sequential \n +*******************************************************************************/ +void usage(void) +{ + printf("Usage:\n"); + printf(" [-d device] [-r %%rd] [-q queuedepth] [-n nblocks] " + "[-s secs] [-S vlunsize] [-p] [-i] [-R0]\n"); + exit(0); +} + +/** +******************************************************************************** +** \brief run sync IO in a thread +*******************************************************************************/ +void run_sync(void *p) +{ + int tid = (uint32_t)(uint64_t)p; + int cid = ids[tid % devN]; + int rc = 0; + int flags = 0; + uint8_t *rbuf = NULL; + uint8_t *wbuf = NULL; + uint32_t lba = 0; + uint64_t stime = 0; + uint64_t sticks = 0; + uint64_t cticks = 0; + uint32_t RD = 0; + uint32_t WR = 0; + uint32_t cnt = 0; + uint64_t lat = 0; + + if (!seq) {lba = lrand48() % nblks;} + if (plun) {flags = CBLK_GROUP_ID;} + + /*-------------------------------------------------------------------------- + * alloc data for IO + *------------------------------------------------------------------------*/ + if ((rc=posix_memalign((void**)&rbuf, _4K, _4K*nblocks))) + { + fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n", + _4K*nblocks, rc); + cblk_term(NULL,0); + exit(0); + } + if ((rc=posix_memalign((void**)&wbuf, _4K, _4K*nblocks))) + { + fprintf(stderr,"posix_memalign failed, size=%d, rc=%d\n", + _4K*nblocks, rc); + cblk_term(NULL,0); + exit(0); + } + memset(wbuf,0x79,_4K*nblocks); + + debug("start tid:%d cid:%d\n", tid, cid); + + /*-------------------------------------------------------------------------- + * loop running IO for nsecs + *------------------------------------------------------------------------*/ + stime = getticks(); + do + { + /* setup #read ops and #write ops to send */ + if (!RD && !WR) {RD=nRD; WR=100-RD;} + + if (RD) + { + debug("read cid:%d lba:%d\n", cid, lba); + sticks=getticks(); + rc = cblk_read(cid, rbuf, lba, nblocks, flags); + if (nblocks == rc) + { + cticks=getticks()-sticks; + --RD; BMP_LBA(); + lat +=cticks; + ++cnt; + } + else if (EBUSY != errno) {IO_ERR(READ,lba,errno);} + } + if (WR) + { + debug("write cid:%d lba:%d\n", cid, lba); + sticks=getticks(); + rc = cblk_write(cid, rbuf, lba, nblocks,flags); + if (nblocks == rc) + { + cticks=getticks()-sticks; + --WR; BMP_LBA(); + lat +=cticks; + ++cnt; + } + else if (EBUSY != errno) {IO_ERR(WRITE,lba,errno);} + } + } while (SDELTA(stime,ns_per_tick) < nsecs); + + pthread_mutex_lock(&lock); + tlat += lat; + tcnt += cnt; + pthread_mutex_unlock(&lock); + + free(rbuf); + free(wbuf); + debug("exiting cid:%d\n", cid); + return; +} + +/** +******************************************************************************** +** \brief main +** \details +** process input parms \n +** open device \n +** alloc memory \n +** loop running IO until secs expire \n +** print IO stats \n +** cleanup +*******************************************************************************/ +int main(int argc, char **argv) +{ + char *devStr = NULL; + char FF = 0xFF; + char c = '\0'; + chunk_ext_arg_t ext = 1; + int flags = 0; + int i,rc = 0; + char *_secs = NULL; + char *_QD = NULL; + char *_RD = NULL; + char *_nblocks = NULL; + char *_vlunsize = NULL; + char *_R = NULL; + uint32_t intrp_thds = 0; + uint32_t pths = 0; + uint32_t QD = 128; + uint32_t vlunsize = 1; + uint32_t esecs = 0; + + /*-------------------------------------------------------------------------- + * process and verify input parms + *------------------------------------------------------------------------*/ + while (FF != (c=getopt(argc, argv, "d:r:q:n:s:S:R:phiD"))) + { + switch (c) + { + case 'd': devStr = optarg; break; + case 'r': _RD = optarg; break; + case 'q': _QD = optarg; break; + case 'n': _nblocks = optarg; break; + case 'S': _vlunsize = optarg; break; + case 's': _secs = optarg; break; + case 'R': _R = optarg; break; + case 'i': intrp_thds = 1; break; + case 'p': plun = 1; break; + case 'D': DEBUG = 1; break; + case 'h': + case '?': usage(); break; + } + } + if (_secs) nsecs = atoi(_secs); + if (_QD) QD = atoi(_QD); + if (_nblocks) nblocks = atoi(_nblocks); + if (_vlunsize) vlunsize = atoi(_vlunsize); + if (_RD) nRD = atoi(_RD); + if (_R && _R[0]=='0') seq= 1; + + if (QD > _8K) QD = _8K; + if (nRD > 100) nRD = 100; + + if (plun && vlunsize > 1) + { + printf("error: <-S %d> can only be used with a virtual lun\n",vlunsize); + usage(); + } + if (!plun && nblocks > 1) + { + printf("error: <-n %d> can only be used with a physical lun\n",nblocks); + usage(); + } + if (devStr == NULL) usage(); + + vlunsize = vlunsize*256*1024; + + srand48(time(0)); + ns_per_tick = time_per_tick(1000, 100); + + while ((pstr=strsep(&devStr,":"))) + { + if (devN == MAX_SGDEVS) {break;}; + debug("%s\n",pstr); + sprintf(devStrs[devN++],"%s",pstr); + } + + pths = (devN*QD > _8K) ? _8K : devN*QD; + + /*-------------------------------------------------------------------------- + * open device and set lun size + *------------------------------------------------------------------------*/ + rc = cblk_init(NULL,0); + if (rc) + { + fprintf(stderr,"cblk_init failed with rc = %d and errno = %d\n", + rc,errno); + exit(1); + } + if (plun) {flags = CBLK_OPN_GROUP; ext=9;} + else {flags = CBLK_OPN_VIRT_LUN;} + if (!intrp_thds) {flags |= CBLK_OPN_NO_INTRP_THREADS;} + + for (i=0; i nblks ? nblks : vlunsize; + rc = cblk_set_size(ids[i], nblks, flags); + if (rc) + { + fprintf(stderr, "cblk_set_size failed, errno: %d\n", errno); + exit(errno); + } + } + debug("open: %s id:%d nblks:%ld\n", devStrs[i%devN], ids[i], nblks); + } + pthread_mutex_init(&lock,NULL); + + /*-------------------------------------------------------------------------- + * create threads to run sync IO + *------------------------------------------------------------------------*/ + start_ticks = getticks(); + + pthread_t pth[_8K]; + void* (*fp)(void*) = (void*(*)(void*))run_sync; + + for (i=0; i 1) {printf(" 4k-iops:%d", (tcnt*nblocks)/esecs);} + printf("\n"); fflush(stdout); + + if (plun) {flags=CBLK_GROUP_ID;} + for (i=0; i +#include + +extern "C" +{ +#include +uint64_t block_number = 0; +int mode = O_RDWR; +int num_opens = 0; +uint32_t thread_count = 0; +uint64_t max_xfer = 0; +int num_loops; +int thread_flag; +int num_threads; +int virt_lun_flags=0; +int share_cntxt_flags=0; +int io_bufcnt = 1; +int num_listio = 500; +int test_max_cntx = 0; +int host_type = 0; +chunk_id_t chunks[MAX_OPENS+15]; +void *blk_fvt_data_buf = NULL; +void *blk_fvt_comp_data_buf = NULL; +char *env_filemode = getenv("BLOCK_FILEMODE_ENABLED"); +char *env_max_xfer = getenv("CFLSH_BLK_MAX_XFER"); +char *env_num_cntx = getenv("MAX_CNTX"); +extern char dev_paths[MAX_LUNS][128]; +extern chunk_ext_arg_t ext; +} + +#define MAX_NUM_CMDS 8192 + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_open) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_multi_opens) +{ + chunk_id_t id[4] = {0}; + int flags = CBLK_GROUP_ID; + int max_reqs = 64; + int rc = 0; + int i = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + + for (i=0; i<10; i++) + { + if (env_filemode && atoi(env_filemode)==1) {ext=1;} else {ext=1;} + id[0] = cblk_cg_open(dev_paths[0], max_reqs, mode, ext, 0, flags); + ASSERT_NE(NULL_CHUNK_ID, id[0]); + if (env_filemode && atoi(env_filemode)==1) {ext=1;} else {ext=2;} + id[1] = cblk_cg_open(dev_paths[0], max_reqs, mode, ext, 0, flags); + ASSERT_NE(NULL_CHUNK_ID, id[1]); + if (env_filemode && atoi(env_filemode)==1) {ext=1;} else {ext=8;} + id[2] = cblk_cg_open(dev_paths[0], max_reqs, mode, ext, 0, flags); + ASSERT_NE(NULL_CHUNK_ID, id[2]); + if (env_filemode && atoi(env_filemode)==1) {ext=1;} else {ext=32;} + id[3] = cblk_cg_open(dev_paths[0], max_reqs, mode, ext, 0, flags); + ASSERT_NE(NULL_CHUNK_ID, id[3]); + + rc = cblk_close(id[2], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[0], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[3], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[1], flags); + EXPECT_EQ(0, rc); + } +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_plun_open_READONLY) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + uint64_t lba; + int io_flags = CBLK_GROUP_ID; + size_t nblks; + int cmd; + int ret = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + + mode = O_RDONLY; + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id ); + + cmd = FV_WRITE; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + cmd = FV_READ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_EQ(1 , ret); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_plun_open_WRONLY) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + uint64_t lba; + int io_flags = CBLK_GROUP_ID; + size_t nblks; + int cmd; + int ret = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + + mode = O_WRONLY; + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + cmd = FV_READ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + cmd = FV_WRITE; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_EQ(1, ret); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_open_RDWR) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + uint64_t lba; + int io_flags = CBLK_GROUP_ID; + size_t nblks; + int cmd; + int ret = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + EXPECT_EQ(0, er_no); + + cmd = FV_WRITE; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_EQ(1, ret); + + cmd = FV_READ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_EQ(1, ret); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/****************************************************************************** + * \brief + ******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_open_exceed_max_reqs) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_ID; + int max_reqs = MAX_NUM_CMDS; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + EXPECT_NE(NULL_CHUNK_ID, id); + + if (id == NULL_CHUNK_ID) + { + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + return; + } + + max_reqs = MAX_NUM_CMDS + 1; + blk_open_tst( &id, max_reqs, &er_no, open_cnt, flags, mode); + + // expect failure + EXPECT_EQ(NULL_CHUNK_ID, id); + EXPECT_EQ(12, er_no); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/****************************************************************************** + * \brief + ******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_plun_open_Max_context_tst) +{ + int open_flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int ret = 0; + int max_cntxt = 508; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + + ext = 4; + + #ifdef _AIX + max_cntxt = 494; + #else + if (host_type==HOST_TYPE_VM) {max_cntxt = 502;} + #endif + + max_cntxt /= ext; + max_context(&ret, &er_no, max_reqs, max_cntxt, open_flags,mode); + EXPECT_EQ(0, ret); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/******************************************************************************* + * \brief + ******************************************************************************/ + TEST(Block_FVT_Suite, BLK_API_FVT_CG_plun_open_Exceed_Max_context_tst) + { + int open_flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int ret = 0; + int max_cntxt = 508; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + + #ifdef _AIX + max_cntxt = 495; /* Should fail on 495 */ + #else + if (host_type==HOST_TYPE_VM) {max_cntxt = 503;} + #endif + + max_cntxt /= ext; + max_cntxt += 1; + max_context(&ret, &er_no, max_reqs, max_cntxt, open_flags,mode); + + EXPECT_NE(0, ret); + + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/******************************************************************************* + * \brief + ******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_vlun_open_not_allowed) +{ + chunk_id_t id = 0; + int flags = CBLK_OPN_VIRT_LUN | CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + + ASSERT_EQ(NULL_CHUNK_ID, id); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_open_invalid_dev_path) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + strcpy(dev_paths[0],"junk"); + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + + EXPECT_EQ(NULL_CHUNK_ID, id); + + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_close_unopen) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int close_flag = CBLK_GROUP_ID; + int ret = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + + ASSERT_NE(NULL_CHUNK_ID, id); + + blk_close_tst(id, &ret, &er_no, close_flag); + + EXPECT_EQ(0, ret); + + // Close un-opened lun + blk_close_tst(id, &ret, &er_no, close_flag); + + EXPECT_EQ(EINVAL, er_no); + EXPECT_EQ(-1, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_get_size) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_ID; + int sz_flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + size_t phys_lun_sz = 0; + int get_set_flag = 0; // 0 = get phys + // 1 = get chunk + // 2 = set size + size_t ret_size = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst( &id, max_reqs, &er_no, open_cnt, flags, mode); + + ASSERT_NE(NULL_CHUNK_ID, id); + + // Get phys lun size + blk_fvt_get_set_lun_size(id, &phys_lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + + EXPECT_EQ(0, ret); + EXPECT_NE((size_t)0, phys_lun_sz); + + // get size on phys lun should report whole phys lun size + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &ret_size, sz_flags, get_set_flag, + &ret, &er_no); + + EXPECT_EQ(0, ret); + + // test chunk size should be equal whole phys lun size + EXPECT_EQ(phys_lun_sz , ret_size); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_set_size) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_ID; + int sz_flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + size_t lun_sz = 0; + int get_set_flag = 0; // 0 = get phys + // 1 = get chunk + // 2 = set size + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst( &id, max_reqs, &er_no, open_cnt, flags, mode); + + ASSERT_NE(NULL_CHUNK_ID, id ); + + // set size on phy lun, it should fail, since it can not be changed + get_set_flag = 2; + lun_sz = 4096; + + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, er_no); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_read_wo_setsz) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + size_t nblks; + uint64_t lba; + int io_flags = CBLK_GROUP_ID; + int cmd; // 0 = read, 1, write + + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + + ASSERT_NE(NULL_CHUNK_ID, id); + + cmd = FV_READ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + + // expect good ret code + EXPECT_EQ(1, ret); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_1M_thru_16M_xfersize) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_ID; + int sz_flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba = 0; + int io_flags = CBLK_GROUP_ID; + size_t nblks = 4096; + size_t lun_sz = 0; + size_t xfersz = 0; + int cmd = 0; + size_t i = 0; + int get_set_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(nblks)); + ext = 4; + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + + ASSERT_NE(NULL_CHUNK_ID, id); + + get_set_flag = 0; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, &ret, &er_no); + EXPECT_EQ(0, ret); + + if (lun_sz < nblks) {nblks = lun_sz;} + + for (i = 256; i < nblks ; i+= 256) + { + xfersz = i; + cmd = FV_WRITE; + lba = 0; + + blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); + + EXPECT_EQ(xfersz, (size_t)ret); + if ((int)xfersz != ret) { + fprintf(stderr,"Write failed xfersz 0x%lx\n",i); + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); + return; + } + + cmd = FV_READ; + lba = 0; + // read what was wrote xfersize + blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); + + EXPECT_EQ(xfersz, (size_t)ret); + if ((int)xfersz != ret) { + fprintf(stderr,"Read failed xfersz 0x%lx\n",i); + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); + return; + } + // compare buffers + + blk_fvt_cmp_buf(xfersz, &ret); + if (ret != 0) { + fprintf(stderr,"Compare failed xfersz 0x%lx\n",i); + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); + } + ASSERT_EQ(0, ret); + } + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_rd_wr_outside_lunsz) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_ID; + int sz_flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba; + size_t temp_sz,nblks; + int io_flags = CBLK_GROUP_ID; + int cmd; // 0 = read, 1, write + int get_set_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + temp_sz = 0; + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + + cmd = FV_READ; + lba = temp_sz+1; + nblks = 1; + + // verify read outside lun size fails + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + cmd = FV_WRITE; + // verify write outside lun size fails + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_UMC_plun_sync_blocking_rw_tst) +{ + int open_flags = CBLK_GROUP_ID; + int io_flags = CBLK_GROUP_ID | SYNC_IO_ONLY; + int er_no = 0; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + + ext = 4; + num_loops = 1; + num_threads = 40; + thread_flag = 1; + + blk_thread_tst(&ret, &er_no, open_flags, io_flags); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_UMC_plun_sync_polling_rw_tst) +{ + int open_flags = CBLK_GROUP_ID | CBLK_OPN_NO_INTRP_THREADS; + int io_flags = CBLK_GROUP_ID | SYNC_IO_ONLY; + int er_no = 0; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + + ext = 4; + num_loops = 1; + num_threads = 40; + thread_flag = 1; + + blk_thread_tst(&ret, &er_no, open_flags, io_flags); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_UMC_plun_rd_wr_cmp_get_stats) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_ID; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba; + int io_flags = CBLK_GROUP_ID; + size_t nblks; + int cmd; + chunk_stats_t stats; + + ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + nblks = 1; + for ( lba = 1; lba <= 1000; lba++) + { + cmd = FV_WRITE; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(1 , ret); + + cmd = FV_READ; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(1 , ret); + + // compare buffers + blk_fvt_cmp_buf(nblks, &ret); + EXPECT_EQ(0, ret); + } + + ret = cblk_get_stats (id, &stats, io_flags); + EXPECT_EQ(0 , ret); + EXPECT_TRUE(1000 == stats.num_reads); + EXPECT_TRUE(1000 == stats.num_writes); + EXPECT_TRUE(1000 == stats.num_blocks_read); + EXPECT_TRUE(1000 == stats.num_blocks_written); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_CG_FM_plun_persistence) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_ID; + int max_reqs= 64; + int er_no = 0; + int open_cnt= 1; + int ret = 0; + + uint64_t lba; + int io_flags = CBLK_GROUP_ID; + size_t nblks; + int cmd; + + ASSERT_EQ(0,blk_fvt_setup(256)); + ext = 4; + if (env_filemode && atoi(env_filemode)==1) {ext=1;} + + blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id ); + EXPECT_EQ(0, er_no ); + + // clear physcial lun , write all 0 + // write data at known lba + cmd = FV_WRITE; + lba = 1; + nblks = 1; + + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(1 , ret); + EXPECT_EQ(0, er_no ); + + // read data at known lba + cmd = FV_READ; + lba = 1; + nblks = 1; + + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(1 , ret); + + // compare data is written correctly + ret = memcmp(blk_fvt_data_buf,blk_fvt_comp_data_buf,BLK_FVT_BUFSIZE); + er_no = errno; + EXPECT_EQ(0, ret); + + // close physical lun + cblk_close (id,CBLK_GROUP_ID); + num_opens--; + + // open physical lun again + blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + // read data from known lba + cmd = FV_READ; + lba = 1; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(1 , ret); + + // compare data , it should have persisted after closed lun. + ret = memcmp(blk_fvt_data_buf,blk_fvt_comp_data_buf,BLK_FVT_BUFSIZE); + er_no = errno; + EXPECT_EQ(0 , ret); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_open) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id ); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_open_READONLY) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + uint64_t lba; + int io_flags = CBLK_GROUP_RAID0; + size_t nblks; + int cmd; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + mode = O_RDONLY; + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id ); + + // Try to write , it should fail + cmd = FV_WRITE; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_NE(1 , ret); + + // Try to read it should pass + cmd = FV_READ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_EQ(1 , ret); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/******************************************************************************* + * \brief + ******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_open_WRONLY) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + uint64_t lba; + int io_flags = CBLK_GROUP_RAID0; + size_t nblks; + int cmd; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + mode = O_WRONLY; + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + // Try to read , it should fail + cmd = FV_READ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + // Try to write it should pass + cmd = FV_WRITE; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_EQ(1, ret); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/****************************************************************************** + * \brief + ******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_open_RDWR) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + uint64_t lba; + int io_flags = CBLK_GROUP_RAID0; + size_t nblks; + int cmd; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + //Try to write it should pass + cmd = FV_WRITE; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_EQ(1, ret); + + // Try to read , it should pass + cmd = FV_READ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + EXPECT_EQ(1, ret); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/****************************************************************************** + * \brief + ******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_open_exceed_max_reqs) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0; + int max_reqs = MAX_NUM_CMDS; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + EXPECT_NE(NULL_CHUNK_ID, id); + + if (id == NULL_CHUNK_ID) + { + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + return; + } + + max_reqs = (MAX_NUM_CMDS * cblk_cg_get_num_chunks(id,flags)) + 1; + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + + // expect failure + EXPECT_EQ(NULL_CHUNK_ID, id); + EXPECT_EQ(12, er_no); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_open_invalid_dev_path) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + strcpy(dev_paths[0],"junk"); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + EXPECT_EQ(NULL_CHUNK_ID, id); + EXPECT_NE(0, er_no); + + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_close_unopen) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int close_flag = CBLK_GROUP_RAID0; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + blk_close_tst(id, &ret, &er_no, close_flag); + EXPECT_EQ(0, ret); + + // Close un-opened lun + blk_close_tst(id, &ret, &er_no, close_flag); + EXPECT_EQ(EINVAL, er_no); + EXPECT_EQ(-1, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_get_size) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + size_t phys_lun_sz = 0; + int get_set_flag = 0; // 0 = get phys + // 1 = get chunk + // 2 = set size + size_t ret_size = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst( &id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + // Get phys lun size + blk_fvt_get_set_lun_size(id, &phys_lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_NE((size_t)0, phys_lun_sz); + + // get size on phys lun should report whole phys lun size + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &ret_size, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + // test chunk size should be equal whole phys lun size + EXPECT_EQ(phys_lun_sz, ret_size); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_set_size) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + size_t lun_sz = 0; + int get_set_flag = 0; // 0 = get phys + // 1 = get chunk + // 2 = set size + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst( &id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id ); + + // set size on phy lun, it should fail, since it can not be changed + get_set_flag = 2; + lun_sz = 4096; + + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(-1, ret); + EXPECT_EQ(EINVAL, er_no); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_read_wo_setsz) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + size_t nblks; + uint64_t lba; + int io_flags = CBLK_GROUP_RAID0; + int cmd; // 0 = read, 1, write + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + cmd = FV_READ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(1, ret); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_1M_thru_16M_xfersize) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + int get_set_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz + uint64_t lba; + int io_flags = CBLK_GROUP_RAID0; + size_t nblks = 4096; + size_t lun_sz = 0; + size_t xfersz = 0; + int cmd; + size_t i = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0, blk_fvt_setup(nblks)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + + ASSERT_NE(NULL_CHUNK_ID, id); + + get_set_flag = 0; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, &ret, &er_no); + EXPECT_EQ(0, ret); + + if (lun_sz < nblks) {nblks = lun_sz;} + + for (i = 256; i < nblks ; i+= 256) + { + xfersz = i; + cmd = FV_WRITE; + lba = 0; + + blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(xfersz, (size_t)ret); + if ((int)xfersz != ret) + { + fprintf(stderr,"Write failed xfersz 0x%lx\n",i); + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); + return; + } + + cmd = FV_READ; + lba = 0; + // read what was wrote xfersize + blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); + + EXPECT_EQ(xfersz, (size_t)ret); + if ((int)xfersz != ret) { + fprintf(stderr,"Read failed xfersz 0x%lx\n",i); + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); + return; + } + // compare buffers + blk_fvt_cmp_buf(xfersz, &ret); + if (ret != 0) + { + fprintf(stderr,"Compare failed xfersz 0x%lx\n",i); + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); + } + ASSERT_EQ(0, ret); + } + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_rd_wr_outside_lunsz) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba; + size_t temp_sz,nblks; + int io_flags = CBLK_GROUP_RAID0; + int cmd; // 0 = read, 1, write + int get_set_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + + ASSERT_NE(NULL_CHUNK_ID, id ); + + temp_sz = 0; + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret,&er_no); + EXPECT_EQ(0, ret); + + cmd = FV_READ; + lba = temp_sz+1; + nblks = 1; + + // verify read outside lun size fails + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + cmd = FV_WRITE; + // verify write outside lun size fails + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_sync_blocking_rw_tst) +{ + int flags = CBLK_GROUP_RAID0; + int er_no = 0; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + num_loops = 1; + num_threads = 40; + thread_flag = 1; + + blk_thread_tst(&ret, &er_no, flags, flags); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_sync_polling_rw_tst) +{ + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_NO_INTRP_THREADS; + int io_flags = CBLK_GROUP_RAID0; + int er_no = 0; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + num_loops = 1; + num_threads = 40; + thread_flag = 1; + + blk_thread_tst(&ret, &er_no, open_flags, io_flags); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_rd_wr_cmp_get_stats) +{ + + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba; + int io_flags = CBLK_GROUP_RAID0; + size_t nblks; + int cmd; + chunk_stats_t stats; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + nblks = 1; + for ( lba = 1; lba <= 1000; lba++) + { + cmd = FV_WRITE; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, io_flags); + EXPECT_EQ(1 , ret); + + cmd = FV_READ; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, io_flags); + EXPECT_EQ(1 , ret); + + // compare buffers + blk_fvt_cmp_buf(nblks, &ret); + EXPECT_EQ(0, ret); + } + + ret = cblk_get_stats (id, &stats, io_flags); + EXPECT_EQ(0 , ret); + EXPECT_TRUE(1000 == stats.num_reads); + EXPECT_TRUE(1000 == stats.num_writes); + EXPECT_TRUE(1000 == stats.num_blocks_read); + EXPECT_TRUE(1000 == stats.num_blocks_written); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_persistence) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0; + int max_reqs= 64; + int er_no = 0; + int open_cnt= 1; + int ret = 0; + uint64_t lba; + int io_flags = CBLK_GROUP_RAID0; + size_t nblks; + int cmd; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(256)); + blk_fvt_RAID0_setup(); + + blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id ); + + // clear physcial lun , write all 0 + // write data at known lba + cmd = FV_WRITE; + lba = 1; + nblks = 1; + + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(1 , ret); + + // read data at known lba + cmd = FV_READ; + lba = 1; + nblks = 1; + + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(1 , ret); + + // compare data is written correctly + ret = memcmp(blk_fvt_data_buf,blk_fvt_comp_data_buf,BLK_FVT_BUFSIZE); + er_no = errno; + EXPECT_EQ(0, ret); + + // close physical lun + cblk_close(id,CBLK_GROUP_RAID0); + num_opens--; + + // open physical lun again + blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + // read data from known lba + cmd = FV_READ; + lba = 1; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(1, ret); + + // compare data , it should have persisted after closed lun. + ret = memcmp(blk_fvt_data_buf,blk_fvt_comp_data_buf,BLK_FVT_BUFSIZE); + er_no = errno; + EXPECT_EQ(0, ret); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_async_read_wo_setsz) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba = 0; + size_t nblks = 1; + int io_flags = CBLK_GROUP_RAID0; + int cmd; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + cmd = FV_AREAD; + + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(1, ret); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_async_read_outside_lunsz) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba = 0; + size_t lun_sz,nblks = 1; + int io_flags = CBLK_GROUP_RAID0; + int cmd; // 0 = read, 1, write + int get_set_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + /* get lun size */ + get_set_flag = 0; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + cmd = FV_AREAD; + lba = lun_sz+1; + + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_EQ(EINVAL, er_no); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_rd_wr_greater_than_1_blk) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba = 0; + size_t nblks = 1; + int io_flags = CBLK_GROUP_RAID0; + int cmd; // 0 = read, 1, write + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + cmd = FV_AREAD; + nblks = 2; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_EQ(EINVAL, er_no); + + cmd = FV_AWRITE; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_EQ(EINVAL, er_no); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_async_polling_perf_test) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_NO_INTRP_THREADS; + int max_reqs = 4096; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + int io_flags = CBLK_GROUP_RAID0; + int num_cmds = 4096; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(num_cmds)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + io_perf_tst(id, &ret, &er_no, io_flags); + EXPECT_EQ(0, ret); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_plun_async_blocking_perf_test) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0; + int max_reqs = 4096; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + int io_flags = CBLK_GROUP_RAID0 | \ + CBLK_ARESULT_NEXT_TAG | \ + CBLK_ARESULT_BLOCKING; + int num_cmds = 4096; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(num_cmds)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + io_perf_tst(id, &ret, &er_no, io_flags); + EXPECT_EQ(0, ret); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_open) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + + ASSERT_NE(NULL_CHUNK_ID, id ); + EXPECT_EQ(0, er_no ); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_open_READONLY) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + uint64_t lba; + int io_flags = CBLK_GROUP_RAID0; + size_t nblks; + int cmd; + int ret = 0; + size_t lun_sz = 0; + int get_set_flag = 0; // 0 = get phys + // 1 = get chunk + // 2 = set size + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + mode = O_RDONLY; + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id ); + EXPECT_EQ(0, er_no ); + + get_set_flag = 2; + lun_sz = 4096; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + // Try to write , it should fail + cmd = FV_WRITE; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + + EXPECT_NE(1 , ret); + EXPECT_NE(0 , er_no); + + // Try to read it should pass + cmd = FV_READ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + + EXPECT_EQ(1 , ret); + EXPECT_EQ(0 , er_no); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); +} + +/******************************************************************************* + * \brief + ******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_open_WRONLY) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + uint64_t lba; + int io_flags = CBLK_GROUP_RAID0; + size_t nblks; + int cmd; + int get_set_flag = 2; + size_t lun_sz = 100; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + mode = O_WRONLY; + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + EXPECT_EQ(0, er_no); + + get_set_flag = 2; + lun_sz = 4096; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + // Try to read , it should fail + cmd = FV_READ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + // Try to write it should pass + cmd = FV_WRITE; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + + EXPECT_EQ(1, ret); + EXPECT_EQ(0, er_no); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); +} + +/******************************************************************************* + * \brief + ******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_open_RDWR) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + uint64_t lba; + int io_flags = CBLK_GROUP_RAID0; + size_t nblks; + int cmd; + int get_set_flag = 2; + size_t lun_sz = 100; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + EXPECT_EQ(0, er_no); + + get_set_flag = 2; + lun_sz = 4096; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + //Try to write it should pass + cmd = FV_WRITE; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + + EXPECT_EQ(1 , ret); + EXPECT_EQ(0 , er_no); + + // Try to read , it should pass + cmd = FV_READ ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, flags); + + EXPECT_EQ(1 , ret); + EXPECT_EQ(0 , er_no); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); +} + + +/****************************************************************************** + * \brief + ******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_open_exceed_max_reqs) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int max_reqs = MAX_NUM_CMDS; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + EXPECT_NE(NULL_CHUNK_ID, id); + EXPECT_EQ(0, er_no); + + if (id == NULL_CHUNK_ID) + { + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; + } + + max_reqs = (MAX_NUM_CMDS * cblk_cg_get_num_chunks(id,flags)) + 1; + blk_open_tst(&id, max_reqs, &er_no, open_cnt, flags, mode); + + // expect failure + EXPECT_EQ(NULL_CHUNK_ID, id); + EXPECT_EQ(12, er_no); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_open_invalid_dev_path) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + + ASSERT_EQ(0,blk_fvt_setup(1)); + strcpy(dev_paths[0],"junk"); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + + EXPECT_EQ(NULL_CHUNK_ID, id); + EXPECT_NE(0, er_no); + + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_close_unopen) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int close_flag = CBLK_GROUP_RAID0; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + blk_close_tst(id, &ret, &er_no, close_flag); + + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + + // Close un-opened lun + blk_close_tst(id, &ret, &er_no, close_flag); + + EXPECT_EQ(EINVAL, er_no); + EXPECT_EQ(-1, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_set_get_size) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + size_t _4096 = 4096; + size_t lun_sz = 0; + size_t ret_size = 0; + int get_set_flag = 0; // 0 = get phys + // 1 = get chunk + // 2 = set size + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst( &id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + // Get lun size + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_NE((size_t)0, lun_sz); + + // get size on vlun should report whole phys lun size + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &ret_size, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + /* set lun size to 4096 */ + get_set_flag = 2; + lun_sz = _4096; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + // get size for vlun + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &ret_size, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + // test chunk size should be equal 4096 + EXPECT_EQ(_4096, ret_size); + + /* set lun size to 4097 */ + get_set_flag = 2; + lun_sz = _4096+1; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + // get size for vlun + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &ret_size, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + // test chunk size should be equal 4096 + num_chunks + EXPECT_EQ(_4096 + cblk_cg_get_num_chunks(id, flags), ret_size); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_set_size) +{ + chunk_id_t id = 0; + int flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + size_t lun_sz = 0; + int get_set_flag = 0; // 0 = get phys + // 1 = get chunk + // 2 = set size + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst( &id, max_reqs, &er_no, open_cnt, flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id ); + + get_set_flag = 2; + lun_sz = 4096; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + blk_open_tst_cleanup(flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_read_wo_setsz) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + size_t nblks; + uint64_t lba; + int io_flags = CBLK_GROUP_RAID0; + int cmd; // 0 = read, 1, write + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + cmd = FV_READ; + lba = 0; + nblks = 1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + + // expect fail ret code + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_rd_wr_outside_chunksz) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba; + size_t lun_sz,nblks; + int io_flags = CBLK_GROUP_RAID0; + int cmd; // 0 = read, 1, write + int get_set_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + /* set lun size to 4096 */ + get_set_flag = 2; + lun_sz = 4096; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + cmd = FV_READ; + lba = lun_sz+1; + nblks = 1; + + // verify read outside lun size fails + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + cmd = FV_WRITE; + // verify write outside lun size fails + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_async_rd_wr_wo_setsz) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba = 0; + size_t nblks = 1; + int io_flags = CBLK_GROUP_RAID0; + int cmd; // 0 = read, 1, write + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + cmd = FV_AREAD; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + cmd = FV_AWRITE; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_NE(0, er_no); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_async_rd_wr_outside_lunsz) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba = 0; + size_t lun_sz,nblks = 1; + int io_flags = CBLK_GROUP_RAID0; + int cmd; // 0 = read, 1, write + int get_set_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + /* set lun size to 4096 */ + get_set_flag = 2; + lun_sz = 4096; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + cmd = FV_AREAD; + lba = lun_sz+1; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_EQ(EINVAL, er_no); + + cmd = FV_AWRITE; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_EQ(EINVAL, er_no); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_rd_wr_greater_than_1_blk) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 64; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + uint64_t lba = 0; + size_t lun_sz,nblks = 1; + int io_flags = CBLK_GROUP_RAID0; + int cmd; // 0 = read, 1, write + int get_set_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); -extern "C" + /* set lun size to 4096 */ + get_set_flag = 2; + lun_sz = 4096; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, + &ret, &er_no); + EXPECT_EQ(0, ret); + + cmd = FV_AREAD; + nblks = 2; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_EQ(EINVAL, er_no); + + cmd = FV_AWRITE; + blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + EXPECT_NE(1, ret); + EXPECT_EQ(EINVAL, er_no); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_sync_polling_rw_tst) { -#include "blk_tst.h" -int num_opens = 0; -uint32_t thread_count = 0; -uint64_t block_number; -uint64_t max_xfer = 0; -int num_loops; -int thread_flag; -int num_threads; -int virt_lun_flags=0; -int share_cntxt_flags=0; -int io_bufcnt = 1; -int num_listio = 500; -int test_max_cntx = 0; -chunk_id_t chunks[MAX_OPENS+15]; -void *blk_fvt_data_buf = NULL; -void *blk_fvt_comp_data_buf = NULL; -char *env_filemode = getenv("BLOCK_FILEMODE_ENABLED"); -char *env_max_xfer = getenv("CFLSH_BLK_MAX_XFER"); -char *env_num_cntx = getenv("MAX_CNTX"); + int open_flags = CBLK_GROUP_RAID0 | \ + CBLK_OPN_VIRT_LUN | \ + CBLK_OPN_NO_INTRP_THREADS; + int io_flags = CBLK_GROUP_RAID0; + int er_no = 0; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + num_loops = 1; + num_threads = 40; + thread_flag = 1; + + blk_thread_tst(&ret, &er_no, open_flags, io_flags); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_sync_blocking_rw_tst) +{ + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int io_flags = CBLK_GROUP_RAID0; + int er_no = 0; + int ret = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(1)); + blk_fvt_RAID0_setup(); + + num_loops = 1; + num_threads = 40; + thread_flag = 1; + + blk_thread_tst(&ret, &er_no, open_flags, io_flags); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_async_polling_perf_test) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0 | \ + CBLK_OPN_VIRT_LUN | \ + CBLK_OPN_NO_INTRP_THREADS; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 4096; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + size_t lun_sz = 1; + int io_flags = CBLK_GROUP_RAID0; + int num_cmds = 4096; + int get_set_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(num_cmds)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + /* set lun size */ + get_set_flag = 2; + lun_sz = 10000; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, &ret, &er_no); + EXPECT_EQ(0, ret); + + io_perf_tst(id, &ret, &er_no, io_flags); + EXPECT_EQ(0, ret); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_vlun_async_blocking_perf_test) +{ + chunk_id_t id = 0; + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int sz_flags = CBLK_GROUP_RAID0; + int max_reqs = 4096; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + size_t lun_sz = 1; + int io_flags = CBLK_GROUP_RAID0 | \ + CBLK_ARESULT_NEXT_TAG | \ + CBLK_ARESULT_BLOCKING; + int num_cmds = 4096; + int get_set_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz + TESTCASE_SKIP_IF_FILEMODE; + + ASSERT_EQ(0,blk_fvt_setup(num_cmds)); + blk_fvt_RAID0_setup(); + + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); + ASSERT_NE(NULL_CHUNK_ID, id); + + /* set lun size */ + get_set_flag = 2; + lun_sz = 10000; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, &ret, &er_no); + EXPECT_EQ(0, ret); + + io_perf_tst(id, &ret, &er_no, io_flags); + EXPECT_EQ(0, ret); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_RAID0_clone_chunk) +{ + int open_flags = CBLK_GROUP_RAID0 | CBLK_OPN_VIRT_LUN; + int io_flags = CBLK_GROUP_RAID0; + int er_no = 0; + int ret = 0; + int rc = 0; + + TESTCASE_SKIP_IF_FILEMODE; + + rc = fork_and_clone(&ret, &er_no, mode, open_flags, io_flags); + ASSERT_NE(-1, rc); + EXPECT_EQ(1, ret); + EXPECT_EQ(0, er_no); + + blk_open_tst_cleanup(open_flags, &ret, &er_no); + EXPECT_EQ(0, ret); } -int mode = O_RDWR; +/*****************************************************************/ TEST(Block_FVT_Suite, BLK_API_FVT_FM_open_phys_lun) { @@ -63,21 +2533,91 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_open_phys_lun) int max_reqs = 64; int er_no = 0; int open_cnt = 1; + int ret = 0; // Setup dev name and allocated test buffers - ASSERT_EQ(0,blk_fvt_setup(1)); + ext = 1; + blk_open_tst(&id, max_reqs, &er_no, open_cnt, open_flags, mode); ASSERT_NE(NULL_CHUNK_ID, id ); EXPECT_EQ(0, er_no ); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + +} + +/** +******************************************************************************** +** \brief +*******************************************************************************/ +TEST(Block_FVT_Suite, BLK_API_FVT_FM_plun_multi_opens) +{ + chunk_id_t id[20] = {0}; + int flags = 0; + int max_reqs = 64; + int rc = 0; + int i = 0; + int j = 0; + ASSERT_EQ(0,blk_fvt_setup(1)); + for (i=0; i<100; i++) + { + for (j=0; j<20; j++) + { + id[j] = cblk_open(dev_paths[0], max_reqs, mode, 0, flags); + ASSERT_NE(NULL_CHUNK_ID, id[j]); + } + rc = cblk_close(id[0], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[2], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[4], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[6], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[8], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[10], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[12], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[14], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[16], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[18], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[19], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[1], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[17], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[3], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[15], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[5], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[13], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[7], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[11], flags); + EXPECT_EQ(0, rc); + rc = cblk_close(id[9], flags); + EXPECT_EQ(0, rc); + } } + + TEST(Block_FVT_Suite, BLK_API_FVT_open_phys_lun_RDONLY_mode_test) { chunk_id_t id = 0; @@ -92,19 +2632,16 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_phys_lun_RDONLY_mode_test) size_t nblks; int cmd; + ASSERT_EQ(0,blk_fvt_setup(1)); mode = O_RDONLY; // Setup dev name and allocated test buffers - - ASSERT_EQ(0,blk_fvt_setup(1)); - blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); ASSERT_NE(NULL_CHUNK_ID, id ); EXPECT_EQ(0, er_no ); - // Try to write , it should fail cmd = FV_WRITE; lba = 0; @@ -123,9 +2660,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_phys_lun_RDONLY_mode_test) EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - - blk_open_tst_cleanup(); - + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } TEST(Block_FVT_Suite, BLK_API_FVT_open_phys_lun_WRONLY_mode_test) @@ -142,12 +2679,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_phys_lun_WRONLY_mode_test) size_t nblks; int cmd; - mode = O_WRONLY; - - // Setup dev name and allocated test buffers - ASSERT_EQ(0,blk_fvt_setup(1)); + // Setup dev name and allocated test buffers + mode = O_WRONLY; blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); ASSERT_NE(NULL_CHUNK_ID, id ); @@ -173,7 +2708,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_phys_lun_WRONLY_mode_test) EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -191,12 +2729,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_open_phys_lun_RDWR_mode_test) size_t nblks; int cmd; - mode = O_RDWR; - - // Setup dev name and allocated test buffers - ASSERT_EQ(0,blk_fvt_setup(1)); + // Setup dev name and allocated test buffers blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); ASSERT_NE(NULL_CHUNK_ID, id ); @@ -222,7 +2757,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_open_phys_lun_RDWR_mode_test) EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -235,7 +2773,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_virt_lun_RDONLY_mode_test) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -244,12 +2782,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_virt_lun_RDONLY_mode_test) size_t temp_sz,nblks; int cmd; - mode = O_RDONLY; - - // Setup dev name and allocated test buffers - ASSERT_EQ(0,blk_fvt_setup(1)); + // Setup dev name and allocated test buffers + mode = O_RDONLY; blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); ASSERT_NE(NULL_CHUNK_ID, id ); @@ -258,8 +2794,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_virt_lun_RDONLY_mode_test) temp_sz = 1; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -282,7 +2818,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_virt_lun_RDONLY_mode_test) EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -295,7 +2834,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_virt_lun_WRONLY_mode_test) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -304,12 +2843,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_virt_lun_WRONLY_mode_test) size_t temp_sz,nblks; int cmd; - mode = O_WRONLY; - - // Setup dev name and allocated test buffers - ASSERT_EQ(0,blk_fvt_setup(1)); + // Setup dev name and allocated test buffers + mode = O_WRONLY; blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); ASSERT_NE(NULL_CHUNK_ID, id ); @@ -318,8 +2855,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_virt_lun_WRONLY_mode_test) temp_sz = 1; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -342,7 +2879,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_open_virt_lun_WRONLY_mode_test) EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -355,7 +2895,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_virt_lun_RDWR_mode_test) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -364,12 +2904,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_virt_lun_RDWR_mode_test) size_t temp_sz,nblks; int cmd; - mode = O_RDWR; - - // Setup dev name and allocated test buffers - ASSERT_EQ(0,blk_fvt_setup(1)); + // Setup dev name and allocated test buffers blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); ASSERT_NE(NULL_CHUNK_ID, id ); @@ -378,8 +2915,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_virt_lun_RDWR_mode_test) temp_sz = 1; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -397,13 +2934,12 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_virt_lun_RDWR_mode_test) lba = 0; nblks = 1; blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); - EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - - blk_open_tst_cleanup(); - + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_virt_lun) @@ -414,6 +2950,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_virt_lun) int max_reqs = 64; int er_no = 0; int open_cnt = 1; + int ret = 0; ASSERT_EQ(0,blk_fvt_setup(1)); @@ -423,7 +2960,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_virt_lun) EXPECT_EQ(0, er_no ); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } /** Need to check max_requests no, if valid */ @@ -434,6 +2974,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_open_phys_lun_exceed_max_reqs) int max_reqs = 8192; int er_no = 0; int open_cnt = 1; + int ret = 0; ASSERT_EQ(0,blk_fvt_setup(1)); @@ -445,7 +2986,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_open_phys_lun_exceed_max_reqs) EXPECT_EQ(0, er_no ); if (id == NULL_CHUNK_ID) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -458,7 +3002,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_open_phys_lun_exceed_max_reqs) EXPECT_EQ(12, er_no ); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -470,6 +3017,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_virt_lun_exceed_max_reqs) int max_reqs = 8192; int er_no = 0; int open_cnt = 1; + int ret = 0; ASSERT_EQ(0,blk_fvt_setup(1)); @@ -482,7 +3030,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_virt_lun_exceed_max_reqs) EXPECT_EQ(0, er_no ); if (id == NULL_CHUNK_ID) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -495,7 +3046,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_virt_lun_exceed_max_reqs) EXPECT_EQ(12, er_no ); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -511,19 +3065,23 @@ TEST(Block_FVT_Suite, BLK_API_FVT_UMC_Max_context_tst) int max_cntxt = 508; + ASSERT_EQ(0,blk_fvt_setup(1)); + #ifdef _AIX - max_cntxt = 494; + max_cntxt = 494; +#else + if (host_type==HOST_TYPE_VM) {max_cntxt = 502;} #endif - - ASSERT_EQ(0,blk_fvt_setup(1)); - max_context(&ret, &er_no, max_reqs, max_cntxt, open_flags,mode); EXPECT_EQ(0, ret ); EXPECT_EQ(0, er_no ); - blk_open_tst_cleanup (); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -539,13 +3097,17 @@ TEST(Block_FVT_Suite, BLK_API_FVT_UMC_Exceed_Max_context_tst) int ret = 0; int max_cntxt = 509; -#ifdef _AIX - max_cntxt = 496; /* Should fail on 495 */ -#endif + TESTCASE_SKIP_IF_FILEMODE; ASSERT_EQ(0,blk_fvt_setup(1)); +#ifdef _AIX + max_cntxt = 495; /* Should fail on 495 */ +#else + if (host_type==HOST_TYPE_VM) {max_cntxt = 503;} +#endif + if (test_max_cntx ) { max_cntxt = test_max_cntx; } @@ -555,8 +3117,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_UMC_Exceed_Max_context_tst) EXPECT_NE(0, ret ); EXPECT_NE(0, er_no ); - blk_open_tst_cleanup (); - + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_invalid_dev_path) @@ -568,19 +3131,18 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_invalid_dev_path) const char *dev_path = "junk"; int er_no = 0; int open_cnt = 1; - - + int ret = 0; ASSERT_EQ(0,blk_fvt_setup(1)); - blk_open_tst_inv_path(dev_path, &id, max_reqs, &er_no, open_cnt, open_flags, mode); EXPECT_EQ(NULL_CHUNK_ID, id); EXPECT_NE(0, er_no); - blk_open_tst_cleanup(); - + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_open_close_phys_lun) @@ -707,7 +3269,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_get_lun_size_physical) int open_cnt= 1; int ret = 0; size_t phys_lun_sz = 0; - int get_set_size_flag = 0; // 0 = get phys + int get_set_flag = 0; // 0 = get phys // 1 = get chunk // 2 = set size size_t ret_size = 0; @@ -721,15 +3283,15 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_get_lun_size_physical) // Get phys lun size - blk_fvt_get_set_lun_size(id, &phys_lun_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &phys_lun_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); EXPECT_NE((size_t)0, phys_lun_sz); // get size on phys lun should report whole phys lun size - get_set_size_flag = 1; - blk_fvt_get_set_lun_size(id, &ret_size, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &ret_size, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -737,7 +3299,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_get_lun_size_physical) // test chunk size should be equal whole phys lun size EXPECT_EQ(phys_lun_sz , ret_size); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_get_lun_siz_virtual) @@ -750,7 +3315,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_get_lun_siz_virtual) int open_cnt= 1; int ret = 0; size_t lun_sz = 0; - int get_set_size_flag = 0; // 0 = get phys + int get_set_flag = 0; // 0 = get phys // 1 = get chunk // 2 = set size @@ -765,15 +3330,18 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_get_lun_siz_virtual) // Get virtual lun size , it should fail, since it is not set - get_set_size_flag = 1; + get_set_flag = 1; - blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); EXPECT_EQ((size_t)0, lun_sz); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -788,7 +3356,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_set_size_physical) int open_cnt= 1; int ret = 0; size_t lun_sz = 0; - int get_set_size_flag = 0; // 0 = get phys + int get_set_flag = 0; // 0 = get phys // 1 = get chunk // 2 = set size @@ -802,15 +3370,18 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_set_size_physical) // Try to set size on phy lun , it should fail, since it can not be change - get_set_size_flag = 2; + get_set_flag = 2; lun_sz = 4096; - blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(-1 , ret); EXPECT_EQ(EINVAL , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -825,7 +3396,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_set_size_virt_lun) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys + int get_set_flag = 0; // 0 = get phys // 1 = get chunk // 2 = set size size_t chunk_sz = 0; @@ -843,25 +3414,28 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_set_size_virt_lun) // Try to set size on virt lun e - get_set_size_flag = 2; + get_set_flag = 2; chunk_sz = 4096; - blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); // Try to get size on virt lun and verify the set size - get_set_size_flag = 1; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); EXPECT_EQ(chunk_sz, temp_sz); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -877,7 +3451,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_set_invalid_chunk_size) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys + int get_set_flag = 0; // 0 = get phys // 1 = get chunk // 2 = set chunk size @@ -893,9 +3467,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_set_invalid_chunk_size) ASSERT_NE(NULL_CHUNK_ID, id ); - get_set_size_flag = 1; // get phy lun size as 1 chunk + get_set_flag = 1; // get phy lun size as 1 chunk - blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -914,15 +3488,16 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_set_invalid_chunk_size) // Set size > lun size temp_sz = lun_sz + 1; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(-1 , ret); EXPECT_EQ(EINVAL ,er_no); - blk_open_tst_cleanup(); - + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // TODO current block layer code donot support this functions @@ -937,7 +3512,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_decrease_chunk_size) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -955,9 +3530,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_decrease_chunk_size) // Get virt lun size , it sould be 0 - get_set_size_flag = 1; + get_set_flag = 1; - blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -966,8 +3541,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_decrease_chunk_size) // set chunk size (100) temp_sz = chunk_sz + 100; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); @@ -975,8 +3550,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_decrease_chunk_size) // Get the chunk size and Verify it set correctly - get_set_size_flag = 1; - blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); @@ -987,8 +3562,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_decrease_chunk_size) // set size () temp_sz += 20; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); @@ -997,8 +3572,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_decrease_chunk_size) // Get size and Verify it is increased correctly to 220 chunk_sz = 0; - get_set_size_flag = 1; - blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); @@ -1009,8 +3584,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_decrease_chunk_size) // set size () temp_sz = (chunk_sz - 10); - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); @@ -1018,8 +3593,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_decrease_chunk_size) // Get the size and Verify it is decreased correctly to 210 chunk_sz = 0; - get_set_size_flag = 1; - blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); @@ -1027,7 +3602,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_decrease_chunk_size) EXPECT_EQ(temp_sz, chunk_sz); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -1045,7 +3623,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_size_preserve_data) uint64_t lba; int cmd; // 0 = read, 1, write int io_flags = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -1065,8 +3643,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_size_preserve_data) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -1081,8 +3659,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_size_preserve_data) // Increase virt lun size 1 additional block temp_sz = 2; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -1099,7 +3677,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_increase_size_preserve_data) blk_fvt_cmp_buf (nblks, &ret); EXPECT_EQ(0,ret); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -1118,7 +3699,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_get_chunk_status) int io_flags = 0; size_t chunk_sz = 0; int cmd; // 0 = read, 1, write - int get_set_size_flag = 0; + int get_set_flag = 0; @@ -1131,10 +3712,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_get_chunk_status) // Try to set size on virt lun e - get_set_size_flag = 2; + get_set_flag = 2; chunk_sz = 4096; - blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &chunk_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -1152,7 +3733,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_get_chunk_status) EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -1189,12 +3773,15 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_read_phys_lun_wo_setsz) blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); - // expect fail ret code + // expect good ret code EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -1234,7 +3821,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_virt_lun_wo_setsz) EXPECT_NE(1 , ret); EXPECT_NE(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -1251,7 +3841,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_vir_lun_outside_lunsz) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz uint64_t lba; @@ -1269,9 +3859,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_vir_lun_outside_lunsz) temp_sz = 1; lba = 0; - get_set_size_flag = 2; + get_set_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -1292,7 +3882,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_vir_lun_outside_lunsz) EXPECT_NE(1 , ret); EXPECT_NE(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -1312,7 +3905,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_read_phys_lun_outside_lunsz) size_t temp_sz,nblks; int io_flags = 0; int cmd; // 0 = read, 1, write - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -1325,8 +3918,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_read_phys_lun_outside_lunsz) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 0; - get_set_size_flag = 1; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -1342,7 +3935,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_read_phys_lun_outside_lunsz) EXPECT_NE(1 , ret); EXPECT_NE(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Verify read fails when tried to read more than 1 block size @@ -1359,7 +3955,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_greater_than_1_blk) size_t temp_sz, nblks; int cmd; // 0 = read, 1, write int io_flags = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -1372,8 +3968,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_greater_than_1_blk) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -1389,7 +3985,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_greater_than_1_blk) EXPECT_NE(2 , ret); EXPECT_EQ(22 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -1405,7 +4004,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_write_compare) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -1414,8 +4013,6 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_write_compare) size_t temp_sz,nblks; int cmd; - - ASSERT_EQ(0,blk_fvt_setup(1)); // open virtual lun @@ -1424,8 +4021,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_write_compare) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -1449,7 +4046,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_write_compare) EXPECT_EQ(0, ret); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Run I/O with xfersize 1M thru 16M @@ -1460,37 +4060,22 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_1M_thru_16M_xfersize_physical) int open_flags = 0; int sz_flags= 0; int max_reqs= 64; - int er_no = 0; - int open_cnt= 1; - int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz - // 1 = get chunk sz - // 2 = set chunk sz - - uint64_t lba; - int io_flags = 0; - size_t nblks = 0; - size_t lun_sz = 0; - size_t xfersz = 0; - int cmd; - size_t i = 0; - - - if ((env_filemode) && (atoi(env_filemode) == 1)) { - if (env_max_xfer && (atoi(env_max_xfer))) - nblks = atoi(env_max_xfer); - else { - /* - * This test will not work filemode and no CFLSH_BLK_MAX_XFER set. - */ - TESTCASE_SKIP("env CFLSH_BLK_MAX_XFER is _not_ set"); - return; + int er_no = 0; + int open_cnt= 1; + int ret = 0; + int get_set_flag = 0; // 0 = get phys lun sz + // 1 = get chunk sz + // 2 = set chunk sz - } - } else { - nblks = 4096; /* 256 = 1 M, 4096 = 16M */ - } + uint64_t lba; + int io_flags = 0; + size_t nblks = 4096; + size_t lun_sz = 0; + size_t xfersz = 0; + int cmd; + size_t i = 0; + TESTCASE_SKIP_IF_FILEMODE; ASSERT_EQ(0,blk_fvt_setup(nblks)); @@ -1499,10 +4084,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_1M_thru_16M_xfersize_physical) ASSERT_NE(NULL_CHUNK_ID, id ); - get_set_size_flag = 0; - blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 0; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); - EXPECT_EQ(0 , er_no); if (lun_sz < nblks) nblks = lun_sz; @@ -1517,7 +4101,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_1M_thru_16M_xfersize_physical) EXPECT_EQ(xfersz, (size_t)ret); if ((int)xfersz != ret) { fprintf(stderr,"Write failed xfersz 0x%lx\n",i); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); return; } @@ -1529,7 +4114,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_1M_thru_16M_xfersize_physical) EXPECT_EQ(xfersz, (size_t)ret); if ((int)xfersz != ret) { fprintf(stderr,"Read failed xfersz 0x%lx\n",i); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); return; } // compare buffers @@ -1537,12 +4123,16 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_1M_thru_16M_xfersize_physical) blk_fvt_cmp_buf(xfersz, &ret); if (ret != 0) { fprintf(stderr,"Compare failed xfersz 0x%lx\n",i); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); } ASSERT_EQ(0, ret); } - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Run I/O with 1M xfersize @@ -1557,7 +4147,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_1M_xfersize_physical) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -1567,6 +4157,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_1M_xfersize_physical) size_t lun_sz = 0; size_t xfersz = 0; int cmd; + if ((env_filemode) && (atoi(env_filemode) == 1)) { if (env_max_xfer && (atoi(env_max_xfer))) nblks = atoi(env_max_xfer); @@ -1582,7 +4173,6 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_1M_xfersize_physical) nblks = 256; /* 256 = 1 M, 4096 = 16M */ } - ASSERT_EQ(0,blk_fvt_setup(nblks)); // open physical lun @@ -1590,48 +4180,59 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_1M_xfersize_physical) ASSERT_NE(NULL_CHUNK_ID, id ); - get_set_size_flag = 0; - blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 0; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); - if (lun_sz < nblks) - nblks = lun_sz; + if (lun_sz < nblks) {nblks = lun_sz;} - xfersz = nblks; - cmd = FV_WRITE; - lba = 1; + xfersz = nblks; + cmd = FV_WRITE; + lba = 1; - blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); - - EXPECT_EQ(xfersz, (size_t)ret); - if ((int)xfersz != ret) { - fprintf(stderr,"Write failed 1M xfersz \n"); - blk_open_tst_cleanup(); - return; - } + blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); - cmd = FV_READ; - lba = 1; - // read what was wrote xfersize - blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(xfersz, (size_t)ret); + if ((int)xfersz != ret) { + fprintf(stderr,"Write failed 1M xfersz \n"); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); - EXPECT_EQ(xfersz, (size_t)ret); - if ((int)xfersz != ret) { - fprintf(stderr,"Read failed 1M xfersz \n"); - blk_open_tst_cleanup(); - return; - } - // compare buffers - - blk_fvt_cmp_buf(xfersz, &ret); - if (ret != 0) { - fprintf(stderr,"Compare failed 1M xfersz \n"); - blk_open_tst_cleanup(); - } - ASSERT_EQ(0, ret); + return; + } + + cmd = FV_READ; + lba = 1; + // read what was wrote xfersize + blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); + + EXPECT_EQ(xfersz, (size_t)ret); + if ((int)xfersz != ret) { + fprintf(stderr,"Read failed 1M xfersz \n"); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + + return; + } + // compare buffers + + blk_fvt_cmp_buf(xfersz, &ret); + if (ret != 0) { + fprintf(stderr,"Compare failed 1M xfersz \n"); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + + } + ASSERT_EQ(0, ret); + + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); } // Run I/O with variable xfersize , use env_max_xfer if set @@ -1646,7 +4247,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_large_xfer_test) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -1675,8 +4276,6 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_large_xfer_test) nblks = 513; /* 2M +*/ } - - ASSERT_EQ(0,blk_fvt_setup(nblks)); // open physical lun @@ -1684,48 +4283,59 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_large_xfer_test) ASSERT_NE(NULL_CHUNK_ID, id ); - get_set_size_flag = 0; - blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 0; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); - if (lun_sz < nblks) - nblks = lun_sz; + if (lun_sz < nblks) {nblks = lun_sz;} - xfersz = nblks; - cmd = FV_WRITE; - lba = 1; + xfersz = nblks; + cmd = FV_WRITE; + lba = 1; - blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); - - EXPECT_EQ(xfersz, (size_t)ret); - if ((int)xfersz != ret) { - fprintf(stderr,"Write failed 1M+ xfersz \n"); - blk_open_tst_cleanup(); - return; - } + blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); - cmd = FV_READ; - lba = 1; - // read what was wrote xfersize - blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); + EXPECT_EQ(xfersz, (size_t)ret); + if ((int)xfersz != ret) { + fprintf(stderr,"Write failed 1M+ xfersz \n"); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); - EXPECT_EQ(xfersz, (size_t)ret); - if ((int)xfersz != ret) { - fprintf(stderr,"Read failed 1M+ xfersz \n"); - blk_open_tst_cleanup(); - return; - } - // compare buffers - - blk_fvt_cmp_buf(xfersz, &ret); - if (ret != 0) { - fprintf(stderr,"Compare failed 1M+ xfersz \n"); - blk_open_tst_cleanup(); - } - ASSERT_EQ(0, ret); + return; + } + + cmd = FV_READ; + lba = 1; + // read what was wrote xfersize + blk_fvt_io(id, cmd, lba, xfersz, &ret, &er_no, io_flags, open_flags); + + EXPECT_EQ(xfersz, (size_t)ret); + if ((int)xfersz != ret) { + fprintf(stderr,"Read failed 1M+ xfersz \n"); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + + return; + } + // compare buffers + + blk_fvt_cmp_buf(xfersz, &ret); + if (ret != 0) { + fprintf(stderr,"Compare failed 1M+ xfersz \n"); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + + } + ASSERT_EQ(0, ret); + + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); } // Run I/O to debug failing block xfer size @@ -1740,7 +4350,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_514_blksz_physical) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -1773,8 +4383,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_514_blksz_physical) ASSERT_NE(NULL_CHUNK_ID, id ); - get_set_size_flag = 0; - blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 0; + blk_fvt_get_set_lun_size(id, &lun_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -1790,7 +4400,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_514_blksz_physical) EXPECT_EQ(xfersz, (size_t)ret); if ((int)xfersz != ret) { fprintf(stderr,"Write failed 1M+ xfersz \n"); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -1802,7 +4415,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_514_blksz_physical) EXPECT_TRUE(xfersz == (size_t)ret); if ((int)xfersz != ret) { fprintf(stderr,"Read failed 1M+ xfersz \n"); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } // compare buffers @@ -1813,7 +4429,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_514_blksz_physical) } EXPECT_EQ(0, ret); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -1833,7 +4452,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_read_not_16_byte_aligned) int io_flags = 1; size_t temp_sz; - int get_set_size_flag; + int get_set_flag; int cmd; // 0 = read, 1, write @@ -1844,9 +4463,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_read_not_16_byte_aligned) blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); temp_sz = 1; - get_set_size_flag = 2; + get_set_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -1861,7 +4480,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_read_not_16_byte_aligned) EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -1901,7 +4523,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_write_phys_lun_wo_setsz) EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -1941,7 +4566,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_write_virt_lun_wo_setsz) EXPECT_NE(1 , ret); EXPECT_NE(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -1959,7 +4587,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_write_virt_lun_outside_lunsz) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz uint64_t lba; @@ -1978,9 +4606,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_write_virt_lun_outside_lunsz) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1; - get_set_size_flag = 2; + get_set_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2005,7 +4633,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_write_virt_lun_outside_lunsz) EXPECT_NE(1 , ret); EXPECT_NE(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -2025,7 +4656,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_write_phys_lun_outside_lunsz) int io_flags = 0; size_t temp_sz,nblks; int cmd; // 0 = read, 1, write - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -2038,8 +4669,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_write_phys_lun_outside_lunsz) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 0; - get_set_size_flag = 1; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2053,7 +4684,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_write_phys_lun_outside_lunsz) EXPECT_NE(1 , ret); EXPECT_NE(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Verify write fails when tried to write more than 1 block size @@ -2070,7 +4704,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_write_greater_than_1_blk) int io_flags = 0; size_t temp_sz, nblks; int cmd; // 0 = read, 1, write - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -2083,8 +4717,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_write_greater_than_1_blk) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2107,7 +4741,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_write_greater_than_1_blk) EXPECT_NE(2 , ret); EXPECT_EQ(22 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -2128,7 +4765,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_write_not_16_byte_aligned) int io_flags = 1; size_t temp_sz; - int get_set_size_flag; + int get_set_flag; int cmd; // 0 = read, 1, write @@ -2139,9 +4776,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_write_not_16_byte_aligned) blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); temp_sz = 1; - get_set_size_flag = 2; + get_set_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2156,7 +4793,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_write_not_16_byte_aligned) EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Verify async read issued prior to set size succeed on physical lun @@ -2194,7 +4834,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_async_read_phys_lun_wo_setsz) EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -2234,7 +4877,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_read_virt_lun_wo_setsz) EXPECT_NE(1 , ret); EXPECT_EQ(EINVAL , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Test async read fails on read outside of lun size @@ -2249,7 +4895,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_read_vir_lun_outside_lunsz) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz uint64_t lba; @@ -2268,9 +4914,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_read_vir_lun_outside_lunsz) temp_sz = 1; lba = 0; - get_set_size_flag = 2; + get_set_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2284,7 +4930,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_read_vir_lun_outside_lunsz) EXPECT_NE(1 , ret); EXPECT_NE(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -2304,7 +4953,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_read_phys_lun_outside_lunsz) int io_flags = 0; size_t temp_sz,nblks; int cmd; // 0 = read, 1, write - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -2317,8 +4966,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_read_phys_lun_outside_lunsz) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 0; - get_set_size_flag = 1; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2334,7 +4983,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_read_phys_lun_outside_lunsz) EXPECT_NE(0 , ret); EXPECT_EQ(22 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Verify async read fails when tried to read more than 1 block size @@ -2351,7 +5003,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_read_virt_lun_greater_than_1_blk) int io_flags = 0; size_t temp_sz, nblks; int cmd; // 0 = read, 1, write - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -2363,8 +5015,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_read_virt_lun_greater_than_1_blk) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2387,7 +5039,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_read_virt_lun_greater_than_1_blk) EXPECT_NE(2 , ret); EXPECT_EQ(22 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -2428,7 +5083,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM__async_write_phys_lun_wo_setsz) EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -2468,7 +5126,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_virt_lun_wo_setsz) EXPECT_NE(1 , ret); EXPECT_EQ(EINVAL , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -2484,7 +5145,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_vir_lun_outside_lunsz) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz uint64_t lba; @@ -2503,9 +5164,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_vir_lun_outside_lunsz) temp_sz = 1; lba = 0; - get_set_size_flag = 2; + get_set_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2527,7 +5188,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_vir_lun_outside_lunsz) EXPECT_NE(1 , ret); EXPECT_NE(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -2547,7 +5211,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_phys_lun_outside_lunsz) int io_flags = 0; size_t temp_sz,nblks; int cmd; // 0 = read, 1, write - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -2560,8 +5224,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_phys_lun_outside_lunsz) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 0; - get_set_size_flag = 1; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 1; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2575,7 +5239,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_phys_lun_outside_lunsz) EXPECT_NE(1 , ret); EXPECT_NE(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Verify async write fails when tried to write more than 1 block size @@ -2592,7 +5259,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_greater_than_1_blk) int io_flags = 0; size_t temp_sz, nblks; int cmd; // 0 = read, 1, write - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -2605,8 +5272,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_greater_than_1_blk) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2620,7 +5287,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_greater_than_1_blk) EXPECT_NE(2 , ret); EXPECT_EQ(22 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -2636,7 +5306,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_read_compare) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -2655,8 +5325,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_read_compare) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2680,7 +5350,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_async_write_read_compare) EXPECT_EQ(0, ret); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Verify async write fails when data buffer is not 16 byte aligned. @@ -2700,7 +5373,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_awrite_not_16_byte_aligned) int io_flags = 1; size_t temp_sz; - int get_set_size_flag; + int get_set_flag; int cmd; // 0 = read, 1, write @@ -2711,9 +5384,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_awrite_not_16_byte_aligned) blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); temp_sz = 1; - get_set_size_flag = 2; + get_set_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2728,7 +5401,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_awrite_not_16_byte_aligned) EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Verify async read fails when data buffer is not 16 byte aligned. @@ -2747,7 +5423,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_aread_not_16_byte_aligned) int io_flags = 1; size_t temp_sz; - int get_set_size_flag; + int get_set_flag; int cmd; // 0 = read, 1, write @@ -2758,9 +5434,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_aread_not_16_byte_aligned) blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); temp_sz = 1; - get_set_size_flag = 2; + get_set_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -2775,95 +5451,43 @@ TEST(Block_FVT_Suite, BLK_API_FVT_aread_not_16_byte_aligned) EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); -} - - -// Verify CBLK_ARESULT_BLOCKING . waits till cmd completes -TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aresult_blocking) -{ - chunk_id_t id = 0; - int open_flags = CBLK_OPN_VIRT_LUN; - int sz_flags= 0; - int max_reqs= 64; - int er_no = 0; - int open_cnt= 1; - int ret = 0; - uint64_t lba; - int io_flags = 0; - size_t temp_sz, nblks; - int cmd; // 0 = read, 1, write - int get_set_size_flag = 0; // 0 = get phys lun sz - // 1 = get chunk sz - // 2 = set chunk sz - - - ASSERT_EQ(0,blk_fvt_setup(1)); - - // open Virtual lun - blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); - - ASSERT_NE(NULL_CHUNK_ID, id ); - - temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); - EXPECT_EQ(0 , ret); - EXPECT_EQ(0 , er_no); - - cmd = FV_AWRITE; - - lba = 0; - nblks = 1; - io_flags = FV_ARESULT_BLOCKING; - io_flags = 0; - - blk_fvt_io(id, cmd, lba, nblks, &ret, &er_no, io_flags, open_flags); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); - EXPECT_EQ(1 , ret); - EXPECT_EQ(0 , er_no); - - blk_open_tst_cleanup(); } // Verify if CBLK_ARESULT_NEXT_TAG set,call doesn't returns till // async req completes -TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aresult_next_tag) +TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aresult_blocking_next_tag) { - chunk_id_t id = 0; + chunk_id_t id = 0; int open_flags = CBLK_OPN_VIRT_LUN; - int max_reqs= 1024; - int er_no = 0; - int open_cnt= 1; - int ret = 0; - int sz_flags= 0; - size_t temp_sz; - int get_set_size_flag = 0; // 0 = get phys lun sz + int max_reqs = 1024; + int er_no = 0; + int open_cnt = 1; + int ret = 0; + int sz_flags = 0; + size_t temp_sz = 1024; + int get_set_flag = 2; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz - - - ASSERT_EQ(0,blk_fvt_setup(1)); // open virt lun blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); - ASSERT_NE(NULL_CHUNK_ID, id ); + ASSERT_NE(NULL_CHUNK_ID, id); - temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); - ASSERT_NE(-1 , ret); + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); + ASSERT_NE(-1, ret); - blocking_io_tst ( id, &ret, &er_no); - - EXPECT_EQ(1 , ret); - EXPECT_EQ(0 , er_no); - - blk_open_tst_cleanup(); + blocking_io_tst(id, &ret, &er_no); + EXPECT_EQ(1, ret); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); } // Verify CBLK_ARESULT_USER_TAG @@ -2877,7 +5501,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aresult_user_tag) int ret = 0; int sz_flags = 0; size_t temp_sz = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz ASSERT_EQ(0,blk_fvt_setup(1)); @@ -2887,17 +5511,20 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aresult_user_tag) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); ASSERT_NE(-1 , ret); - user_tag_io_tst ( id, &ret, &er_no); + user_tag_io_tst ( id, &ret, &er_no, 0); EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Verify CBLK_ARESULT_USER_TAG with CBLK_OPN_NO_INTRP_THREADS; @@ -2911,7 +5538,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aresult_user_tag_no_intrp) int ret = 0; int sz_flags = 0; size_t temp_sz = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz ASSERT_EQ(0,blk_fvt_setup(1)); @@ -2923,17 +5550,20 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aresult_user_tag_no_intrp) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); ASSERT_NE(-1 , ret); - user_tag_io_tst ( id, &ret, &er_no); + user_tag_io_tst ( id, &ret, &er_no, 0); EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } TEST(Block_FVT_Suite, BLK_API_FVT_virt_lun_perf_test) @@ -2947,7 +5577,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_virt_lun_perf_test) int ret = 0; int sz_flags= 0; size_t temp_sz; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -2960,17 +5590,15 @@ TEST(Block_FVT_Suite, BLK_API_FVT_virt_lun_perf_test) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 10000; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); ASSERT_NE(-1 , ret); - io_perf_tst (id, &ret, &er_no); - + io_perf_tst(id, &ret, &er_no, 0); EXPECT_EQ(0 , ret); - EXPECT_EQ(0 , er_no); - - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); } TEST(Block_FVT_Suite, BLK_API_FVT_phy_lun_perf_test) @@ -2984,10 +5612,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_phy_lun_perf_test) int ret = 0; int sz_flags= 0; size_t temp_sz; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz - int num_cmds = 4096; ASSERT_EQ(0,blk_fvt_setup(num_cmds+1)); @@ -2996,45 +5623,48 @@ TEST(Block_FVT_Suite, BLK_API_FVT_phy_lun_perf_test) blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); ASSERT_NE(NULL_CHUNK_ID, id ); - get_set_size_flag = 0; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 0; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); ASSERT_NE(-1 , ret); /* Test needs atleast 10000 blksz lun */ - if (temp_sz < 10000 ) { - TESTCASE_SKIP("Lun size less than then 10000 blks"); - blk_open_tst_cleanup(); - return; + if (temp_sz < 10000 ) + { + TESTCASE_SKIP("Lun size less than then 10000 blks"); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + return; } - io_perf_tst (id, &ret, &er_no); - + io_perf_tst(id, &ret, &er_no, CBLK_ARESULT_BLOCKING); EXPECT_EQ(0 , ret); - EXPECT_EQ(0 , er_no); - - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } #ifndef _AIX TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_clone_chunk) { - + int open_flags = CBLK_OPN_VIRT_LUN; + int io_flags = 0; int er_no = 0; int ret = 0; int rc = 0; - mode = O_RDWR; - - rc = fork_and_clone (&ret, &er_no, mode); + rc = fork_and_clone (&ret, &er_no, mode, open_flags, io_flags); ASSERT_NE(-1, rc ); EXPECT_EQ(1, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } -TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_RDONLY_mode_test) +TEST(Block_FVT_Suite, BLK_API_FVT_fork_clone_chunk_RDONLY_mode_test) { int er_no = 0; int ret = 0; @@ -3047,11 +5677,14 @@ TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_RDONLY_mode_test) ASSERT_NE(-1, rc ); EXPECT_EQ(0, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } -TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_RDONLY_mode_errpath_1) +TEST(Block_FVT_Suite, BLK_API_FVT_fork_clone_chunk_RDONLY_mode_errpath_1) { int er_no = 0; int ret = 0; @@ -3064,11 +5697,14 @@ TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_RDONLY_mode_errpath_1) ASSERT_NE(-1, rc ); EXPECT_NE(0, ret); EXPECT_NE(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } -TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_RDONLY_mode_errpath_2) +TEST(Block_FVT_Suite, BLK_API_FVT_fork_clone_chunk_RDONLY_mode_errpath_2) { int er_no = 0; int ret = 0; @@ -3081,11 +5717,14 @@ TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_RDONLY_mode_errpath_2) ASSERT_NE(-1, rc ); EXPECT_NE(0, ret); EXPECT_NE(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } -TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_WRONLY_mode_test) +TEST(Block_FVT_Suite, BLK_API_FVT_fork_clone_chunk_WRONLY_mode_test) { int er_no = 0; int ret = 0; @@ -3098,11 +5737,14 @@ TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_WRONLY_mode_test) ASSERT_NE(-1, rc ); EXPECT_EQ(0, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } -TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_WRONLY_mode_errpath_1) +TEST(Block_FVT_Suite, BLK_API_FVT_fork_clone_chunk_WRONLY_mode_errpath_1) { int er_no = 0; int ret = 0; @@ -3115,11 +5757,14 @@ TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_WRONLY_mode_errpath_1) ASSERT_NE(-1, rc ); EXPECT_NE(0, ret); EXPECT_NE(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } -TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_WRONLY_mode_errpath_2) +TEST(Block_FVT_Suite, BLK_API_FVT_fork_clone_chunk_WRONLY_mode_errpath_2) { int er_no = 0; int ret = 0; @@ -3132,11 +5777,14 @@ TEST(BLOCK_FVT_Suite, BLK_API_FVT_fork_clone_chunk_WRONLY_mode_errpath_2) ASSERT_NE(-1, rc ); EXPECT_NE(0, ret); EXPECT_NE(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } -TEST(BLOCK_FVT_Suite, BLK_API_FVT_FM_UMC_fork_clone_chunk_RDWR_mode_test) +TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_fork_clone_chunk_RDWR_mode_test) { int er_no = 0; int ret = 0; @@ -3149,11 +5797,14 @@ TEST(BLOCK_FVT_Suite, BLK_API_FVT_FM_UMC_fork_clone_chunk_RDWR_mode_test) ASSERT_NE(-1, rc ); EXPECT_EQ(0, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } -TEST(BLOCK_FVT_Suite, BLK_API_FVT_FM_UMC_fork_clone_chunk_RDWR_mode_errpath_1) +TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_fork_clone_chunk_RDWR_mode_errpath_1) { int er_no = 0; int ret = 0; @@ -3166,11 +5817,14 @@ TEST(BLOCK_FVT_Suite, BLK_API_FVT_FM_UMC_fork_clone_chunk_RDWR_mode_errpath_1) ASSERT_NE(-1, rc ); EXPECT_EQ(0, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } -TEST(BLOCK_FVT_Suite, BLK_API_FVT_FM_UMC_fork_clone_chunk_RDWR_mode_errpath_2) +TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_fork_clone_chunk_RDWR_mode_errpath_2) { int er_no = 0; int ret = 0; @@ -3183,7 +5837,10 @@ TEST(BLOCK_FVT_Suite, BLK_API_FVT_FM_UMC_fork_clone_chunk_RDWR_mode_errpath_2) ASSERT_NE(-1, rc ); EXPECT_EQ(0, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } @@ -3203,11 +5860,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aresult_explicit_ret_code_tst) int tag = max_reqs + 1; uint64_t status; size_t temp_sz = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz - - - ASSERT_EQ(0,blk_fvt_setup(1)); // open virt lun @@ -3215,8 +5869,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aresult_explicit_ret_code_tst) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); ASSERT_NE(-1 , ret); ret = cblk_aresult(id,&tag,&status,0); @@ -3224,151 +5878,131 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aresult_explicit_ret_code_tst) EXPECT_EQ(-1 , ret); EXPECT_EQ(EINVAL, errno); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } /*** new **/ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_multi_luns_single_thread_rw_tst) { - - int er_no = 0; - int ret = 0; + int flags = CBLK_OPN_VIRT_LUN; + int er_no = 0; + int ret = 0; ASSERT_EQ(0,blk_fvt_setup(1)); - num_loops = 10; + num_loops = 1; num_threads = 1; - num_opens = 0; thread_flag = 1; - virt_lun_flags = TRUE; - blk_thread_tst(&ret,&er_no); - + blk_thread_tst(&ret, &er_no, flags, 0); EXPECT_EQ(0 , ret); - EXPECT_EQ(0 , er_no); - } + TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_phys_multi_luns_single_thread_rw_tst) { - - int er_no = 0; - int ret = 0; + int flags = 0; + int er_no = 0; + int ret = 0; ASSERT_EQ(0,blk_fvt_setup(1)); - num_loops = 10; + num_loops = 1; num_threads = 1; - num_opens = 0; thread_flag = 1; - virt_lun_flags = FALSE; - blk_thread_tst(&ret,&er_no); - - EXPECT_EQ(0 , ret); - EXPECT_EQ(0 , er_no); - + blk_thread_tst(&ret, &er_no, flags, 0); + EXPECT_EQ(0, ret); } -/*** new **/ - TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_multi_luns_multi_thread_rw_tst) { - - int er_no = 0; - int ret = 0; + int flags = CBLK_OPN_VIRT_LUN; + int er_no = 0; + int ret = 0; ASSERT_EQ(0,blk_fvt_setup(1)); - num_loops = 10; + num_loops = 1; num_threads = 10; - num_opens = 0; thread_flag = 1; - virt_lun_flags = TRUE; - blk_thread_tst(&ret,&er_no); - - EXPECT_EQ(0 , ret); - EXPECT_EQ(0 , er_no); - + blk_thread_tst(&ret, &er_no, flags, 0); + EXPECT_EQ(0, ret); } - TEST(Block_FVT_Suite, BLK_API_FVT_FM_phy_lun_multi_thread_rw_tst) { - - int er_no = 0; - int ret = 0; + int flags = 0; + int er_no = 0; + int ret = 0; ASSERT_EQ(0,blk_fvt_setup(1)); - num_loops = 10; + num_loops = 1; num_threads = 10; - num_opens = 0; thread_flag = 1; - virt_lun_flags = FALSE; - blk_thread_tst(&ret,&er_no); - - EXPECT_EQ(0 , ret); - EXPECT_EQ(0 , er_no); - + blk_thread_tst(&ret, &er_no, flags, 0); + EXPECT_EQ(0, ret); } TEST(Block_FVT_Suite, BLK_API_FVT_virt_lun_share_cntxt_rw_tst) { - - int er_no = 0; - int ret = 0; + int flags = CBLK_OPN_VIRT_LUN | CBLK_OPN_SHARE_CTXT; + int er_no = 0; + int ret = 0; ASSERT_EQ(0,blk_fvt_setup(1)); ret = validate_share_context(); if (ret) { TESTCASE_SKIP("context _not_ sharable"); if (blk_fvt_data_buf != NULL) + { free(blk_fvt_data_buf); + blk_fvt_data_buf = NULL; + } if (blk_fvt_comp_data_buf != NULL) + { free(blk_fvt_comp_data_buf); + blk_fvt_comp_data_buf = NULL; + } return; } - num_loops = 10; + num_loops = 1; num_threads = 10; - num_opens = 0; thread_flag = 1; - virt_lun_flags = TRUE; - share_cntxt_flags = TRUE; - blk_thread_tst(&ret,&er_no); - - EXPECT_EQ(0 , ret); - EXPECT_EQ(0 , er_no); + blk_thread_tst(&ret, &er_no, flags, 0); + EXPECT_EQ(0, ret); } TEST(Block_FVT_Suite, BLK_API_FVT_phys_lun_share_cntxt_rw_tst) { - - int er_no = 0; - int ret = 0; + int flags = CBLK_OPN_SHARE_CTXT; + int er_no = 0; + int ret = 0; ASSERT_EQ(0,blk_fvt_setup(1)); ret = validate_share_context(); if (ret) { TESTCASE_SKIP("context _not_ sharable"); if (blk_fvt_data_buf != NULL) + { free(blk_fvt_data_buf); + blk_fvt_data_buf = NULL; + } if (blk_fvt_comp_data_buf != NULL) + { free(blk_fvt_comp_data_buf); + blk_fvt_comp_data_buf = NULL; + } return; } - num_loops = 10; + num_loops = 1; num_threads = 10; - num_opens = 0; thread_flag = 1; - virt_lun_flags = FALSE; - share_cntxt_flags = TRUE; - blk_thread_tst(&ret,&er_no); - - EXPECT_EQ(0 , ret); - EXPECT_EQ(0 , er_no); + blk_thread_tst(&ret, &er_no, flags, 0); + EXPECT_EQ(0, ret); } TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_write_compare_loop_1000) { - - chunk_id_t id = 0; int open_flags = CBLK_OPN_VIRT_LUN; int sz_flags= 0; @@ -3376,7 +6010,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_write_compare_loop_1000) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -3385,20 +6019,17 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_write_compare_loop_1000) size_t temp_sz,nblks; int cmd; - - ASSERT_EQ(0,blk_fvt_setup(1)); // open virtual lun blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); - + ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); - EXPECT_EQ(0 , er_no); nblks = 1; for ( lba = 1; lba <= 1000; lba++ ) { @@ -3424,7 +6055,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_read_write_compare_loop_1000) EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_get_chunk_status_num_commands) @@ -3437,7 +6071,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_get_chunk_status_num_commands) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -3447,8 +6081,6 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_get_chunk_status_num_commands) int cmd; chunk_stats_t stats; - - ASSERT_EQ(0,blk_fvt_setup(1)); // open virtual lun @@ -3457,8 +6089,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_get_chunk_status_num_commands) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); ASSERT_EQ(0 , ret); ASSERT_EQ(0 , er_no); @@ -3490,9 +6122,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_get_chunk_status_num_commands) EXPECT_TRUE(1000 == stats.num_blocks_read); EXPECT_TRUE(1000 == stats.num_blocks_written); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); - } @@ -3507,7 +6140,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aread_awrite_compare_loop_1000) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -3526,8 +6159,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aread_awrite_compare_loop_1000) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); ASSERT_EQ(0 , ret); ASSERT_EQ(0 , er_no); @@ -3550,7 +6183,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_aread_awrite_compare_loop_1000) EXPECT_EQ(0, ret); } - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + } // Test NO_INTRP async io tests @@ -3566,7 +6202,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_NO_INTRP_THREAD_SET_async_io) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -3587,8 +6223,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_NO_INTRP_THREAD_SET_async_io) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -3601,7 +6237,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_NO_INTRP_THREAD_SET_async_io) // If errors no reason to continue if (ret != 1) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } cmd = FV_AREAD; @@ -3611,7 +6250,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_NO_INTRP_THREAD_SET_async_io) // If errors no reason to continue if (ret != 1) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } // compare buffers @@ -3620,7 +6262,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_NO_INTRP_THREAD_SET_async_io) EXPECT_EQ(0, ret); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // Test NO_INTRP synchronous io tests @@ -3636,7 +6280,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_NO_INTRP_THREAD_SET_sync_io) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -3657,8 +6301,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_NO_INTRP_THREAD_SET_sync_io) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -3671,7 +6315,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_NO_INTRP_THREAD_SET_sync_io) // If errors no reason to continue if (ret != 1) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } cmd = FV_READ; @@ -3681,7 +6328,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_NO_INTRP_THREAD_SET_sync_io) // If errors no reason to continue if (ret != 1) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } // compare buffers @@ -3690,7 +6340,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_NO_INTRP_THREAD_SET_sync_io) EXPECT_EQ(0, ret); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // testflag 1 - NO_INTRP set, null status, io_flags ARW_USER set @@ -3707,7 +6359,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_1) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -3727,8 +6379,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_1) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -3737,7 +6389,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_1) EXPECT_EQ(-1 , ret); EXPECT_NE(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // testflag 2 - NO_INTRP _not set, status, io_flags ARW_USER not set @@ -3754,7 +6408,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_2) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -3774,8 +6428,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_2) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -3784,7 +6438,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_2) EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // testflag 3 - NO_INTRP flag _not set, status, io_flags ARW_USER_STATUS set @@ -3801,7 +6457,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_3) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -3821,8 +6477,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_3) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -3831,7 +6487,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_3) EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // testflag 4 - NO_INTRP flag set, status, io_flags ARW_USER set @@ -3848,7 +6506,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_4) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -3868,8 +6526,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_4) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -3878,7 +6536,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_4) EXPECT_EQ(-1 , ret); EXPECT_NE( 0 , er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // testflag 5 - NO_INTRP flag set, status, ARW_USER | ARW_WAIT set @@ -3895,7 +6555,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_5) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -3915,8 +6575,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_5) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -3925,7 +6585,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_5) EXPECT_EQ(-1 , ret); EXPECT_NE( 0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // testflag 6 - NO_INTRP flag set, status, ARW_USER | ARW_WAIT| ARW_USER_TAG set @@ -3942,7 +6604,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_6) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -3962,8 +6624,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_6) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 64; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -3972,7 +6634,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_NO_INTRP_THREAD_ARG_TEST_6) EXPECT_EQ(-1 , ret); EXPECT_NE( 0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // Mix read/write compare with async I/O and sync I/O @@ -3987,7 +6651,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_mix_i_o_loop_1000) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -4006,8 +6670,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_mix_i_o_loop_1000) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -4040,7 +6704,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_mix_i_o_loop_1000) EXPECT_EQ(0, ret); } - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // Mix read/write compare with async I/O and sync I/O with NO_INTRP @@ -4055,7 +6721,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_mix_i_o__no_intrp_loop_1000) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -4074,8 +6740,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_mix_i_o__no_intrp_loop_1000) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -4108,7 +6774,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_mix_i_o__no_intrp_loop_1000) EXPECT_EQ(0, ret); } - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_list_io_args_test) @@ -4122,7 +6790,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_list_io_args_test) int open_cnt= 1; int ret = 0; int i; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -4130,8 +6798,6 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_list_io_args_test) int arg_tst = 0; - mode = O_RDWR; - ASSERT_EQ(0,blk_fvt_setup(1)); // open virtual lun @@ -4140,8 +6806,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_list_io_args_test) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 16; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); for (i=1 ; i<10; i++) { @@ -4159,9 +6825,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_list_io_args_test) } - blk_open_tst_cleanup(); - - + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // Test listio on virt lun, without Timout, and with CBLK_IO_USER_STATUS flag set @@ -4176,7 +6842,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -4199,14 +6865,17 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4219,7 +6888,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test) // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4232,7 +6904,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test) // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4254,7 +6929,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test) break; } } - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // Test listio on virt lun, without Timout, and with CBLK_IO_USER_STATUS flag set and NO_INTRP flag set @@ -4269,7 +6946,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_no_intrp) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -4295,14 +6972,17 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_no_intrp) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4315,7 +6995,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_no_intrp) // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4328,7 +7011,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_no_intrp) // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4350,7 +7036,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_no_intrp) break; } } - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // Test listio on physical lun, without Timout, and with CBLK_IO_USER_STATUS flag set @@ -4365,7 +7053,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -4385,15 +7073,18 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test) ASSERT_NE(NULL_CHUNK_ID, id ); - get_set_size_flag = 0; + get_set_flag = 0; // get phys lun size - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } cmd = FV_WRITE; @@ -4403,7 +7094,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test) // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4414,7 +7108,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test) // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4424,7 +7121,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test) EXPECT_EQ(0, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // Test listio on physical lun, without Timout, and with CBLK_IO_USER_STATUS flag set , NO_INTRP set @@ -4439,7 +7138,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test_no_intrp) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -4462,15 +7161,18 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test_no_intrp) ASSERT_NE(NULL_CHUNK_ID, id ); - get_set_size_flag = 0; + get_set_flag = 0; // get phys lun size - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } cmd = FV_WRITE; @@ -4480,7 +7182,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test_no_intrp) // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4491,7 +7196,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test_no_intrp) // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4501,7 +7209,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test_no_intrp) EXPECT_EQ(0, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } @@ -4519,7 +7229,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_1) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -4541,14 +7251,17 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_1) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4560,7 +7273,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_1) // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4572,7 +7288,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_1) // If errors no reason to continue if ((ret != 0) || (er_no != 0)) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4582,7 +7301,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_1) EXPECT_EQ(0, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // Test listio on virt lun, without timeout and without CBLK_USER_IO_STATUS @@ -4600,7 +7321,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_2) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -4621,8 +7342,8 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_2) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 1024; - get_set_size_flag = 2; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 2; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); @@ -4639,10 +7360,12 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_UMC_virt_lun_list_io_test_2) ret = memcmp(blk_fvt_data_buf,blk_fvt_comp_data_buf, BLK_FVT_BUFSIZE*num_listio); er_no = errno; + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + blk_open_tst_cleanup(0, &ret, &er_no); EXPECT_EQ(0, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); } // Test listio on physical lun without user status flag , no timeout set @@ -4659,7 +7382,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test_1) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -4680,13 +7403,16 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test_1) temp_sz = 0; - get_set_size_flag = 0; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 0; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); if ((temp_sz < 1024) || ret || er_no ){ - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } @@ -4707,7 +7433,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test_1) EXPECT_EQ(0, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // Test listio on Physical lun, without timeout and without CBLK_USER_IO_STATUS @@ -4725,7 +7453,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test_2) int er_no = 0; int open_cnt= 1; int ret = 0; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -4746,13 +7474,16 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test_2) ASSERT_NE(NULL_CHUNK_ID, id ); temp_sz = 0; - get_set_size_flag = 0; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 0; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); if (temp_sz < 1024) { - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } cmd = FV_WRITE; @@ -4768,10 +7499,12 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_phys_lun_list_io_test_2) ret = memcmp(blk_fvt_data_buf,blk_fvt_comp_data_buf, BLK_FVT_BUFSIZE*num_listio); er_no = errno; + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + blk_open_tst_cleanup(0, &ret, &er_no); EXPECT_EQ(0, ret); EXPECT_EQ(0, er_no); - blk_open_tst_cleanup(); } // Verifies data perisists across closes on physical luns @@ -4790,10 +7523,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_Persistence_phys_lun_test) size_t nblks; int cmd; - mode = O_RDWR; - // Setup dev name and allocated test buffers - ASSERT_EQ(0,blk_fvt_setup(256)); // open physical lun @@ -4859,8 +7589,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_FM_Persistence_phys_lun_test) EXPECT_EQ(0, er_no ); - blk_open_tst_cleanup(); - + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } #ifdef _AIX @@ -4884,8 +7615,6 @@ TEST(Block_FVT_Suite, BLK_API_FVT_scsi_reserve_test) size_t nblks; int cmd; - mode = O_RDWR; - // Setup dev name and allocated test buffers ASSERT_EQ(0,blk_fvt_setup(1)); @@ -4917,8 +7646,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_scsi_reserve_test) EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); - + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // Open phys lun with scsi reserve and mpio_fo mode @@ -4939,8 +7669,6 @@ TEST(Block_FVT_Suite, BLK_API_FVT_scsi_reserve_err_path_test) size_t nblks; int cmd; - mode = O_RDWR; - // Setup dev name and allocated test buffers ASSERT_EQ(0,blk_fvt_setup(1)); @@ -4952,8 +7680,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_scsi_reserve_err_path_test) EXPECT_EQ (NULL_CHUNK_ID, id ); EXPECT_NE (0 , er_no); - blk_open_tst_cleanup(); - + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // validate CBLK_OPN_RESERVE and CBLK_OPN_FORCED_RESERVE flags @@ -4973,8 +7702,6 @@ TEST(Block_FVT_Suite, BLK_API_FVT_scsi_force_reserve_test) size_t nblks; int cmd; - mode = O_RDWR; - // Setup dev name and allocated test buffers ASSERT_EQ(0,blk_fvt_setup(1)); @@ -4995,8 +7722,10 @@ TEST(Block_FVT_Suite, BLK_API_FVT_scsi_force_reserve_test) EXPECT_EQ(0 , er_no); if ((ret !=1 ) || (errno)) { - blk_open_tst_cleanup(); - return; + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } // close physical lun @@ -5018,8 +7747,9 @@ TEST(Block_FVT_Suite, BLK_API_FVT_scsi_force_reserve_test) EXPECT_EQ(1 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); - + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } // export env variable FVT_NUM_LOOPS to 2000 @@ -5041,7 +7771,7 @@ TEST(Block_FVT_Suite, BLK_API_FVT_phy_mpio_cabl_pull_io) int ret = 0; int sz_flags= 0; size_t temp_sz; - int get_set_size_flag = 0; // 0 = get phys lun sz + int get_set_flag = 0; // 0 = get phys lun sz // 1 = get chunk sz // 2 = set chunk sz @@ -5056,24 +7786,28 @@ TEST(Block_FVT_Suite, BLK_API_FVT_phy_mpio_cabl_pull_io) blk_open_tst( &id, max_reqs, &er_no, open_cnt, open_flags, mode); ASSERT_NE(NULL_CHUNK_ID, id ); - get_set_size_flag = 0; - blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_size_flag, &ret, &er_no); + get_set_flag = 0; + blk_fvt_get_set_lun_size(id, &temp_sz, sz_flags, get_set_flag, &ret, &er_no); ASSERT_NE(-1 , ret); /* Test needs atleast 10000 blksz lun */ if (temp_sz < 10000 ) { TESTCASE_SKIP("Lun size less than then 10000 blks"); - blk_open_tst_cleanup(); + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); + return; } - io_perf_tst (id, &ret, &er_no); + io_perf_tst(id, &ret, &er_no, 0); EXPECT_EQ(0 , ret); EXPECT_EQ(0 , er_no); - blk_open_tst_cleanup(); - + blk_open_tst_cleanup(0, &ret, &er_no); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, er_no); } #endif /* _AIX */ diff --git a/src/block/test/makefile b/src/block/test/makefile index 9ddf2e64..2a41e5d0 100644 --- a/src/block/test/makefile +++ b/src/block/test/makefile @@ -36,9 +36,9 @@ ALLOW_WARNINGS = yes export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${ROOTPATH}/img LIBPATHS = -L${ROOTPATH}/img -LINKLIBS = -lcflsh_block +LINKLIBS = -lcflsh_block -lm -BTESTS=blockio blocklistio blockplistio +BTESTS=blockio blocklistio blockplistio blocksio blockr0io blockr0sio BIN_TESTS=$(addprefix ${TESTDIR}/, ${BTESTS}) # AIX only @@ -55,8 +55,7 @@ LINKLIBS+=-lpthread -ludev endif VPATH += \ - ${ROOTPATH}/src/common \ - ${ROOTPATH}/src/kv + ${ROOTPATH}/src/common GTESTS = run_block_fvt @@ -65,15 +64,14 @@ GTESTS64 = $(addsuffix 64, $(GTESTS)) GTESTS64_DIR = $(addprefix $(TESTDIR)/, $(GTESTS64)) run_block_fvt_OFILES = fvt_block.o blk_api_tst.o cflash_tools_user.o -blockio_OFILES = ut.o DEPS=$(addprefix $(TESTDIR)/, $(run_block_fvt_OFILES:.o=.dep)) CFLAGS += \ - -g -D__FVT__\ + -g -D__FVT__ -Wno-unused-result \ -I$(ROOTPATH)/src/block \ -I$(ROOTPATH)/src/include \ - -I$(ROOTPATH)/src/test/framework/googletest/googletest/include + -I$(ROOTPATH)/${GTESTINC} \ CXXFLAGS+=$(CFLAGS) diff --git a/src/build/install/makefile b/src/build/install/makefile index 7de1f0d6..dd557e1a 100644 --- a/src/build/install/makefile +++ b/src/build/install/makefile @@ -25,7 +25,7 @@ UNAME=$(shell uname) ROOTPATH = ../../.. -APPNAME?=capikv +APPNAME?=cflash PREFIX?=${PKGDIR}/install_root TEST_PREFIX?=${PKGDIR}/test_root @@ -65,11 +65,13 @@ _SYSTEMD_DIR=${PREFIX}/usr/lib/systemd/system _UDEV_RULES_DIR=${PREFIX}/lib/udev/rules.d IMAGE="${_RESOURCEDIR}/corsa*" IMAGEH="${_RESOURCEDIR}/1410*" + all: rm -rf ${INSTALL_ROOT} - ${MAKE} capikv capikv-test afuimage + ${MAKE} cflash cflash-test cflashimage + ${INSTALL} ${_RESOURCEDIR}/cflash_configure ${PKGDIR} -capikv: +cflash: @mkdir -p ${_BIN_DIR} @mkdir -p ${_ETC_DIR} @mkdir -p ${_LIB_DIR} @@ -79,6 +81,7 @@ capikv: @mkdir -p -m 755 ${_INC_DIR} @mkdir -p ${_SYSTEMD_DIR} @mkdir -p ${_UDEV_RULES_DIR} + @mkdir -p ${PREFIX}/var/log/cflash/misc ifeq ($(UNAME),AIX) ${INSTALL} ${ROOTPATH}/src/kv/arkdb.h ${_INC_DIR} ${INSTALL} ${ROOTPATH}/src/include/zmalloc.h ${_INC_DIR} @@ -100,8 +103,14 @@ ifeq ($(UNAME),AIX) ${INSTALL} ${_TESTSRCDIR}/asyncstress64 ${_BIN_DIR} ${INSTALL} ${_TESTSRCDIR}/_tst_ark64 ${_BIN_DIR} ${INSTALL} ${_TESTSRCDIR}/blockio64 ${_BIN_DIR} + ${INSTALL} ${_TESTSRCDIR}/blocksio64 ${_BIN_DIR} + ${INSTALL} ${_TESTSRCDIR}/blockr0io64 ${_BIN_DIR} + ${INSTALL} ${_TESTSRCDIR}/blockr0sio64 ${_BIN_DIR} ${INSTALL} ${_TESTSRCDIR}/blocklistio64 ${_BIN_DIR} ${INSTALL} ${_TESTSRCDIR}/blockplistio64 ${_BIN_DIR} + ${INSTALL} ${_TESTSRCDIR}/blocksio ${_BIN_DIR} + ${INSTALL} ${_TESTSRCDIR}/blockr0io ${_BIN_DIR} + ${INSTALL} ${_TESTSRCDIR}/blockr0sio ${_BIN_DIR} ${INSTALL} ${_TESTSRCDIR}/blockio ${_BIN_DIR} ${INSTALL} ${_TESTSRCDIR}/blocklistio ${_BIN_DIR} ${INSTALL} ${_TESTSRCDIR}/blockplistio ${_BIN_DIR} @@ -109,6 +118,8 @@ ifeq ($(UNAME),AIX) ${INSTALL} ${_TESTSRCDIR}/run_kv_sync64 ${_BIN_DIR} ${INSTALL} ${_TESTSRCDIR}/run_kv_persist64 ${_BIN_DIR} ${INSTALL} ${_TESTSRCDIR}/run_kv_benchmark64 ${_BIN_DIR} + ${INSTALL} ${ROOTPATH}/src/test/ioppt.pl ${_BIN_DIR} + ${INSTALL} ${ROOTPATH}/src/test/ioppts.qd ${_BIN_DIR} else ${INSTALL} -s ${PGMDIR}/cxlfd ${_BIN_DIR} ${INSTALL} -s ${PGMDIR}/cxlflashutil ${_BIN_DIR} @@ -116,19 +127,27 @@ else ${INSTALL} -s ${_TESTSRCDIR}/asyncstress ${_BIN_DIR} ${INSTALL} -s ${_TESTSRCDIR}/_tst_ark ${_BIN_DIR} ${INSTALL} -s ${_TESTSRCDIR}/blockio ${_BIN_DIR} + ${INSTALL} -s ${_TESTSRCDIR}/blocksio ${_BIN_DIR} + ${INSTALL} -s ${_TESTSRCDIR}/blockr0io ${_BIN_DIR} + ${INSTALL} -s ${_TESTSRCDIR}/blockr0sio ${_BIN_DIR} ${INSTALL} -s ${_TESTSRCDIR}/blocklistio ${_BIN_DIR} ${INSTALL} -s ${_TESTSRCDIR}/blockplistio ${_BIN_DIR} ${INSTALL} -s ${_TESTSRCDIR}/run_kv_sync ${_BIN_DIR} ${INSTALL} -s ${_TESTSRCDIR}/run_kv_async ${_BIN_DIR} ${INSTALL} -s ${_TESTSRCDIR}/run_kv_persist ${_BIN_DIR} ${INSTALL} -s ${_TESTSRCDIR}/run_kv_benchmark ${_BIN_DIR} + ${INSTALL} ${ROOTPATH}/src/test/ioppt.pl ${_BIN_DIR} + ${INSTALL} ${ROOTPATH}/src/test/ioppts.qd ${_BIN_DIR} ${INSTALL} ${_RESOURCEDIR}/cxlfrefreshluns ${_BIN_DIR} ${INSTALL} ${_RESOURCEDIR}/cablecheck ${_BIN_DIR} ${INSTALL} ${_RESOURCEDIR}/cxlfsetlunmode ${_BIN_DIR} ${INSTALL} ${_RESOURCEDIR}/cxlfstatus ${_BIN_DIR} - ${INSTALL} ${_RESOURCEDIR}/capikvutils.sh ${_BIN_DIR} + ${INSTALL} ${_RESOURCEDIR}/cflashutils.sh ${_BIN_DIR} ${INSTALL} ${_RESOURCEDIR}/setup.sh ${_BIN_DIR} + ${INSTALL} ${_RESOURCEDIR}/cflash_version ${_BIN_DIR} + ${INSTALL} ${_RESOURCEDIR}/machine_info ${_BIN_DIR} + ${INSTALL} ${_RESOURCEDIR}/cflash_logs_cron ${_BIN_DIR} #TEMPORARY install - remove this after kernel driver can run workaround @@ -138,6 +157,9 @@ else @#Dev Permissions for cxl - set mode to 644 explicitly ${INSTALL} -m 644 ${_RESOURCEDIR}/80-cxl.rules ${_UDEV_RULES_DIR} ${INSTALL} -m 644 ${_RESOURCEDIR}/80-cxlflash.rules ${_UDEV_RULES_DIR} + + ${INSTALL} ${_RESOURCEDIR}/cxlffdc.debug ${_ETC_DIR} + ${INSTALL} ${_RESOURCEDIR}/cxlffdc.cleardebug ${_ETC_DIR} endif @#Libs @@ -179,7 +201,7 @@ endif @#Version tags and useful info echo "${GITREVISION}" > ${INSTALL_ROOT}/version.txt -capikv-test: +cflash-test: @#TEST ONLY CONTENT - DO NOT SHIP @mkdir -p ${_TEST_DIR} @@ -201,6 +223,7 @@ capikv-test: ${INSTALL} ${_TESTSRCDIR}/fvt_ark_perf_check ${_TEST_DIR} ${INSTALL} ${_TESTSRCDIR}/fvt_ark_perf_tool ${_TEST_DIR} ${INSTALL} ${_TESTSRCDIR}/fvt_kv_tst_ark ${_TEST_DIR} + ${INSTALL} ${ROOTPATH}/src/test/run_ioppts ${_TEST_DIR} ${INSTALL} ${ROOTPATH}/src/test/multi_process_perf ${_TEST_DIR} ${INSTALL} ${ROOTPATH}/src/block/test/block_perf_check ${_TEST_DIR} @#Enable factory flash if the test image is installed @@ -226,7 +249,15 @@ ifeq ($(UNAME),AIX) ${INSTALL} ${_TESTSRCDIR}/fvt_kv_tst_ark64 ${_TEST_DIR} ${INSTALL} ${ROOTPATH}/src/test/multi_process_perf ${_TEST_DIR} ${INSTALL} ${ROOTPATH}/src/block/test/block_perf_check ${_TEST_DIR} +else + ${INSTALL} ${ROOTPATH}/src/test/vpd/surelock_vpd2rbf.pl ${_TEST_DIR} + ${INSTALL} ${ROOTPATH}/src/test/vpd/vpd_FlashGT.csv ${_TEST_DIR} + ${INSTALL} -s ${_TESTSRCDIR}/flashgt_vpd_access ${_TEST_DIR} + ${INSTALL} ${_RESOURCEDIR}/cflash_inject.pl ${_TEST_DIR} + ${INSTALL} -s ${_TESTSRCDIR}/cxl_afu_inject ${_TEST_DIR} + ${INSTALL} -s ${_RESOURCEDIR}/flashgt_nvme_override ${_TEST_DIR} endif + ifeq ($(UNAME),AIX) @#Manpages - this is highly repetative else @@ -242,22 +273,37 @@ else gzip -f ${_MAN_DIR}/*.h.3 endif -afuimage: +cflashimage: @mkdir -p ${_AFU_IMAGES_DIR} +#install if image PAPR header is promoted +ifneq ("$(wildcard ${_RESOURCEDIR}/1410*)","") + ${INSTALL} ${_RESOURCEDIR}/1410* ${_AFU_IMAGES_DIR} + ${INSTALL} ${_RESOURCEDIR}/BTV* ${_AFU_IMAGES_DIR} +ifneq ($(UNAME),AIX) + cd ${_AFU_IMAGES_DIR}; /usr/bin/md5sum 1410* > afu_fw.md5sum; /usr/bin/md5sum *.bin >> afu_fw.md5sum +endif +endif ifeq ($(UNAME),AIX) # need to have a valid ffdc mechanism for AIX... else #Linux only ${INSTALL} -s ${_TESTSRCDIR}/cxl_afu_dump ${AFU_ROOT} + ${INSTALL} -s ${_TESTSRCDIR}/cxl_afu_status ${AFU_ROOT} + ${INSTALL} -s ${_TESTSRCDIR}/flashgt_temp ${AFU_ROOT} ${INSTALL} ${_RESOURCEDIR}/capi_flash.pl ${AFU_ROOT} + ${INSTALL} ${_RESOURCEDIR}/cflash_perst.pl ${AFU_ROOT} + ${INSTALL} ${_RESOURCEDIR}/cflash_perf.pl ${AFU_ROOT} + ${INSTALL} ${_RESOURCEDIR}/cflash_capacity.pl ${AFU_ROOT} + ${INSTALL} ${_RESOURCEDIR}/cflash_devices.pl ${AFU_ROOT} + ${INSTALL} ${_RESOURCEDIR}/cflash_temp.pl ${AFU_ROOT} + ${INSTALL} ${_RESOURCEDIR}/cflash_wear.pl ${AFU_ROOT} + ${INSTALL} ${_RESOURCEDIR}/cflash_stick.pl ${AFU_ROOT} + ${INSTALL} ${_RESOURCEDIR}/cflash_perfcheck.pl ${AFU_ROOT} ${INSTALL} ${_RESOURCEDIR}/blacklist-cxlflash.conf ${AFU_ROOT} ${INSTALL} ${_RESOURCEDIR}/cxlffdc ${AFU_ROOT} ${INSTALL} ${_RESOURCEDIR}/flash_all_adapters ${AFU_ROOT} ${INSTALL} ${_RESOURCEDIR}/reload_all_adapters ${AFU_ROOT} ${INSTALL} ${_RESOURCEDIR}/psl_trace_dump ${AFU_ROOT} -#install if image PAPR header is promoted -ifneq ("$(wildcard ${_RESOURCEDIR}/1410*)","") - ${INSTALL} ${_RESOURCEDIR}/1410* ${_AFU_IMAGES_DIR} -endif + ${INSTALL} ${_RESOURCEDIR}/cxl_flash_vm ${AFU_ROOT} #install if image without header is promoted ifneq ("$(wildcard ${_RESOURCEDIR}/corsa*)","") diff --git a/src/build/install/resources/1410000614100006.00000001160903N0 b/src/build/install/resources/1410000614100006.00000001160903N0 new file mode 120000 index 00000000..a587e56b --- /dev/null +++ b/src/build/install/resources/1410000614100006.00000001160903N0 @@ -0,0 +1 @@ +/gsa/ausgsa/projects/s/surelock/images/1410000614100006.00000001160903N0 \ No newline at end of file diff --git a/src/build/install/resources/1410000614100006.00000001161114N1 b/src/build/install/resources/1410000614100006.00000001161114N1 new file mode 120000 index 00000000..e40eb252 --- /dev/null +++ b/src/build/install/resources/1410000614100006.00000001161114N1 @@ -0,0 +1 @@ +/gsa/ausgsa/projects/s/surelock/images/1410000614100006.00000001161114N1 \ No newline at end of file diff --git a/src/build/install/resources/1410000614100006.factoryselect.hdr b/src/build/install/resources/1410000614100006.factoryselect.hdr new file mode 100755 index 00000000..256a592e Binary files /dev/null and b/src/build/install/resources/1410000614100006.factoryselect.hdr differ diff --git a/src/build/install/resources/1410000614100006.userselect.hdr b/src/build/install/resources/1410000614100006.userselect.hdr new file mode 100755 index 00000000..6a0fbe9c Binary files /dev/null and b/src/build/install/resources/1410000614100006.userselect.hdr differ diff --git a/src/build/install/resources/1410f0041410f004.00000001151029D1 b/src/build/install/resources/1410f0041410f004.00000001151029D1 deleted file mode 120000 index d9f6794d..00000000 --- a/src/build/install/resources/1410f0041410f004.00000001151029D1 +++ /dev/null @@ -1 +0,0 @@ -/gsa/ausgsa/projects/s/surelock/images/1410f0041410f004.00000001151029D1 \ No newline at end of file diff --git a/src/build/install/resources/1410f0041410f004.00000001160512D1 b/src/build/install/resources/1410f0041410f004.00000001160512D1 new file mode 120000 index 00000000..b29fb0aa --- /dev/null +++ b/src/build/install/resources/1410f0041410f004.00000001160512D1 @@ -0,0 +1 @@ +/gsa/ausgsa/projects/s/surelock/images/1410f0041410f004.00000001160512D1 \ No newline at end of file diff --git a/src/build/install/resources/BTV73011_NF.bin b/src/build/install/resources/BTV73011_NF.bin new file mode 120000 index 00000000..1a9ef000 --- /dev/null +++ b/src/build/install/resources/BTV73011_NF.bin @@ -0,0 +1 @@ +/gsa/ausgsa/projects/s/surelock/images/BTV73011_NF.bin \ No newline at end of file diff --git a/src/build/install/resources/BTV7301Q.bin b/src/build/install/resources/BTV7301Q.bin new file mode 120000 index 00000000..ed37b6f2 --- /dev/null +++ b/src/build/install/resources/BTV7301Q.bin @@ -0,0 +1 @@ +/gsa/ausgsa/projects/s/surelock/images/BTV7301Q.bin \ No newline at end of file diff --git a/src/build/install/resources/cablecheck b/src/build/install/resources/cablecheck index 8d965fad..80dc72fb 100755 --- a/src/build/install/resources/cablecheck +++ b/src/build/install/resources/cablecheck @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # @@ -25,7 +25,7 @@ # IBM_PROLOG_END_TAG CAPIKV=/opt/ibm/capikv -source $CAPIKV/bin/capikvutils.sh +source $CAPIKV/bin/cflashutils.sh KERNELMOD=cxlflash @@ -50,7 +50,7 @@ disablemodule() then die "ERROR: Linux multipathing is enabled, and must be disabled prior to running this tool as it typically prevents the unloading of the $KERNELMOD driver. Please temporarily flush the multipath tables by running \"multipath -F\" and try again." 4 else - die "Unable to remove the $KERNELMOD driver. Please ensure that all applications that exploit $KERNELMOD or ibmcapikv are terminated." + die "Unable to remove the $KERNELMOD driver. Please ensure that all applications that exploit $KERNELMOD or cflash are terminated." fi; fi; echo "INFO: $KERNELMOD unloaded." diff --git a/src/build/install/resources/capi_flash.pl b/src/build/install/resources/capi_flash.pl index 4c97c09d..ee599d1e 100755 --- a/src/build/install/resources/capi_flash.pl +++ b/src/build/install/resources/capi_flash.pl @@ -24,19 +24,21 @@ # # IBM_PROLOG_END_TAG ## +use strict; +use warnings; use Fcntl; +use Fcntl ':seek'; use Getopt::Long; -# ------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- # Variables -# ------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- my $a; # Flash Address my $ra; # Flash Read Address my $dat; # Data my $d; # Packed data my $edat; # Expected Data my $fdat; # Flash Data -my $cfgadr; # Configuration Register Address my $rc; # Return Code from PSL when performing an operation my $bc; # Block Counter my $dif; # Return code from file read @@ -45,7 +47,6 @@ my $lt; # Last Poll Time my $ct; # Current Time -my $dif; # Data in file flag (End of File) my $i; # Loop Counter my $et; # Total Elasped Time @@ -53,591 +54,790 @@ my $eet; # Total/End Erase Time my $set; # Start Erase Time -my $ept; # Total Program Time my $spt; # Start Program Time my $ept; # End Program Time -my $evt; # Total Verify Time my $svt; # Start Verify Time my $evt; # End Verify Time my $fsize; # File size my $n; # Number of Blocks to Program -my $numdev; # Number of CORSA Devices in the lspci output +my $fwords; # Number of words to Program my $devstr; # Device String from lspci output my @dsev; # Device String split apart to find device location - -# -- My Change - - -my $hdr = 0; # Set if PAPR header is present in image +my $vsec; +my $tmp; +my $factory_addr=0; +my $header_addr=0x840000; +my $user_addr=0x850000; +my $vpd_addr=0x1FF0000; +my $wr_user; +my $cardN; +my $offset=0; # Set if PAPR header is present in image my $hdat = 0; # Vendoer Dev id from header if present -my $devId = 0x101404cf; # 0x101404cf default for capi -my $size = 0; # verify size at offset 24 , 8 bytes for actual image +my $cfg_devid=""; # devid read from config space +my $ma; +my $_VM=0; +my $imgdir="/opt/ibm/capikv/afu/images"; -# ------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- # Defaults: -# ------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- my $prthelp = 0; # Print Help Flag -my $partition0 = 0; # Partition 0 Flag -my $partition1 = 1; # Partition 1 Flag -my $partition = "User Partition"; # Programming Partition Name +my $partition0 = ""; # Programming Partition0 +my $partition1 = ""; # Programming Partition1 my $list = "0"; my $optOK = 0; # Options are OK my $filename = ""; # RBF File Name +my $userselect_hdr="1410000614100006.userselect.hdr"; +my $factoryselect_hdr="1410000614100006.factoryselect.hdr"; my $target = ""; # pcie target my $cfgfile = ""; # RBF File Name - -my $ec = 0; # Error Code +my $vpd = ""; +my $parm_verbose=1; +my $verbose; my $ADDR_REG = 0x920; # Flash Address Configuration Register my $SIZE_REG = 0x924; # Flash Size Configuration Register -my $CNTL_REG = 0x928; # Flash Control / Status Configuration Register -my $DATA_REG = 0x92C; # Flash Data Configuration Register - -# ------------------------------------------------------------------------------- -# Parse Options -# ------------------------------------------------------------------------------- -$optOK = GetOptions ( "f|rbf=s" => \$filename, - "p0" , \$partition0, - "p1" , \$partition1, - "v" , \$vpd, - "l" , \$list, - "d=o" => \$devId, - "t|target=s" => \$target, - "h|help!" , \$prthelp - ); - -if ($ARGV[0]) { - print "\nUnknown Command Line Options:\n"; - foreach(@ARGV) { - print " $_\n"; - } - print "\n"; - $prthelp = 1; -} - -if (!($optOK) | $prthelp) { +my $CNTL_REG = 0x928; # Flash Control / Status Config Reg +my $DATA_REG = 0x92C; # Flash Data Configuration Registermy +my $VSEC_REG = 0x904; # Flash VSEC Register + +#------------------------------------------------------------------------------- +# sub +# Print usage +#------------------------------------------------------------------------------- +sub usage +{ print "\n"; - print "Usage: capi_flash_pgm [-h | --help] [-p0 | -p1 | -v] [-f | --rbf] \n"; - print " -p0 : Program Partition 0 - Factory Image \n"; - print " -p1 : Program Partition 1 - User Image {Default} \n"; - print " -v : Program VPD Information \n"; - print " -f or --rbf : Raw Binary File Containing the Bitstream \n"; + print "Usage: capi_flash.pl [-f fw_file] [-l] [-t ] [-v]\n"; print " -l : List pci location of capi devices \n"; - print " -d : Dev and Ven ID in header of fw image \n"; + print " -f : Firmware image \n"; print " -t or --target : Target pcie device to flash \n"; + print " -v : verbose, Print debug messages \n"; print " -h or --help : Help - Print this message \n"; - die "\n"; -} - -if ( !(($partition == 0) | ($partition == 1)) ) { - print "\nPartition can only be 0 or 1\n"; - die; -} - - -#added to support programming VPD - Charlie Johns -if ( $vpd == 1 ) { - #note we override the "address" set by the user above, so we cannot program VP - #simultaneously with another flash region (factory or user). - $a = 0x1FF0000 -} elsif ($partition0) { - #if we are not programming VPD, see if we should program the "partition 0" image - $a = 0x10000; - $partition = "Factory Image" -} else { - #default to programming partition 1 otherwise. - $a = 0x850000; -} - - - - -# ------------------------------------------------------------------------------- -# Open Bitstream File -# ------------------------------------------------------------------------------- -if (-e $filename) { - open(IN, "< $filename"); - binmode(IN); -} elsif (!$list){ - die "Raw Binary Bitstream File $filename does not exist.\n"; + if ($_VM) + { + print "\n ex: capi_flash.pl -f 1410f0041410f004.00000001160413D1 -t afu0.0\n\n"; + } + else + { + print "\n ex: capi_flash.pl -f 1410f0041410f004.00000001160413D1 -t 0000:01:00.0\n\n"; + } + exit 0; } +#------------------------------------------------------------------------------- +# sub +# wait for a byte pattern in the flash CNTL_REG +#------------------------------------------------------------------------------- +sub wait_for_flash +{ + my $mask = shift; + my $cmp = shift; -# ------------------------------------------------------------------------------- -# Make stdout autoflush -# ------------------------------------------------------------------------------- -select(STDOUT); # default -$| = 1; - -if ($list) { - - my @files = ; - my ($vendor, $device); + sysseek CFG, $CNTL_REG, SEEK_SET; + sysread(CFG,$d,4); + $dat = unpack("V",$d); - $numdev = 0; - for my $file ( @files ) { - open(F, $file . "/vendor") or die "Can't open $filename: $!"; - read(F, $vendor, 6); - close (F); - open(F, $file . "/device") or die "Can't open $filename: $!"; - read(F, $device, 6); - close (F); + $st = $lt = time(); + $cp = 1; - if (($vendor eq "0x1014") && (($device eq "0x0477") || ($device eq "0x04cf"))) { - $cfgfile = $file; - print "Found CAPI Adapter : $cfgfile\n"; - $numdev++; - } + while ($cp == 1) + { + sysseek(CFG, $CNTL_REG, SEEK_SET); + sysread(CFG,$d,4); + $rc = unpack("V",$d); + if (($rc & $mask) == $cmp) + { + $cp = 0; + } + $ct = time(); + if (($ct - $lt) > 5) + { + print "."; + $lt = $ct; + } + if (($ct - $st) > 120) + { + sysseek CFG, $CNTL_REG, SEEK_SET; + syswrite(CFG,0,4); + die "\nFAILURE --> Flash not ready after 2 min\n"; } -exit; -} -# ------------------------------------------------------------------------------- -# Find the CAPI Device -# ------------------------------------------------------------------------------- -my @files = ; -my ($vendor, $device); - -if ($target){ - print "Target specified: $target\n"; - $cfgfile = $target; -} -else{ -$numdev = 0; -for my $file ( @files ) { - open(F, $file . "/vendor") or die "Can't open $filename: $!"; - read(F, $vendor, 6); - close (F); - open(F, $file . "/device") or die "Can't open $filename: $!"; - read(F, $device, 6); - close (F); - - if (($vendor eq "0x1014") && (($device eq "0x0477") || ($device eq "0x04cf"))) { - $cfgfile = $file; - $numdev++; } - } -} - -if ($numdev == 0 and $cfgfile eq "") { - die "CAPI Device (ID = 0x0477) does not exist.\n"; } -if ($numdev > 1) { - die "\n $numdev capi devices detected"; +#------------------------------------------------------------------------------- +# sub +# clear and wait for the flash to be ready for the next operation +#------------------------------------------------------------------------------- +sub reset_sequence_wait4rdy +{ + $dat = 0; + $d = pack("V",$dat); + sysseek CFG, $CNTL_REG, SEEK_SET; + syswrite(CFG,$d,4); + wait_for_flash(0x80000000, 0x80000000); } -print "\nCAPI Adapter is : $cfgfile\n"; - -# ------------------------------------------------------------------------------- -# Open the CAPI Device's Configuration Space -# ------------------------------------------------------------------------------- -sysopen(CFG, $cfgfile . "/config", O_RDWR) or die $!; +#------------------------------------------------------------------------------- +# sub +# ensure the filename has a valid checksum +#------------------------------------------------------------------------------- +sub validate_checksum +{ + my $fn = shift; + + #----------------------------------------------------------------------------- + # verify checksum + #----------------------------------------------------------------------------- + my $filekey=( split '/', $fn )[-1]; + my $found=`grep $filekey $imgdir/afu_fw.md5sum`; + if (! $found) {die " ERROR: unsupported firmware level: $fn\n";} + my $out=`cd $imgdir; /usr/bin/md5sum -c $imgdir/afu_fw.md5sum 2>&1 | grep $filekey|grep -v md5sum`; + if ($out =~ /FAILED/) {die " ERROR: bad checksum: $fn\n";} +} -# ------------------------------------------------------------------------------- -# Read the Device/Vendor ID from the Configuration Space -# ------------------------------------------------------------------------------- -sysseek(CFG, 0, SEEK_SET); -sysread(CFG,$dat,4); -$d = unpack("V",$dat); - -printf(" Device/Vendor ID: 0x%08x\n\n", $d); - -# ------------------------------------------------------------------------------- -# Check the image file has header -# ------------------------------------------------------------------------------- -sysseek(IN, 0, SEEK_SET); -sysread(IN,$dat,2); # read first 16 bits of word hdr -$d = unpack("n",$dat); -if ($d == 0x0001) { - printf(" The AFU image has PAPR header\n"); +#------------------------------------------------------------------------------- +# sub +# write a file to a Flash addr +# _verbose == 0, print nothing (hdrs) +# _verbose == 1, print selectivly +# _verbose == 2, print everything +#------------------------------------------------------------------------------- +sub write_file_to_flash +{ + my $fn = shift; + my $addr = shift; + my $_verbose = shift; + my $fsize; + my $st; + my $lt; + my $n; + my $fwords; + my $bc; + my $dat; + my $i; + my $cp; + my $d; + my $hdr; + + #----------------------------------------------------------------------------- + # open file + #----------------------------------------------------------------------------- + if (! -e $fn) + { + if (! ($fn =~ /images/)) + { + if (-e "$imgdir/$fn") + { + $fn = "$imgdir/$fn"; + } + } + } + if (-e $fn) + { + open(IN, "< $fn"); + binmode(IN); + } + else + { + die " ERROR: File $fn cannot be opened\n"; + } + $fsize = -s $fn; + + ($_verbose==2) && print "\n Writing filename: $fn\n"; + + #----------------------------------------------------------------------------- + # Check if the image has a header + #----------------------------------------------------------------------------- + sysseek(IN, 0, SEEK_SET); + #read first 16 bits of word hdr + (sysread(IN,$dat,2) == 2) or die " ERROR: Image file has an invalid header\n"; + $d = unpack("n",$dat); + if ($d == 0x0001) + { + ($_verbose==2) && print " Image has PAPR header\n"; sysseek(IN, 8, SEEK_SET); sysread(IN,$dat,4); - $hdat = unpack("N",$dat); + my $hdat = unpack("N",$dat); + my $img_devid = sprintf("%x", $hdat); - if ($hdat == $devId) { - printf(" Image has header for DevVenId : 0x%08x\n\n", $hdat); - $hdr = 1; + if ($img_devid eq $cfg_devid) + { + ($_verbose==2) && printf(" Image has header for 0x%08x\n", $hdat); sysseek(IN, 0, SEEK_SET); sysseek(IN, 28, SEEK_SET); sysread(IN,$hdat,8); - $size = unpack ("N",$hdat); - printf(" Image size in header :0x%08x\n\n", $size); - } else { - printf(" Header doesn't match with DevVenId : 0x%08x\n\n",$devId); - printf(" Image has header for DevVenId : 0x%08x\n\n", $hdat); - die " Use correct image for the card \n"; + my $hsize = unpack ("N",$hdat); + ($_verbose==2) && printf(" Image size in header: %d\n\n", $hsize); + $fsize -= 0x80; + if ($fsize != $hsize) + { + die " ERROR: Image size didn't match with size in header.\n"; + } + $hdr=1; } -} else { - printf(" Image has no header\n\n"); - -} - -sysseek(IN, 0, SEEK_SET); - -# ------------------------------------------------------------------------------- -# Read the VSEC Length / VSEC ID from the Configuration Space -# ------------------------------------------------------------------------------- -sysseek(CFG, 0x904, SEEK_SET); -sysread(CFG,$dat,4); -$d = unpack("V",$dat); - -printf(" VSEC Length/VSEC Rev/VSEC ID: 0x%08x\n", $d); -if ( ($d & 0x08000000) == 0x08000000 ) { - printf(" Version 0.12\n\n"); - $ADDR_REG = 0x950; # Flash Address Configuration Register - $SIZE_REG = 0x954; # Flash Size Configuration Register - $CNTL_REG = 0x958; # Flash Control / Status Configuration Register - $DATA_REG = 0x95C; # Flash Data Configuration Register -} else { - printf(" Version 0.10\n\n"); -} - -# ------------------------------------------------------------------------------- -# Reset Any Previously Aborted Sequences -# ------------------------------------------------------------------------------- -$cfgadr = $CNTL_REG; -$dat = 0; -$d = pack("V",$dat); -sysseek CFG, $cfgadr, seek_set; -syswrite(CFG,$d,4); - -# ------------------------------------------------------------------------------- -# Wait for Flash to be Ready -# ------------------------------------------------------------------------------- -sysseek CFG, $cfgadr, seek_set; -sysread(CFG,$d,4); -$dat = unpack("V",$d); - -$st = $lt = time(); -$cp = 1; - -$cfgadr = $CNTL_REG; -while ($cp == 1) { - sysseek(CFG, $cfgadr, SEEK_SET); - sysread(CFG,$d,4); - $rc = unpack("V",$d); - if ( ($rc & 0x80000000) == 0x80000000 ) { - $cp = 0; - } - $ct = time(); - if (($ct - $lt) > 5) { - print "."; - $lt = $ct; - } - if (($ct - $st) > 120) { - print "\nFAILURE --> Flash not ready after 2 min\n"; - $cp = 0; - $ec = 1; - } -} - -# ------------------------------------------------------------------------------- -# Calculate the number of blocks to write -# ------------------------------------------------------------------------------- -$fsize = -s $filename; -if ($hdr == 1) { - $fsize -= 0x80; # Linux image don't need header for Linux image - if ($fsize == $size) { - printf(" FW image size matched with size in header = 0x%x\n", $size); - printf(" Header will be stripped from image\n"); - } else { - die "Image size didn't match with size in header.\n"; + else + { + print " Header $img_devid doesn't match $cfg_devid\n"; + die " ERROR: Use correct image for the card \n"; } + } - sysseek(IN,0x80, SEEK_SET); -} -$n = $fsize / (64 * 1024 * 4); -$n = (($n == int($n)) ? $n : int($n + 1)); - -printf("Programming %s with %s\n",$partition, $filename); -printf(" Program -> Address: 0x%x for Size: %d in blocks (32K Words or 128K Bytes)\n\n",$a,$n); + #calculate how much to write + $n = $fsize / (64 * 1024 * 4); + $n = (($n == int($n)) ? $n : int($n + 1)); + $fwords = 64*1024*$n; -$n -= 1; + reset_sequence_wait4rdy; -$set = time(); + #----------------------------------------------------------------------------- + # Erase Flash at $addr for the size of $fn + #----------------------------------------------------------------------------- + ($_verbose>0) && printf " Erase Flash -> Address: 0x%x, blocks %d ", + $addr, $n; + $st = $lt = time(); -if ($ec == 0) { -# ------------------------------------------------------------------------------- -# Setup for Program From Flash -# ------------------------------------------------------------------------------- - $cfgadr = $ADDR_REG; - $d = pack("V",$a); - sysseek CFG, $cfgadr, SEEK_SET; + $d = pack("V",$addr); + sysseek CFG, $ADDR_REG, SEEK_SET; syswrite(CFG,$d,4); - - $cfgadr = $SIZE_REG; - $d = pack("V",$n); - sysseek CFG, $cfgadr, SEEK_SET; + + $d = pack("V",$n-1); + sysseek CFG, $SIZE_REG, SEEK_SET; syswrite(CFG,$d,4); - $cfgadr = $CNTL_REG; $dat = 0x04000000; $d = pack("V",$dat); - sysseek CFG, $cfgadr, SEEK_SET; + sysseek CFG, $CNTL_REG, SEEK_SET; syswrite(CFG,$d,4); - print "Erasing Flash\n"; - -# ------------------------------------------------------------------------------- -# Wait for Flash Erase to complete. -# ------------------------------------------------------------------------------- + # Wait for Flash Erase to complete. $st = $lt = time(); $cp = 1; - - $cfgadr = $CNTL_REG; - while ($cp == 1) { - sysseek(CFG, $cfgadr, SEEK_SET); + while ($cp == 1) + { + sysseek(CFG, $CNTL_REG, SEEK_SET); sysread(CFG,$d,4); $rc = unpack("V",$d); if ( (($rc & 0x00008000) == 0x00000000) && - (($rc & 0x00004000) == 0x00004000) ) { + (($rc & 0x00004000) == 0x00004000) ) + { $cp = 0; } $ct = time(); - if (($ct - $lt) > 5) { - print "."; + if (($ct - $lt) > 5) + { + ($_verbose>0) && print "."; $lt = $ct; } - if (($ct - $st) > 240) { - print "\nFAILURE --> Erase did not complete in 4 min\n"; - $cp = 0; - $ec += 2; + if (($ct - $st) > 240) + { + sysseek CFG, $CNTL_REG, SEEK_SET; + syswrite(CFG,0,4); + die "\nFAILURE --> Erase did not complete in 4 min\n"; } } -} + printf "\n"; -$eet = $spt = time(); + #----------------------------------------------------------------------------- + # write the file to the Flash addr + #----------------------------------------------------------------------------- + ($_verbose>0) && printf(" Program Flash -> Address: 0x%x, blocks %d bytes %d\n", + $addr, $n, $fwords*4); -# ------------------------------------------------------------------------------- -# Program Flash -# ------------------------------------------------------------------------------- -if ($ec == 0) { - print "\n\nProgramming Flash\n"; + if ($hdr) {seek IN, 0x80, SEEK_SET;} # skip the header + else {seek IN, 0, SEEK_SET;} # Reset to beginning of file $bc = 0; - print "Writing Block: $bc \r"; - $cfgadr = $DATA_REG; - for($i=0; $i<(64*1024*($n+1)); $i++) { + for ($i=0; $i<$fwords; $i++) + { + #read the next bytes of data to be written $dif = read(IN,$d,4); $dat = unpack("V",$d); - if (!($dif)) { - $dat = 0xFFFFFFFF; - } + if (!($dif)) {$dat = 0xFFFFFFFF;} + + #prepare the data to be written $d = pack("V",$dat); - sysseek CFG, $cfgadr, SEEK_SET; - syswrite(CFG,$d,4); - if ((($i+1) % (512)) == 0) { - print "Writing Buffer: $bc \r"; + if (($i % (64*1024)) == 0) + { + ($_verbose>0) && print " Write Block: $bc of $n \r"; $bc++; } - } -} -print "\n\n"; + wait_for_flash(0x00001000, 0x00000000); -# ------------------------------------------------------------------------------- -# Wait for Flash Program to complete. -# ------------------------------------------------------------------------------- -$st = $lt = time(); -$cp = 1; - -$cfgadr = $CNTL_REG; -while ($cp == 1) { - sysseek(CFG, $cfgadr, SEEK_SET); - sysread(CFG,$d,4); - $rc = unpack("V",$d); - if ( ($rc & 0x40000000) == 0x40000000 ) { - $cp = 0; - } - $ct = time(); - if (($ct - $lt) > 5) { - print "."; - $lt = $ct; - } - if (($ct - $st) > 120) { - print "\nFAILURE --> Programming did not complete after 2 min\n"; - $cp = 0; - $ec += 4; + #do the write + sysseek CFG, $DATA_REG, SEEK_SET; + syswrite(CFG,$d,4); } -} -$ept = time(); - -# ------------------------------------------------------------------------------- -# Reset Program Sequence -# ------------------------------------------------------------------------------- -$cfgadr = $CNTL_REG; -$dat = 0; -$d = pack("V",$dat); -sysseek CFG, $cfgadr, seek_set; -syswrite(CFG,$d,4); - -# ------------------------------------------------------------------------------- -# Wait for Flash to be Ready -# ------------------------------------------------------------------------------- -sysseek CFG, $cfgadr, seek_set; -sysread(CFG,$d,4); -$dat = unpack("V",$d); - -$st = $lt = time(); -$cp = 1; - -$cfgadr = $CNTL_REG; -while ($cp == 1) { - sysseek(CFG, $cfgadr, SEEK_SET); - sysread(CFG,$d,4); - $rc = unpack("V",$d); - if ( ($rc & 0x80000000) == 0x80000000 ) { - $cp = 0; - } - $ct = time(); - if (($ct - $lt) > 5) { - print "."; - $lt = $ct; - } - if (($ct - $st) > 120) { - print "\nFAILURE --> Flash not ready after 2 min\n"; - $cp = 0; - $ec += 8; - } -} + #wait for programming to complete + wait_for_flash(0x40000000, 0x40000000); -$svt = time(); + #reset before doing verify + reset_sequence_wait4rdy; -# ------------------------------------------------------------------------------- -# Verify Flash Programmming -# ------------------------------------------------------------------------------- -if ($ec == 0) { - print "Verifying Flash\n"; + #----------------------------------------------------------------------------- + # pre-verify + #----------------------------------------------------------------------------- + ($_verbose==2) && printf " Pre-Verify Flash -> Address: 0x%x, blocks %d\n", + $addr, 1; + $bc = 0; + $ra = $addr; + for($i=0; $i<512; $i++) + { + if (($i % 512) == 0) + { + reset_sequence_wait4rdy; - if ($hdr == 1 ) { - seek IN, 0x80, SEEK_SET; # skip the header for Linux - } else { - seek IN, 0, SEEK_SET; # Reset to beginning of file + # Setup for Reading From Flash + $d = pack("V",$ra); + sysseek CFG, $ADDR_REG, SEEK_SET; + syswrite(CFG,$d,4); + $ra += 0x200; + $d = pack("V",0x1FF); + sysseek CFG, $SIZE_REG, SEEK_SET; + syswrite(CFG,$d,4); + $dat = 0x08000000; + $d = pack("V",$dat); + sysseek CFG, $CNTL_REG, SEEK_SET; + syswrite(CFG,$d,4); + } + + sysseek CFG, $DATA_REG, SEEK_SET; + sysread(CFG,$d,4); } - #close(IN); - #open(IN, "< $filename"); - #binmode(IN); - $bc = 0; - $ra = $a; - print "Reading Block: $bc \r"; - for($i=0; $i<(64*1024*($n+1)); $i++) { + #----------------------------------------------------------------------------- + # Verify + #----------------------------------------------------------------------------- + ($_verbose>0) && printf " Verify Flash -> Address: 0x%x, blocks %d\n", + $addr, $n; - $dif = read(IN,$d,4); + if ($hdr) {seek IN, 0x80, SEEK_SET;} # skip the header + else {seek IN, 0, SEEK_SET;} # Reset to beginning of file + + $bc = 0; + $ra = $addr; + for($i=0; $i<$fwords; $i++) + { + $dif = read(IN,$d,4); $edat = unpack("V",$d); - if (!($dif)) { + if (!($dif)) + { $edat = 0xFFFFFFFF; } + if (($i % 512) == 0) + { + reset_sequence_wait4rdy; - if (($i % 512) == 0) { - $cfgadr = $CNTL_REG; - $dat = 0; - $d = pack("V",$dat); - sysseek CFG, $cfgadr, seek_set; - syswrite(CFG,$d,4); - - # ------------------------------------------------------------------------------- - # Wait for Flash to be Ready - # ------------------------------------------------------------------------------- - $st = $lt = time(); - $cp = 1; - - $cfgadr = $CNTL_REG; - while ($cp == 1) { - sysseek(CFG, $cfgadr, SEEK_SET); - sysread(CFG,$d,4); - $rc = unpack("V",$d); - if ( ($rc & 0x80000000) == 0x80000000 ) { - $cp = 0; - } - $ct = time(); - if (($ct - $lt) > 5) { - print "."; - $lt = $ct; - } - if (($ct - $st) > 120) { - print "\nFAILURE --> Flash not ready after 2 min\n"; - $cp = 0; - $ec += 16; - last; - } - } - - # ------------------------------------------------------------------------------- # Setup for Reading From Flash - # ------------------------------------------------------------------------------- - $cfgadr = $ADDR_REG; $d = pack("V",$ra); - sysseek CFG, $cfgadr, SEEK_SET; + sysseek CFG, $ADDR_REG, SEEK_SET; syswrite(CFG,$d,4); $ra += 0x200; - - $cfgadr = $SIZE_REG; $d = pack("V",0x1FF); - sysseek CFG, $cfgadr, SEEK_SET; + sysseek CFG, $SIZE_REG, SEEK_SET; syswrite(CFG,$d,4); - - $cfgadr = $CNTL_REG; $dat = 0x08000000; $d = pack("V",$dat); - sysseek CFG, $cfgadr, SEEK_SET; + sysseek CFG, $CNTL_REG, SEEK_SET; syswrite(CFG,$d,4); } - $cfgadr = $DATA_REG; - sysseek CFG, $cfgadr, SEEK_SET; + if (($i % (64*1024)) == 0) + { + ($_verbose>0) && print " Read Block: $bc \r"; + $bc++; + } + + sysseek CFG, $DATA_REG, SEEK_SET; sysread(CFG,$d,4); $fdat = unpack("V",$d); - if ($edat != $fdat) { + if ($edat != $fdat) + { $ma = $ra + ($i % 512) - 0x200; - printf("Data Miscompare @: %08x --> %08x expected %08x\r",$ma, $fdat, $edat); - $rc = ; + sysseek CFG, $CNTL_REG, SEEK_SET; + syswrite(CFG,0,4); + my $pmsg = sprintf "\n Miscompare: %08x --> %08x expected %08x\n", + $ma, $fdat, $edat; + die $pmsg; } + } - if ((($i+1) % (64*1024)) == 0) { - print "Reading Block: $bc \r"; - $bc++; + reset_sequence_wait4rdy; + + ($_verbose>0) && printf " "; + close(IN); +} + +#------------------------------------------------------------------------------- +# sub +# ignore Ctrl-C during a download +#------------------------------------------------------------------------------- +sub sigint_handler +{ + print "?\n"; +} + +#------------------------------------------------------------------------------- +# sub +# Get vendor/devid for vm +#------------------------------------------------------------------------------- +sub set_vm_cfgid +{ + $cardN=shift; + + my $vendor=`cat /sys/devices/platform/*ibm,coherent*/cxl/card$cardN/afu$cardN.0/cr*/vendor`; + my $devid=`cat /sys/devices/platform/*ibm,coherent*/cxl/card$cardN/afu$cardN.0/cr*/device`; + chomp $vendor; + $vendor=substr($vendor,2,4); + $devid=substr($devid,2,4); + if ($devid eq "04f0") {$devid="04cf";} + $cfg_devid="$vendor$devid"; +} + + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- +# M A I N +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +#check if VM +`grep pSeries /proc/cpuinfo 2>/dev/null`; +if ($? == 0) {$_VM=1;} +if ($verbose && $_VM) {print "Running on a VM system.\n";} + +#------------------------------------------------------------------------------- +# Parse Options +#------------------------------------------------------------------------------- +if ($#ARGV<0) {usage();} + +if (! GetOptions ("f=s" => \$filename, + "p0" => \$partition0, + "p1" => \$partition1, + "vpd" => \$vpd, + "l" => \$list, + "t|target=s" => \$target, + "h|help!" => \$prthelp, + "v" => \$verbose + )) +{ + usage(); +} +if ($ARGV[0]) +{ + print "\nUnknown Command Line Options:\n"; + foreach(@ARGV) + { + print " $_\n"; + } + print "\n"; + usage(); +} +if ($prthelp) {usage();} + +#check sudo permissions +(`id -u` == 0) || die "Run with sudo permissions\n"; + +#------------------------------------------------------------------------------- +# Make stdout autoflush +#------------------------------------------------------------------------------- +select(STDOUT); +$| = 1; + +#------------------------------------------------------------------------------- +# VM - list CAPI Devices +#------------------------------------------------------------------------------- +if ($_VM) +{ + my @cards = `ls -d /sys/class/cxl/card* 2>/dev/null|awk -F/ '{print \$5}'`; + for my $card ( @cards ) + { + chomp $card; + $cardN=substr $card,-1; + set_vm_cfgid $cardN; + my @afuD; + @afuD=`/opt/ibm/capikv/afu/cxl_afu_dump /dev/cxl/afu$cardN.0m -v 2>/dev/null`; + my $afuver=""; + for my $ver ( @afuD ) + { + if ($ver eq "") {last;} + if ($ver =~ /Version/) {$afuver=( split ' ', $ver )[ -1 ]; last;} } + $list && print "Found CAPI device afu$cardN.0 $cfg_devid =>$afuver\n"; + } +} +#------------------------------------------------------------------------------- +# BMC -list CAPI Devices +#------------------------------------------------------------------------------- +else +{ + my @files = ; + my ($vendor, $device); + + for my $file ( @files ) + { + open(F, $file . "/vendor") or die " ERROR: Can't open $file: $!\n"; + read(F, $vendor, 6); + close (F); + open(F, $file . "/device") or die " ERROR: Can't open $file: $!\n"; + read(F, $device, 6); + close (F); + + if (($vendor eq "0x1014") && (($device eq "0x0477") || + ($device eq "0x04cf") || + ($device eq "0x0601"))) + { + $cfg_devid = substr($vendor,2,4) . substr($device,2,4); + my @dds = split /\//, $file; + my $adap = pop @dds; + my $serial=`lspci -s $adap -vv|grep "Serial number:"|awk '{print \$4}'`; + chomp $serial; + $serial = sprintf("%-14s", $serial); + my $cardstr=`ls -d /sys/devices/*/*/$adap/cxl/card* 2>/dev/null`; + chomp $cardstr; + my $cardN=substr $cardstr,-1; + my @afuD=`/opt/ibm/capikv/afu/cxl_afu_dump /dev/cxl/afu$cardN.0m -v 2>/dev/null`; + my $afuver=""; + for my $ver ( @afuD ) + { + if ($ver eq "") {last;} + if ($ver =~ /AFU Version/) {$afuver=( split ' ', $ver )[ -1 ]; last;} + } + my $devspec=`cat /sys/bus/pci/devices/$adap/devspec`; + my $loccode=`cat /proc/device-tree$devspec/ibm,loc-code 2>/dev/null`; + chomp $loccode; + $loccode=substr($loccode,0,23); + $list && print "Found CAPI device $cfg_devid afu$cardN $adap $loccode $serial =>$afuver\n"; + } + } +} + +#------------------------------------------------------------------------------- +# exit if (-l) list option specified +#------------------------------------------------------------------------------- +$list && exit; + +#------------------------------------------------------------------------------- +# VM - validate target +#------------------------------------------------------------------------------- +if ($_VM) +{ + if (! $target =~ /afu/) {die " ERROR: bad target format: ($target)\n";} + if (! -e "/sys/class/cxl/$target") + { + die " ERROR: bad target: /sys/class/cxl/$target\n"; + } + $cardN=substr $target,3,1; + set_vm_cfgid $cardN; + $verbose && print "Target specified: $cfg_devid $target\n\n"; +} +#------------------------------------------------------------------------------- +# BMC - validate target - Open the target CAPI Device's Configuration Space +#------------------------------------------------------------------------------- +else +{ + if (! ($target =~ /bus/)) + { + $target = "/sys/bus/pci/devices/$target"; + } + if (! -e $target) + { + die " ERROR: bad target: $target\n"; + } + sysopen(CFG, $target . "/config", O_RDWR) or die " ERROR: $!\n"; + $verbose && print "Target specified: $target\n\n"; +} + +#------------------------------------------------------------------------------- +# BMC - Read the Device/Vendor ID from the Configuration Space +#------------------------------------------------------------------------------- +if (!$_VM) +{ + sysseek(CFG, 0, SEEK_SET); + sysread(CFG,$dat,4); + $d = unpack("V",$dat); + $tmp = sprintf "%08x", $d; + $cfg_devid = substr($tmp,4,4) . substr($tmp,0,4); + $verbose && print " Config Space: Device/Vendor ID: $cfg_devid\n"; +} + +#ensure cxlflash is not loaded before continuing +`lsmod|grep cxlflash`; +if ($? == 0) {print "rmmod cxlflash before doing the download\n"; exit 0;} + +#------------------------------------------------------------------------------- +# VM - Finish - call binary to do the AFU FW Download +#------------------------------------------------------------------------------- +if ($_VM) +{ + my $addhdr=""; + if ($filename =~ /.bin/ || $filename =~ /.rbf/) {$addhdr=" -a ";} + my $cmd="/opt/ibm/capikv/afu/cxl_flash_vm -c $cardN -f $filename $addhdr"; + $verbose && printf(" Running: $cmd\n"); + system($cmd); + exit $?; +} + +#------------------------------------------------------------------------------- +# Read the VSEC Length / VSEC ID from the Configuration Space +#------------------------------------------------------------------------------- +if ($cfg_devid eq "10140601") +{ + $VSEC_REG = 0x404; +} +my $vsec_msg=""; +sysseek(CFG, $VSEC_REG, SEEK_SET); +sysread(CFG,$dat,4); +$vsec = unpack("V",$dat); +$verbose && printf(" VSEC Addr:0x%04x Len/Rev: 0x%08x\n", $VSEC_REG, $vsec); + +if ($cfg_devid eq "101404cf") +{ + if ( ($vsec & 0x08000000) == 0x08000000 ) + { + $vsec_msg=" Corsa 04cf: Altera flash 0.12"; + $ADDR_REG = 0x950; # Flash Address Configuration Register + $SIZE_REG = 0x954; # Flash Size Configuration Register + $CNTL_REG = 0x958; # Flash Control / Status Config Reg + $DATA_REG = 0x95C; # Flash Data Configuration Register + } + else + { + $vsec_msg=" Corsa 04cf: Altera flash 0.10"; + $ADDR_REG = 0x920; # Flash Address Configuration Register + $SIZE_REG = 0x924; # Flash Size Configuration Register + $CNTL_REG = 0x928; # Flash Control / Status Config Reg + $DATA_REG = 0x92C; # Flash Data Configuration Register + } + $factory_addr = 0x10000; +} +elsif ($cfg_devid eq "10140601") +{ + $header_addr=0x0800000; + $user_addr=0x0810000; + + if ( ($vsec & 0x08000000) == 0x08000000 ) + { + $vsec_msg=" FlashGT 0601: Xilinx flash"; + $ADDR_REG = 0x450; # Flash Address Configuration Register + $SIZE_REG = 0x454; # Flash Size Configuration Register + $CNTL_REG = 0x458; # Flash Control / Status Config Reg + $DATA_REG = 0x45C; # Flash Data Configuration Register + } + else + { + $vsec_msg=" FlashGT 0601: Altera flash"; + $ADDR_REG = 0x920; # Flash Address Configuration Register + $SIZE_REG = 0x924; # Flash Size Configuration Register + $CNTL_REG = 0x928; # Flash Control / Status Config Reg + $DATA_REG = 0x92C; # Flash Data Configuration Register } } +elsif ($cfg_devid eq "10140477") +{ + if ( ($vsec & 0x00000000) == 0x0000F000 ) + { + $vsec_msg=" Corsa 0477: Xilinx flash"; + $ADDR_REG = 0x450; # Flash Address Configuration Register + $SIZE_REG = 0x454; # Flash Size Configuration Register + $CNTL_REG = 0x458; # Flash Control / Status Config Reg + $DATA_REG = 0x45C; # Flash Data Configuration Register + } + else + { + $factory_addr = 0x10000; + $vsec_msg=" Corsa 0477: Altera flash"; + $ADDR_REG = 0x950; # Flash Address Configuration Register + $SIZE_REG = 0x954; # Flash Size Configuration Register + $CNTL_REG = 0x958; # Flash Control / Status Config Reg + $DATA_REG = 0x95C; # Flash Data Configuration Register + } +} +else +{ + die " ERROR: unknown device: $cfg_devid\n"; +} +$verbose && print "$vsec_msg\n\n"; + +#------------------------------------------------------------------------------- +# determine the config space addr to program +#------------------------------------------------------------------------------- +if ($vpd) +{ + #note we override the "address" set by the user above, so we cannot program + #VPD simultaneously with another flash region (factory or user). + $a = $vpd_addr; + printf " VPD selected, addr: 0x%x\n", $a; +} +elsif ($partition0) +{ + $a = $factory_addr; + printf " Factory Image selected, addr: 0x%x\n", $a; +} +else +{ + #default to programming partition 1 otherwise. + $wr_user=1; + $a = $user_addr; + $verbose && printf " User Image selected, addr: 0x%x\n", $a; +} -$evt = time(); +# start the elapse timer +$set = time(); -print "\n\n"; +#------------------------------------------------------------------------------- +# validate the checksum on the download files before starting the download +#------------------------------------------------------------------------------- +validate_checksum $filename; +if ($cfg_devid eq "10140601" && $wr_user) +{ + validate_checksum $factoryselect_hdr; + validate_checksum $userselect_hdr; +} -# ------------------------------------------------------------------------------- -# Check for Errors during Programming -# ------------------------------------------------------------------------------- -if ($ec != 0) { - print "\nErrors Occured : Error Code => $ec\n"; +# install the SIGINT handler +$SIG{INT} = \&sigint_handler; + +#------------------------------------------------------------------------------- +# if GT and we are writing the user image, then program factoryselect header +#------------------------------------------------------------------------------- +if ($cfg_devid eq "10140601" && $wr_user) +{ + if ($verbose) {$parm_verbose=2;} + else {$parm_verbose=0;} + write_file_to_flash($factoryselect_hdr, $header_addr, $parm_verbose); } -# ------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- +# Program the flash +#------------------------------------------------------------------------------- +if ($verbose) {$parm_verbose=2;} +else {$parm_verbose=1;} +write_file_to_flash($filename, $a, $parm_verbose); + +#------------------------------------------------------------------------------- +# if GT and we are writing the user image, then program userselect header +#------------------------------------------------------------------------------- +if ($cfg_devid eq "10140601" && $wr_user) +{ + if ($verbose) {$parm_verbose=2;} + else {$parm_verbose=0;} + write_file_to_flash($userselect_hdr, $header_addr, $parm_verbose); +} + +#------------------------------------------------------------------------------- # Calculate and Print Elapsed Times -# ------------------------------------------------------------------------------- -$et = $evt - $set; -$eet = $eet - $set; -$ept = $ept - $spt; -$evt = $evt - $svt; - -print "Erase Time: $eet seconds\n"; -print "Program Time: $ept seconds\n"; -print "Verify Time: $evt seconds\n"; -print "Total Time: $et seconds\n\n"; - -# ------------------------------------------------------------------------------- -# Reset Read Sequence -# ------------------------------------------------------------------------------- -$cfgadr = $CNTL_REG; -$dat = 0; -$d = pack("V",$dat); -sysseek CFG, $cfgadr, seek_set; -syswrite(CFG,$d,4); - -close(IN); +#------------------------------------------------------------------------------- +$et = time() - $set; +print "\n Total Time: $et seconds\n\n"; + close(CFG); exit; diff --git a/src/build/install/resources/cflash_capacity.pl b/src/build/install/resources/cflash_capacity.pl new file mode 100755 index 00000000..04b38d8c --- /dev/null +++ b/src/build/install/resources/cflash_capacity.pl @@ -0,0 +1,120 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_capacity.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +## +use strict; +use warnings; +use Fcntl; +use Fcntl ':seek'; +use Getopt::Long; + +#------------------------------------------------------------------------------- +# Variables +my $type="0601"; +my $devices=""; +my $size; +my $tsize=0; +my $prthelp=0; +my $verbose; + +sub usage() +{ + print "\n"; + print "Usage: cflash_capacity.pl [-t type] \n"; + print " -t or --type : Type to list \n"; + print " -h or --help : Help - Print this message \n"; + print "\n ex: cflash_devices.pl -t 0601 \n\n"; + exit 0; +} + +#------------------------------------------------------------------------------- +# Parse Options +#------------------------------------------------------------------------------- +if (! GetOptions ("t|type=s" => \$type, + "v" => \$verbose, + "h|help!" => \$prthelp + )) +{ + usage(); +} +if ($ARGV[0]) +{ + print "\nUnknown Command Line Options:\n"; + foreach(@ARGV) + { + print " $_\n"; + } + print "\n"; + usage(); +} +if ($prthelp) {usage();} + +#------------------------------------------------------------------------------- +# Make stdout autoflush +#------------------------------------------------------------------------------- +select(STDOUT); +$| = 1; + +#------------------------------------------------------------------------------- +# list Devices +#------------------------------------------------------------------------------- +my @cards = `lspci |grep $type`; +my $cmd=""; +my $sym; + +for my $cardN (@cards) +{ + my @Acard=split / /, $cardN; + my $card=$Acard[0]; + chomp $card; + my $afustr=`ls -d /sys/devices/*/*/$Acard[0]/cxl/card*/afu*.0/afu*m 2>/dev/null`; + my @afu=split /\//, $afustr; + my $afu=pop @afu; + if (length $afu) {chomp $afu;} + else {$afu="";} + my $cardstr=`ls -d /sys/devices/*/*/$Acard[0]/cxl/card* 2>/dev/null`; + chomp $cardstr; + my $cardN=substr $cardstr,-1; + + for (my $i=0; $i<2; $i++) + { + if ($i == 1 && $type =~ /04cf/) {next;} + my @Adev=`ls -d /sys/devices/*/*/$Acard[0]/*/*/host*/target*:$i:*/*:*:*:*/block/*|awk -F/ '{print \$13}' 2>/dev/null`; + if ($? != 0) {next;} + foreach my $dev (@Adev) + { + chomp $dev; + $size=`fdisk -l /dev/$dev |grep "Disk /dev"|awk '{print \$3}'|awk -F. '{print \$1}'`; + $sym=`fdisk -l /dev/$dev |grep "Disk /dev"|awk '{print \$4}'`; + chomp $size; + if ($sym =~ /TiB/) {$size *= 1000;} + $verbose && print "/dev/$dev: $size\n"; + $tsize+=$size; + } + } +} + +print "$tsize\n"; + diff --git a/src/build/install/resources/cflash_configure b/src/build/install/resources/cflash_configure new file mode 100755 index 00000000..54e5f27c --- /dev/null +++ b/src/build/install/resources/cflash_configure @@ -0,0 +1,685 @@ +#!/bin/bash +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_configure $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +if [[ $(id -u) != 0 ]]; then echo "use sudo"; exit 1; fi + +if [[ ! -e /opt ]]; then mkdir /opt; fi +if [[ ! -e /opt/.lic ]]; then mkdir /opt/.lic; fi + +export PATH=$PATH:/opt/ibm/capikv/bin:/opt/ibm/capikv/afu +rc=0 +LD=.EaienF48ajfFgBMIhsalfc +HD=.Jfa82FBMIhsalfc +force=no +ADV_TC=at10 +K_VER="4.4.0-36.55" +K_VER_INST="4.4.0-36" +K_VER_GRUB="4.4.0-36-generic" + +wget www.ibm.com >/dev/null 2>&1 +internet=$? + +################################## +# Determine distro +if [[ $(lsb_release -a 2>/dev/null) =~ Ubuntu ]]; then ubuntu=1; fi +if [[ $(cat /etc/redhat-release 2>/dev/null) =~ Red ]]; then redhat=1; fi +if [[ $(grep platform /proc/cpuinfo 2>/dev/null) =~ pSeries ]]; then VM=1; fi + +CF_VER=$(cat ver.txt) + +################################################################################ +################################################################################ +################################################################################ +# # +# # +# Ubuntu # +# # +# # +################################################################################ +################################################################################ +################################################################################ +if [[ $ubuntu ]] +then + + ################################## + # install Adv Toolchain and others + if [[ -z $(dpkg -l|grep $ADV_TC) && $internet -ne 0 ]] + then + echo "ERROR: unable to reach the internet for required packages. A firewall may be up, telnet to www.ibm.com and rerun to update" + else + echo "------------------------------" + echo -e "INFO: Checking other dependencies\n" + + if [[ $(dpkg -l | egrep -c "i software-properties-common|i python3-software-properties|i python3 |i libudev1:|i libudev-dev:|i sg3-utils ") -ne 6 ]] + then + apt-get -y install software-properties-common python3-software-properties python3 libudev1 libudev-dev sg3-utils + fi + if [[ -z $(dpkg -l|grep $ADV_TC) ]] + then + if [[ $(dpkg -l|egrep -c "i make |i cscope |i doxygen |i git |i gitk ") -ne 5 ]] + then + apt-get -fy install make cscope ctags doxygen git + fi + wget ftp://ftp.unicamp.br/pub/linuxpatch/toolchain/at/ubuntu/dists/precise/6976a827.gpg.key + apt-key add 6976a827.gpg.key + add-apt-repository "deb ftp://ftp.unicamp.br/pub/linuxpatch/toolchain/at/ubuntu trusty at10.0 " + apt-get -y update + apt-get -y install advance-toolchain-at10.0-runtime advance-toolchain-at10.0-perf advance-toolchain-at10.0-devel advance-toolchain-at10.0-mcore-libs + fi + fi + + if [[ -z $(dpkg -l|grep "i libudev1:") ]]; then echo "ERROR: libudev1 is not installed."; fi + if [[ -z $(dpkg -l|grep $ADV_TC) ]]; then echo "ERROR: $ADV_TC is not installed."; fi + + ################################## + # Check usr-mode software version + echo "------------------------------" + echo -e "INFO: Checking software versions\n" + + if [[ $ubuntu ]] + then + ls *cxlflash*.deb >/dev/null 2>&1 + if [[ $? -ne 0 ]] + then + echo "ERROR: no cxlflash pkgs found in the current dir" + exit 2 + fi + + if [[ $1 =~ mfg && ! -e /opt/ibm/capikv/lib/libafu.so ]]; then force=yes; fi + + #remove old + EXISTS_LVL=$(dpkg -l|egrep "i ibmcapikv-test|i cxlflash-test") + if [[ $EXISTS_LVL =~ ibmcapikv-test ]] + then + dpkg -r ibmcapikv-test + fi + if [[ $EXISTS_LVL =~ cxlflash-test && ! $EXISTS_LVL =~ $CF_VER ]] + then + dpkg -r cxlflash-test + fi + + EXISTS_LVL=$(dpkg -l|grep -v test|egrep "i ibmcapikv |i cxlflash ") + if [[ $EXISTS_LVL =~ ibmcapikv ]] + then + dpkg -r ibmcapikv + fi + if [[ $EXISTS_LVL =~ cxlflash && ! $EXISTS_LVL =~ $CF_VER ]] + then + dpkg -r cxlflash + fi + + EXISTS_LVL=$(dpkg -l|grep -v test|egrep "i afuimage |i cxlflashimage ") + if [[ $EXISTS_LVL =~ afuimage ]] + then + dpkg -r afuimage + fi + if [[ $EXISTS_LVL =~ cxlflashimage && ! $EXISTS_LVL =~ $CF_VER ]] + then + dpkg -r cxlflashimage + fi + + #install new + EXISTS_LVL=$(dpkg -l|grep -v test|egrep "i cxlflash ") + if [[ ! $EXISTS_LVL =~ "i cxlflash " || $force =~ yes || \ + ($EXISTS_LVL =~ "i cxlflash " && ! $EXISTS_LVL =~ $CF_VER) ]] + then + dpkg -i cxlflash_*$CF_VER*.deb + fi + + EXISTS_LVL=$(dpkg -l|egrep "i cxlflash-test") + if [[ ! $EXISTS_LVL =~ "i cxlflash-test" || \ + ($EXISTS_LVL =~ cxlflash-test && ! $EXISTS_LVL =~ $CF_VER) ]] + then + dpkg -i cxlflash-test*$CF_VER*.deb + fi + + EXISTS_LVL=$(dpkg -l|grep 'i cxlflashimage') + if [[ ! $EXISTS_LVL =~ cxlflashimage || \ + ($EXISTS_LVL =~ cxlflashimage && ! "$EXISTS_LVL" =~ $CF_VER) ]] + then + dpkg -i cxlflashimage*$CF_VER*.deb + fi + fi + + ################################## + # Check LIC + if [[ ! $1 =~ mfg ]] + then + if [[ -e /tmp/cxlflash_headless ]] + then + echo "INFO: running headless" + date > /opt/.lic/$LD + else + if [[ ! $1 =~ dev ]] + then + if [[ ! -e /opt/.lic/$LD ]] + then + echo "" + echo "Please review the licenses available in /opt/ibm/capikv/license/" + echo "Press 'y' to accept the license agreement, or skip now and rerun later" + read char + if [[ $char = [yY] ]] + then + date > /opt/.lic/$LD + if [[ -e /opt/$HD && ! -e /opt/ibm/capikv/lib/libcflsh_block.so ]] + then + mv /opt/$HD/* /opt/ibm/capikv/lib + fi + rm -R /opt/$HD >/dev/null 2>&1 + else + echo "ERROR: The license must be accepted" + if [[ -e /opt/ibm/capikv/lib ]] + then + if [[ ! -e /opt/$HD ]]; then mkdir /opt/$HD; fi + mv /opt/ibm/capikv/lib/* /opt/$HD + fi + exit 128 + fi + fi + fi + fi + fi + + ################################## + # Check OS version + OS_VER=16.04.1 + if [[ $1 =~ dev ]] + then + echo "------------------------------" + echo -e "INFO: Checking OS version\n" + fi + + if [[ ! $(lsb_release -a 2>/dev/null) =~ $OS_VER ]] + then + if [[ $internet -ne 0 ]] + then + echo "ERROR: unable to reach the internet for OS updates. A firewall may be up, telnet to www.ibm.com and rerun to update your OS" + else + if [[ ! $(lsb_release -a 2>/dev/null) =~ $OS_VER ]] + then + do_upgrade=0 + if [[ ! -e /tmp/cxlflash_headless ]] + then + echo "The linux OS must be updated to $OS_VER" + echo " enter 'y' to continue, or to skip the upgrade to $OS_VER" + read -r yes + if [[ $yes = [Yy] ]] + then + do_upgrade=1 + elif [[ ! $1 =~ dev ]] + then + echo "aborting the install, retry when the OS may be upgraded to $OS_VER" + exit 11 + fi + fi + if [[ $do_upgrade -eq 1 ]] + then + apt-get -y install update-manager-core + apt-get -y install --reinstall apt + apt-get -y update + apt-get -y upgrade + do-release-upgrade + apt-get -y dist-upgrade + fi + fi + fi + fi + + ###################################### + # Check the recommended kernel version + if [[ -z $(dpkg -l|grep -v libc|grep "i linux-image-$K_VER_GRUB") ]] + then + apt-get -y install linux-image-"$K_VER_INST"-generic + fi + if [[ -z $(dpkg -l|grep -v libc|grep "i linux-image-extra-$K_VER_GRUB") ]] + then + apt-get -y install linux-image-extra-"$K_VER_INST"-generic + fi + if [[ -z $(dpkg -l|grep -v libc|grep "i linux-tools-$K_VER_GRUB") ]] + then + apt-get -y install linux-tools-"$K_VER_INST"-generic + fi + + ################################## + # check grub.cfg + if [[ ! $(grep initrd /boot/grub/grub.cfg|head -1) =~ $K_VER_GRUB ]] + then + echo -e "\n------------------------------" + echo -e "INFO: Fixing grub.cfg\n" + ln=$(grep -n "menuentry 'Ubuntu" /boot/grub/grub.cfg|head -1|awk -F: '{print $1}') + ln=$((ln-1)) + head -$ln /boot/grub/grub.cfg > /tmp/new + TXT=$(grep -A16 "with Linux $K_VER_GRUB'" /boot/grub/grub.cfg) + echo "$TXT" >> /tmp/new + awk -v ln=$ln 'NR>ln' /boot/grub/grub.cfg >> /tmp/new + cp /boot/grub/grub.cfg /boot/grub/grub.cfg.$(date|awk '{print $4"_"$2"_"$3"_"$6}'|sed 's/:/_/'g) + mv /tmp/new /boot/grub/grub.cfg + fi + + ################################## + # Check cxlflash pkgs + if [[ -z $(dpkg -l|grep "i cxlflash") ]] + then + echo -e "\nERROR: cxlflash packages failed to install\n" + exit 15 + fi +fi + + +################################################################################ +################################################################################ +################################################################################ +# # +# # +# RedHat # +# # +# # +################################################################################ +################################################################################ +################################################################################ +if [[ $redhat ]] +then + K_VER="3.10.0" + K_VER_INST="3.10.0" + K_VER_GRUB="3.10.0" + CF_VER=$(cat ver.txt | sed 's,-,_,g') + + ################################## + # install Adv Toolchain and others + if [[ -z $(rpm -qa|grep $ADV_TC) && $internet -ne 0 ]] + then + echo "ERROR: unable to reach the internet for required packages. A firewall may be up, telnet to www.ibm.com and rerun to update" + else + echo "------------------------------" + echo -e "INFO: Checking other depencencies\n" + fi + if [[ $(rpm -qa | egrep -c -e "sg3_utils-|systemd-devel" ) -ne 3 ]] + then + yum -y install sg3_utils systemd-devel + fi + if [[ -z $(rpm -qa | grep $ADV_TC) ]] + then + if [[ $(rpm -qa| egrep -c -e"make|cscope|doxygen|git-") -ne 4 ]] + then + yum -y install make cscope ctags doxygen git + fi + + #These commands retrieved from : + wget ftp://ftp.unicamp.br/pub/linuxpatch/toolchain/at/redhat/RHEL7/gpg-pubkey-6976a827-5164221b + rpm --import gpg-pubkey-6976a827-5164221b + + if [[ ! -e /etc/yum.repos.d/atX.X.repo ]] + then + #create repo configuration file for advanced tool chain + cat > /etc/yum.repos.d/atX.X.repo < /etc/yum.repos.d/rhel.kte.repo </dev/null 2>&1 + if [[ $? -ne 0 ]] + then + echo "ERROR: no cxlflash pkgs found in the current dir" + exit 2 + fi + + if [[ $1 =~ mfg && ! -e /opt/ibm/capikv/lib/libafu.so ]]; then force=yes; fi + + ################ + #remove old + EXISTS_LVL=$(rpm -qa|egrep "ibmcapikv-test|cxlflash-test") + if [[ $EXISTS_LVL =~ ibmcapikv-test ]] + then + rpm -e ibmcapikv-test + fi + if [[ $EXISTS_LVL =~ cxlflash-test && ! $EXISTS_LVL =~ $CF_VER ]] + then + rpm -e -v cxlflash-test + fi + + EXISTS_LVL=$(rpm -qa|grep -v test|egrep "ibmcapikv|cxlflash") + if [[ $EXISTS_LVL =~ ibmcapikv ]] + then + rpm -e -v ibmcapikv + fi + if [[ $EXISTS_LVL =~ cxlflash && ! $EXISTS_LVL =~ $CF_VER ]] + then + rpm -e -v cxlflash + fi + + EXISTS_LVL=$(rpm -qa|grep -v test|egrep "afuimage|cxlflashimage") + if [[ $EXISTS_LVL =~ afuimage ]] + then + rpm -e afuimage + fi + if [[ $EXISTS_LVL =~ cxlflashimage && ! $EXISTS_LVL =~ $CF_VER ]] + then + rpm -e cxlflashimage + fi + + ################# + #install new: redhat if another version exists can't install + EXISTS_LVL=$(rpm -qa|grep -v test|egrep "cxlflash") + if [[ ! $EXISTS_LVL =~ "cxlflash" || $force =~ yes || \ + ($EXISTS_LVL =~ "cxlflash" && ! $EXISTS_LVL =~ $CF_VER) ]] + then + rpm -i -v cxlflash-*$CF_VER*.rpm + fi + + EXISTS_LVL=$(rpm -qa|egrep "cxlflash-test") + if [[ ! $EXISTS_LVL =~ "cxlflash-test" || \ + ($EXISTS_LVL =~ cxlflash-test && ! $EXISTS_LVL =~ $CF_VER) ]] + then + rpm -i -v cxlflash-test*$CF_VER*.rpm + fi + + EXISTS_LVL=$(rpm -qa|grep 'cxlflashimage') + if [[ ! $EXISTS_LVL =~ cxlflashimage || \ + ($EXISTS_LVL =~ cxlflashimage && ! "$EXISTS_LVL" =~ $CF_VER) ]] + then + rpm -i -v cxlflashimage*$CF_VER*.rpm + fi + + ################################## + # Check LIC + if [[ ! $1 =~ mfg ]] + then + if [[ -e /tmp/cxlflash_headless ]] + then + echo "INFO: running headless" + date > /opt/.lic/$LD + else + if [[ ! $1 =~ dev ]] + then + if [[ ! -e /opt/.lic/$LD ]] + then + echo "" + echo "Please review the licenses available in /opt/ibm/capikv/license/" + echo "Press 'y' to accept the license agreement, or skip now and rerun later" + read char + if [[ $char = [yY] ]] + then + date > /opt/.lic/$LD + if [[ -e /opt/$HD && ! -e /opt/ibm/capikv/lib/libcflsh_block.so ]] + then + mv /opt/$HD/* /opt/ibm/capikv/lib + fi + rm -R /opt/$HD >/dev/null 2>&1 + else + echo "ERROR: The license must be accepted" + if [[ -e /opt/ibm/capikv/lib ]] + then + if [[ ! -e /opt/$HD ]]; then mkdir /opt/$HD; fi + mv /opt/ibm/capikv/lib/* /opt/$HD + fi + exit 128 + fi + fi + fi + fi + fi + + ################################## + # Check OS version + OS_VER=7.3 + if [[ $1 =~ dev ]] + then + echo "------------------------------" + echo -e "INFO: Checking OS version\n" + fi + + if [[ ! $(grep "VERSION_ID" /etc/os-release | awk -F'"' '{print $2}') =~ $OS_VER || -z $(rpm -q kernel |grep $K_VER) ]] + then + if [[ $internet -ne 0 ]] + then + echo "ERROR: unable to reach the internet for OS updates. A firewall may be up, telnet to www.ibm.com and rerun to update your OS" + else + if [[ ! $(grep "VERSION_ID" /etc/os-release | awk -F'"' '{print $2}' ) =~ $OS_VER ]] + then + do_upgrade=0 + if [[ ! -e /tmp/cxlflash_headless ]] + then + echo "The linux OS must be updated to $OS_VER" + echo " enter 'y' to continue, or to skip the upgrade to $OS_VER" + read -r yes + if [[ $yes = [Yy] ]] + then + do_upgrade=1 + elif [[ ! $1 =~ dev ]] + then + echo "aborting the install, retry when the OS may be upgraded to $OS_VER" + exit 11 + fi + fi + if [[ $do_upgrade -eq 1 ]] + then + echo "Don't know how to upgrade to $OS_VER. You must updgrade." + #TODO: + #yum -y install update-manager-core + #yum -y install --reinstall apt + #yum -y update + #yum -y upgrade + #do-release-upgrade + #yum -y dist-upgrade + fi + fi + + if [[ -z $(rpm -q kernel | grep $K_VER) ]] + then + echo "Don't know how to upgrade to $K_VER kernel. You must updgrade." + #TODO: + #yum -y install linux-image-"$K_VER_INST"-generic + #yum -y install linux-image-extra-"$K_VER_INST"-generic + #yum -y install linux-tools-"$K_VER_INST"-generic + #yum -y install linux-headers-"$K_VER_INST"-generic + fi + fi + fi + + ################################## + # Check cxlflash pkgs + if [[ -z $(rpm -qa|grep "cxlflash") ]] + then + echo -e "\nERROR: cxlflash packages failed to install\n" + exit 15 + fi +fi + + +################################################################################ +################################################################################ +################################################################################ +# # +# # +# Common # +# # +# # +################################################################################ +################################################################################ +################################################################################ + + ################################## + # Check AFU version + echo -e "\n------------------------------" + echo -e "INFO: Checking firmware versions\n" + + systemctl stop cxlfd.service + if [[ $1 =~ dev && -e /sbin/multipath && -n $(lsmod | grep multipath) ]]; then multipath -F; fi + + for x in 1 2 3 + do + lsmod | grep cxlflash >/dev/null 2>&1 + if [[ $? -eq 0 ]] + then + rmmod cxlflash >/dev/null 2>&1 + if [[ $? -ne 0 && $x -eq 3 ]] + then + lsof|grep cxlflash + echo "ERROR: unable to unload cxlflash" + if [[ ! $1 =~ dev && -e /sbin/multipath && -n $(lsmod | grep multipath) ]] + then + echo " you could try 'multipath -F'" + fi + exit 3 + else + sleep 1 + fi + fi + done + + adaps=$(lspci|grep 0601|awk '{print $1}') #FlashGT + + flash_all_adapters -reload + sleep 2 + + if [[ -z $(lsmod |grep cxlflash) ]]; then modprobe cxlflash; fi + if [[ $? -ne 0 ]] + then + echo "ERROR: unable to reload cxlflash" + exit 4 + fi + echo "" + + systemctl restart cxlfd.service + + ################################## + # Re-check all versions + echo -e "\n------------------------------" + cflash_version $K_VER + if [[ $? -ne 0 ]] + then + echo -e "\n\n*************************" + echo -e "ERROR: ABORTING..." + echo -e "*************************\n\n" + exit 5 + fi + echo -e "\nINFO: Installed Versions....OK\n" + + ################################## + # Check for FlashGT devices + echo -e "\n------------------------------" + echo -e "INFO: Checking for devices\n" + + #check that sg devices show up + adapN=0 + for adap in $adaps; do let adapN=$adapN+1; done + if [[ $adapN -eq 0 ]] + then + echo "INFO: no capiflash EJ1K adapters found" + exit 0 + fi + + if [[ $VM ]] + then + sgs=$(for adap in $adaps + do find /sys|grep $adap|grep scsi_generic|grep "sg"|grep subsystem|awk -F/ '{print $12}' |tr -d '\r' + done) + else + sgs=$(for adap in $adaps + do find /sys|grep $adap|grep scsi_generic|grep "sg"|grep subsystem|awk -F/ '{print $13}' |tr -d '\r' + done) + fi + + sglist="" + sgN=0 + for sg in $sgs; do sgN=$((sgN+1)); sglist="$sglist /dev/$sg "; done + sglist=$(echo $sglist|tr -d '\r') + if [[ $sgN -ne $((adapN*2)) ]] #FlashGT + then + echo "ERROR: #adaps($adapN) should have $((adapN*2)) sg devices, but there are only $sgN devices" + exit 7 + fi + + ################################## + # Check IO + echo -e "\n------------------------------" + echo -e "INFO: Checking IO\n" + unset CFLSH_BLK_TRC_VERBOSITY + + for sg in $sglist + do + legacy=0 + if [[ $(cxlfstatus|grep ${sg:5:6}) =~ legacy ]] + then + legacy=1 + wwid=$(cxlfstatus|grep ${sg:5:6}|awk '{print $5}') + cxlfsetlunmode $wwid 1 >/dev/null + fi + iops=$(LD_LIBRARY_PATH=/opt/ibm/capikv/lib /opt/ibm/capikv/bin/blockio -d $sg -s5 2>&1) + if [[ ! $iops =~ err && $(echo $iops|awk -Fiops: '{print $2}') -gt 140000 ]] + then + echo " $sg OK" + else + echo " $sg failed ($iops)" + rc=8 + fi + if [[ $legacy -eq 1 ]]; then cxlfsetlunmode $wwid 0 >/dev/null; fi + done + if [[ $rc -ne 0 ]]; then echo "ERROR: IO Test failed"; exit $rc; fi + + if [[ $1 =~ mfg ]] + then + if [[ -e /opt/.lic/$LD ]]; then rm -f /opt/.lic/$LD; fi + if [[ ! -e /opt/$HD ]]; then mkdir /opt/$HD; fi + mv /opt/ibm/capikv/lib/* /opt/$HD + fi + + echo -e "\nINSTALL success" + +exit 0 diff --git a/src/build/install/resources/cflash_devices.pl b/src/build/install/resources/cflash_devices.pl new file mode 100755 index 00000000..e3432ca7 --- /dev/null +++ b/src/build/install/resources/cflash_devices.pl @@ -0,0 +1,148 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_devices.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +## +use strict; +use warnings; +use Fcntl; +use Fcntl ':seek'; +use Getopt::Long; + +#------------------------------------------------------------------------------- +# Variables +my $type="0601"; +my $prthelp=0; +my $single; +my $verbose; + +sub usage() +{ + print "\n"; + print "Usage: cflash_devices.pl [-t type] [-s] [-v] \n"; + print " -t or --type : Type to list \n"; + print " -s or --single : Output is a single line \n"; + print " -v : verbose for debug \n"; + print " -h or --help : Help - Print this message \n"; + print "\n ex: cflash_devices.pl -t 0601 \n\n"; + exit 0; +} + +#------------------------------------------------------------------------------- +# Parse Options +#------------------------------------------------------------------------------- +if (! GetOptions ("t|type=s" => \$type, + "s" => \$single, + "v" => \$verbose, + "h|help!" => \$prthelp + )) +{ + usage(); +} +if ($ARGV[0]) +{ + print "\nUnknown Command Line Options:\n"; + foreach(@ARGV) + { + print " $_\n"; + } + print "\n"; + usage(); +} +if ($prthelp) {usage();} + +#------------------------------------------------------------------------------- +# Make stdout autoflush +#------------------------------------------------------------------------------- +select(STDOUT); +$| = 1; + +#------------------------------------------------------------------------------- +# get wwids matching $type +#------------------------------------------------------------------------------- +my @cards = `lspci |grep $type`; +my $cmd=""; +my @list; +my @ids; +my $sline=""; +my @devs; +my $dev; +my $wwid; +my %lookup; +my @wwids; +my @sorted; + +for my $adap (@cards) +{ + my @Acard=split / /, $adap; + my $card=$Acard[0]; + chomp $card; + my @D=`ls -d /sys/devices/*/*/$card/*/*/host*/target*:*:*/*:*:*:*/scsi_generic/*|awk -F/ '{print \$13}' 2>/dev/null`; + for $dev (@D) + { + chomp $dev; + push(@devs, $dev); + if ($verbose) {print "(" . join(" ", @devs) . ")\n";} + } +} + +for $dev (@devs) +{ + my $tmp=`/lib/udev/scsi_id -d /dev/$dev --page=0x83 --whitelisted`; + $wwid=substr $tmp,1; + chomp $wwid; + #only use devs in superpipe mode + my $out=`grep $wwid /opt/ibm/capikv/etc/sioluntable.ini|grep =1`; + chomp $out; + $verbose && print "$dev $out\n"; + push(@list,$wwid); + $lookup{$dev} = $wwid; +} + +@wwids = sort @list; + +$verbose && print "Sorted by wwid:\n"; + +foreach $wwid (@wwids) +{ + foreach my $key ( keys %lookup ) + { + if ($lookup{$key} =~ /$wwid/) + { + $verbose && print "$key=>$lookup{$key}\n"; + push(@sorted,$key); + $lookup{$key} = "empty"; + last; + } + } +} + +if (! @sorted) {exit 0;} + +if ($single) {my $out="/dev/" . join(":/dev/",@sorted); print "$out\n"; exit 0;} + +for $dev (@sorted) {print "/dev/$dev\n";} + +exit 0; + diff --git a/src/build/install/resources/cflash_inject.pl b/src/build/install/resources/cflash_inject.pl new file mode 100755 index 00000000..79835847 --- /dev/null +++ b/src/build/install/resources/cflash_inject.pl @@ -0,0 +1,220 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_inject.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +## +use strict; +use warnings; +use Fcntl; +use Fcntl ':seek'; +use Getopt::Long qw(:config no_ignore_case); + +#------------------------------------------------------------------------------- +# Variables +#------------------------------------------------------------------------------- +my $dev; +my $afustr; +my $afu=-1; +my $port=-1; +my $entry=0; +my $temp=-1; +my $wear=-1; +my $reset; +my $offline; +my $parity_error; +my $rd; +my $wr; +my $lba=0; +my $status=4; +my $clear; +my $prthelp=0; +my $verbose; +my $cmd; +my $vparm=""; +my $data; +my $parms; +my $opcode="0x02"; +my $perm; +my $parm=""; + +sub usage() +{ + print "\n"; + print "Usage: \n"; + print " -d or --dev : sg dev to inject \n"; + print " -a or --afu : afu to inject \n"; + print " -p or --port : port to inject, 0-nvme0 or 1-nvme1 \n"; + print " -e or --entry : entry to inject (up to 4 active temp/wear injects per port) \n"; + print " -T or --temp : temp to inject \n"; + print " -W or --wear : Wear percentage to inject \n"; + print " -reset : reset NVMe port \n"; + print " -offline : set NVMe port offline \n"; + print " -parity_error : inject parity error on a write \n"; + print " -rd : inject read error \n"; + print " -wr : inject write error \n"; + print " -lba : lba to inject a r/w error \n"; + print " -s or --status : status code for r/w error inject \n"; + print " -P or --perm : make the r/w inject for an afu/port permanent \n"; + print " -C or --clear : clear the inject for an afu/port/entry \n"; + print " -h or --help : Help - Print this message \n\n"; + print " *set sg9, to wear% 33, cflash_inject.pl -d sg9 -W 33 \n"; + print " *set sg9, to temp 64, cflash_inject.pl -d sg9 -T 64 \n"; + print " *clear sg9, wear or temp inject cflash_inject.pl -d sg9 -C \n"; + print " *reset sg9 NVMe port, cflash_inject.pl -d sg9 -reset \n"; + print " *set sg9 NVMe port offline, cflash_inject.pl -d sg9 -offline \n"; + print " *set sg9 NVMe port back online, cflash_inject.pl -d sg9 -reset \n"; + print " *set sg9 rd err, lba 10, status 1, cflash_inject.pl -d sg9 -rd -lba 10 -status 1 \n"; + print " *set sg9 wr err, lba 90, status 4, cflash_inject.pl -d sg9 -wr -lba 90 -status 4 \n"; + print " *perm sg9 wr err, lba 90, status 4, cflash_inject.pl -d sg9 -wr -lba 90 -s 4 -P \n"; + print " *clear sg9 wr err, cflash_inject.pl -d sg9 -wr -C \n"; + print " *set sg9 parity write error, cflash_inject.pl -d sg9 -parity_error \n\n"; + exit 0; +} + +#------------------------------------------------------------------------------- +# Parse Options +#------------------------------------------------------------------------------- +if ($#ARGV<0) {usage();} + +if (! GetOptions ("d|dev=s" => \$dev, + "a|afu=i" => \$afu, + "p|port=i" => \$port, + "e|entry=i" => \$entry, + "T|temp=i" => \$temp, + "W|wear=i" => \$wear, + "C|clear" => \$clear, + "reset|online" => \$reset, + "offline" => \$offline, + "parity_error" => \$parity_error, + "rd" => \$rd, + "wr" => \$wr, + "lba=i" => \$lba, + "s|status=i" => \$status, + "C|clear" => \$clear, + "P|perm" => \$perm, + "v" => \$verbose, + "h|help" => \$prthelp + )) +{ + usage(); +} +if ($ARGV[0]) +{ + print "\nUnknown Command Line Options:\n"; + foreach(@ARGV) + { + print " $_\n"; + } + print "\n"; + usage(); +} +if ($prthelp || (!$dev && ($afu==-1 || $port==-1))) {usage();} + +#check sudo permissions +if (`id -u` != 0) {print "Run with sudo permissions\n"; exit 0;} + +#------------------------------------------------------------------------------- +# Make stdout autoflush +#------------------------------------------------------------------------------- +select(STDOUT); +$| = 1; + +if ($dev) +{ + if ($dev =~ /\/dev\//) {$dev=substr $dev,5,3;} + my $sgdev=`ls -d /sys/devices/pci*/*/00*/*/*/host*/target*:*:*/*:*:*:*/scsi_generic/$dev 2>/dev/null`; + if ($?!=0) {print "sg device ($dev) was not found\n"; exit -1;} + my @sgdirs=split /\//, $sgdev; + my $cardstr=`ls -d /sys/devices/*/*/$sgdirs[5]/cxl/card* 2>/dev/null`; + chomp $cardstr; + $afustr="/dev/cxl/afu" . substr($cardstr,-1) . ".0m"; + my @target=split /:/, $sgdirs[9]; + $port=$target[1]; +} +else +{ + $afustr=`ls /dev/cxl/afu$afu.0m 2>/dev/null`; + if (length $afustr) + { + chomp $afustr; + } + else + { + print "invalid card number: $afu\n"; + exit 0; + } +} + +if ($verbose) {$vparm=" -v "} +if ($perm) {$parm=" -P "} + +if ($parity_error) +{ + $cmd="/opt/ibm/capikv/test/cxl_afu_inject -d $afustr -p $port -a 9 $vparm" +} +elsif ($reset) +{ + $cmd="/opt/ibm/capikv/test/cxl_afu_inject -d $afustr -p $port -a 0 $vparm" +} +elsif ($offline) +{ + $cmd="/opt/ibm/capikv/test/cxl_afu_inject -d $afustr -p $port -a 1 $vparm" + +} +elsif (($rd || $wr) && $clear) +{ + $cmd="/opt/ibm/capikv/test/cxl_afu_inject -d $afustr -p $port -a 8 $vparm" +} +elsif ($rd) +{ + $cmd="/opt/ibm/capikv/test/cxl_afu_inject -d $afustr -p $port -a 2 -s $status -l $lba $parm $vparm" +} +elsif ($wr) +{ + $cmd="/opt/ibm/capikv/test/cxl_afu_inject -d $afustr -p $port -a 3 -s $status -l $lba $parm $vparm" +} +elsif ($clear) +{ + $opcode="0xff"; + $parms="$afustr --opcode $opcode --param 0x02 --port $port --entry $entry --offset 4 --mask 0xFFFF00FF --data 0"; + $cmd="/opt/ibm/capikv/test/flashgt_nvme_override $parms"; +} +elsif ($wear>=0) +{ + $data=sprintf("0x%.2x00", $wear); + $parms="$afustr --opcode $opcode --param 0x02 --port $port --entry $entry --offset 4 --mask 0xFFFF00FF --data $data"; + $cmd="/opt/ibm/capikv/test/flashgt_nvme_override $parms"; +} +elsif ($temp>=0) +{ + $data=sprintf("0x0%.3x00", $temp+273); + $parms="$afustr --opcode $opcode --param 0x02 --port $port --entry $entry --offset 0 --mask 0xFF0000FF --data $data"; + $cmd="/opt/ibm/capikv/test/flashgt_nvme_override $parms"; +} + +system($cmd); +if ($? != 0) {print "ERROR "} +else {$verbose && print "SUCCESS "} +$verbose && print ": $cmd\n"; + diff --git a/src/build/install/resources/cflash_logs_cron b/src/build/install/resources/cflash_logs_cron new file mode 100755 index 00000000..0b9aa451 --- /dev/null +++ b/src/build/install/resources/cflash_logs_cron @@ -0,0 +1,39 @@ +#!/bin/bash +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_logs_cron $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +LOGDIR=/var/log/cflash +if [[ -e $LOGDIR && $(du -sm $LOGDIR|awk '{print $1}') -gt 10000 ]] +then + num=$(ls -1tr $LOGDIR|grep tgz|wc -l) + if [[ $num -gt 4 ]] + then + num=$((num/4)) + for file in $(ls -1tr $LOGDIR|grep tgz|head -$num) + do + rm -f $LOGDIR/$file + done + fi +fi diff --git a/src/build/install/resources/cflash_perf.pl b/src/build/install/resources/cflash_perf.pl new file mode 100755 index 00000000..55b71c68 --- /dev/null +++ b/src/build/install/resources/cflash_perf.pl @@ -0,0 +1,186 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_perf.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +## +use strict; +use warnings; +use Fcntl; +use Fcntl ':seek'; +use Getopt::Long; + +my $adap; +my $ci=0; +my @devs; +my $di=0; +my @start; +my @last; +my @last_rds; +my $cfg_devid; +my @stime; +my $time; +my $sleep=2; +my $secs; +my $long; +my $prthelp; +my $pstr; +my $iops; +my $tiops; +my $rd_iops; +my $t_rd_iops; +my $ratio; + +#------------------------------------------------------------------------------- +# print usage +#------------------------------------------------------------------------------- +sub usage() +{ + print "\n"; + print "Usage: [-l] [-s secs]\n"; + print " -l : histogram format \n"; + print " -s : update interval seconds \n"; + print " -h or --help : Help - Print this message \n\n"; + exit 0; +} + +#check sudo permissions +(`id -u` == 0) || die "Run with sudo permissions"; + +#------------------------------------------------------------------------------- +# Make stdout autoflush +#------------------------------------------------------------------------------- +select(STDOUT); +$| = 1; + +#------------------------------------------------------------------------------- +# Main +#------------------------------------------------------------------------------- +if (@ARGV==0) {usage();} + +if (! GetOptions ("long" => \$long, + "s=i" => \$sleep, + "h|help!" => \$prthelp, + )) +{ + usage(); +} +if ($ARGV[0]) +{ + print "\nUnknown Command Line Options:\n"; + foreach(@ARGV) + { + print " $_\n"; + } + print "\n"; + usage(); +} +if ($prthelp) {usage();} +$sleep = $sleep > 60 ? 60 : $sleep; + +#------------------------------------------------------------------------------- +# BMC -list CAPI Devices +#------------------------------------------------------------------------------- + +while (1) +{ + my @files = ; + my ($vendor, $device); + + for my $file ( @files ) + { + open(F, $file . "/vendor") or die "ERROR: Can't open $file: $!"; + read(F, $vendor, 6); + close (F); + open(F, $file . "/device") or die "ERROR: Can't open $file: $!"; + read(F, $device, 6); + close (F); + + if (($vendor eq "0x1014") && ($device eq "0x0601")) + { + my @dds = split /\//, $file; + $adap = pop @dds; + + for (my $i=0; $i<2; $i++) + { + my $devstr=`ls -d /sys/devices/*/*/$adap/*/*/host*/target*:$i:*/*:*:*:*/scsi_generic/* 2>/dev/null`; + if ($? != 0) {next;} + my @sdev=split /\//, $devstr; + $devs[$di]=$sdev[12]; + chomp $devs[$di]; + my $rds=`sg_logs /dev/$devs[$di] -a|grep "read commands ="|awk '{print \$6}'`; + my $wrs=`sg_logs /dev/$devs[$di] -a|grep "write commands ="|grep -v plus|awk '{print \$6}'`; + chomp $rds; + chomp $wrs; + $last[$di]=$rds+$wrs; + $last_rds[$di]=$rds; + $stime[$di]=time; + ++$di; + } + } + } + + sleep 1; + + #print iops in infinite loop + while (1) + { + my $cnt; + + sleep $sleep; + + if ($long) {$pstr=`date|awk '{print \$1" "\$2" "\$3" "\$4" => "}'`; chomp $pstr;} + else {$pstr="";} + + $tiops=0; $t_rd_iops=0; $ratio=0; + for (my $i=0; $i<$di; $i++) + { + `sg_logs /dev/$devs[$i] -a >/dev/null 2>&1`; + if ($? !=0) {next;} + my $rds=`sg_logs /dev/$devs[$i] -a|grep "read commands ="|awk '{print \$6}'`; + my $wrs=`sg_logs /dev/$devs[$i] -a|grep "write commands ="|grep -v plus|awk '{print \$6}'`; + $secs=time-$stime[$i]; + $stime[$i]=time; + chomp $rds; + chomp $wrs; + $cnt=$rds+$wrs; + $iops=($cnt-$last[$i])/$secs; + $tiops+=$iops; + $rd_iops=($rds-$last_rds[$i])/$secs; + $t_rd_iops+=$rd_iops; + $last[$i]=$cnt; + $last_rds[$i]=$rds; + my $str=sprintf(" %s:%-7d", $devs[$i], $iops); + $pstr.=$str; + } + if ($t_rd_iops>0) {$ratio=$t_rd_iops/$tiops;} + if ($tiops>0) {$ratio+=0.005;} + $ratio*=100; + $ratio=$ratio > 100 ? 100 : $ratio; + if ($long) {printf("%s total:%-8d rds:%d%%\n", $pstr, $tiops, $ratio);} + else {printf("%s total:%-8d rds:%d%% \r", $pstr, $tiops, $ratio);} + + if ($di == 0) {last;} + } +} + diff --git a/src/build/install/resources/cflash_perfcheck.pl b/src/build/install/resources/cflash_perfcheck.pl new file mode 100644 index 00000000..ee1a2928 --- /dev/null +++ b/src/build/install/resources/cflash_perfcheck.pl @@ -0,0 +1,167 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_perfcheck.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +## + +use strict; +use warnings; +use Fcntl; +use Fcntl ':seek'; +use Getopt::Long; +use Time::HiRes qw(usleep); + +my @devs; +my $devstr=""; +my $prthelp=""; +my $all; +my $verbose=""; +my $out_vrd; +my $out_vwr; +my $out_prd; +my $out_pwr; + +#------------------------------------------------------------------------------- +# print usage +#------------------------------------------------------------------------------- +sub usage() +{ + print "\n"; + print "Usage: [-d /dev/sgX:/dev/sgY] [-a] \n"; + print " -d : device(s) to use \n"; + print " -a : run All checks (ioppt, multi-process)\n"; + print " -h or --help : Help - Print this message \n"; + exit 0; +} + +#------------------------------------------------------------------------------- +# Main +#------------------------------------------------------------------------------- +if (@ARGV==0) {usage();} + +if (! GetOptions ("d=s" => \$devstr, + "h|help!" => \$prthelp, + "a" => \$all, + "v" => \$verbose, + )) +{ + usage(); +} +if ($ARGV[0]) +{ + print "\nUnknown Command Line Options:\n"; + foreach(@ARGV) + { + print " $_\n"; + } + print "\n"; + usage(); +} +if ($prthelp) {usage();} + +$ENV{'PATH'}="$ENV{'PATH'}:/opt/ibm/capikv/bin:/opt/ibm/capikv/test:/opt/ibm/capikv/afu"; + +print "latency vlun plun\n"; +$out_vrd=`blockio -d $devstr -s8 -q1 |awk -Flat: \'{print \$2}\'|awk \'{print \$1}\'`; +$out_vwr=`blockio -d $devstr -s8 -q1 -r0 |awk -Flat: \'{print \$2}\'|awk \'{print \$1}\'`; +$out_prd=`blockio -d $devstr -s8 -q1 -p |awk -Flat: \'{print \$2}\'|awk \'{print \$1}\'`; +$out_pwr=`blockio -d $devstr -s8 -q1 -p -r0|awk -Flat: \'{print \$2}\'|awk \'{print \$1}\'`; +printf " rd %7d %7d\n", $out_vrd, $out_prd; +printf " wr %7d %7d\n", $out_vwr, $out_pwr; + +print "1P QD1\n"; +$out_vrd=`blockio -d $devstr -s8 -q1 |awk -Fiops: \'{print \$2}\'`; +$out_vwr=`blockio -d $devstr -s8 -q1 -r0 |awk -Fiops: \'{print \$2}\'`; +$out_prd=`blockio -d $devstr -s8 -q1 -p |awk -Fiops: \'{print \$2}\'`; +$out_pwr=`blockio -d $devstr -s8 -q1 -p -r0|awk -Fiops: \'{print \$2}\'`; +printf " rd %7d %7d\n", $out_vrd, $out_prd; +printf " wr %7d %7d\n", $out_vwr, $out_pwr; + +print "1P QD128\n"; +$out_vrd=`blockio -d $devstr -s8 -q128 |awk -Fiops: \'{print \$2}\'`; +$out_vwr=`blockio -d $devstr -s8 -q128 -r0 |awk -Fiops: \'{print \$2}\'`; +$out_prd=`blockio -d $devstr -s8 -q128 -p |awk -Fiops: \'{print \$2}\'`; +$out_pwr=`blockio -d $devstr -s8 -q128 -p -r0|awk -Fiops: \'{print \$2}\'`; +printf " rd %7d %7d\n", $out_vrd, $out_prd; +printf " wr %7d %7d\n", $out_vwr, $out_pwr; + +print "1P QD32 N16\n"; +$out_prd=`blockio -d $devstr -s8 -q32 -p -n16 |awk -F"4k-iops:" \'{print \$2}\'`; +$out_pwr=`blockio -d $devstr -s8 -q32 -p -n16 -r0|awk -F"4k-iops:" \'{print \$2}\'`; +printf " rd %7d\n", $out_prd; +printf " wr %7d\n", $out_pwr; + +if ($all) +{ + print "\n"; + print "100% reads 1P QD1 "; + print `ioppt.pl -c \"blockio -d $devstr -s5 -q1\" -n 1`; + print "100% reads 1P QD8 "; + print `ioppt.pl -c \"blockio -d $devstr -s5 -q8\" -n 1`; + print "100% reads 1P QD16 "; + print `ioppt.pl -c \"blockio -d $devstr -s5 -q16\" -n 1`; + print "100% reads 1P QD32 "; + print `ioppt.pl -c \"blockio -d $devstr -s5 -q32\" -n 1`; + print "100% reads 1P QD128 "; + print `ioppt.pl -c \"blockio -d $devstr -s5 -q128\" -n 1`; + print "\n"; + print "70% reads 1P QD1 "; + print `ioppt.pl -c \"blockio -d $devstr -s5 -q1 -r70\" -n 1`; + print "70% reads 1P QD8 "; + print `ioppt.pl -c \"blockio -d $devstr -s5 -q8 -r70\" -n 1`; + print "70% reads 1P QD16 "; + print `ioppt.pl -c \"blockio -d $devstr -s5 -q16 -r70\" -n 1`; + print "70% reads 1P QD32 "; + print `ioppt.pl -c \"blockio -d $devstr -s5 -q32 -r70\" -n 1`; + print "70% reads 1P QD128 "; + print `ioppt.pl -c \"blockio -d $devstr -s5 -q128 -r70\" -n 1`; + print "\n"; + + print "400P QD1 blockio vlun 100% reads "; + print `ioppt.pl -c \"blockio -d $devstr -s15 -q1\" -n 400`; + + print "400P QD1 blockio vlun 100% writes "; + print `ioppt.pl -c \"blockio -d $devstr -s15 -q1 -r0\" -n 400`; + + print "100P QD64 blockio vlun 100% reads "; + print `ioppt.pl -c \"blockio -d $devstr -s10 -q64\" -n 100`; + + print "100P QD64 blockio vlun 100% writes "; + print `ioppt.pl -c \"blockio -d $devstr -s10 -q64 -r0\" -n 100`; + + print "100P QD64 blockio plun 100% reads "; + print `ioppt.pl -c \"blockio -d $devstr -s10 -q64 -p\" -n 100`; + + print "100P QD64 blockio plun 100% writes "; + print `ioppt.pl -c \"blockio -d $devstr -s10 -q64 -r0 -p\" -n 100`; + + print "100P QD64 kv_benchmark 100% reads "; + print `ioppt.pl -c \"run_kv_benchmark -d $devstr -s15 -q64\" -n 100`; + + print "100P QD64 kv_benchmark 75% reads "; + print `ioppt.pl -c \"run_kv_benchmark -d $devstr -s15 -q64 -r75\" -n 100`; + + print "100P QD64 blockio vlun 75% reads "; + print `ioppt.pl -c \"blockio -d $devstr -s10 -q64 -r75\" -n 100`; +} diff --git a/src/build/install/resources/cflash_perst.pl b/src/build/install/resources/cflash_perst.pl new file mode 100755 index 00000000..ea7c65e2 --- /dev/null +++ b/src/build/install/resources/cflash_perst.pl @@ -0,0 +1,177 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_perst.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +## +use strict; +use warnings; +use Fcntl; +use Fcntl ':seek'; +use Getopt::Long; + +#------------------------------------------------------------------------------- +# Variables +#------------------------------------------------------------------------------- +my $target=""; +my $partition0; # reload factory image +my $id="2"; +my $prthelp=0; +my $verbose; +my $v=""; +my $rc; + +my $VM=0; +if ( `cat /proc/cpuinfo | grep platform 2>/dev/null` =~ "pSeries" ) { $VM=1;} +if ( $VM ) { print "\nNot Supported on VM\n\n"; exit -1;} + +sub runcmd +{ + my $cmd=shift; + my $rc=0; + system($cmd); $rc=$?; + if ($rc != 0) + { + print "CMD_FAIL: ($cmd), rc=$rc\n"; + exit -1; + } +} + +sub usage() +{ + print "\n"; + print "Usage: cflash_perst.pl [-t ] \n"; + print " -t or --target : Target device to perst \n"; + print " -h or --help : Help - Print this message \n"; + print "\n ex: cflash_perst.pl -t 000N:01:00.0 \n\n"; + exit 0; +} + +#------------------------------------------------------------------------------- +# Parse Options +#------------------------------------------------------------------------------- +if ($#ARGV<0) {usage();} + +if (! GetOptions ("t|target=s" => \$target, + "p0" => \$partition0, + "v" => \$verbose, + "h|help!" => \$prthelp + )) +{ + usage(); +} +if ($ARGV[0]) +{ + print "\nUnknown Command Line Options:\n"; + foreach(@ARGV) + { + print " $_\n"; + } + print "\n"; + usage(); +} +if ($prthelp) {usage();} + +#check sudo permissions +if (`id -u` != 0) +{ + print "Run with sudo permissions to perst\n"; + exit -2; +} + +#check cxlflash is not loaded +system("lsmod | grep cxlflash >/dev/null"); +if ($? == 0) +{ + print "rmmod cxlflash before perst\n"; + exit -3; +} + +#------------------------------------------------------------------------------- +# Make stdout autoflush +#------------------------------------------------------------------------------- +select(STDOUT); +$| = 1; + +#------------------------------------------------------------------------------- +# list Devices +#------------------------------------------------------------------------------- +my $basedir=`ls -d /sys/devices/*/*/$target/cxl/card* 2>/dev/null`; +my $image="user"; + +chomp $basedir; +if (! -d $basedir) +{ + print "bad target: $target\n"; + exit -4; +} +my $cardN=substr $basedir,-1; + +if ($partition0) +{ + $image="factory"; +} + +runcmd("echo $image > $basedir/load_image_on_perst"); +runcmd("echo 0 > $basedir/perst_reloads_same_image"); +runcmd("cp /opt/ibm/capikv/afu/blacklist-cxlflash.conf /etc/modprobe.d"); +runcmd("echo 1 > $basedir/reset"); + +#wait for image_loaded +my $timer=30; +my $image_loaded; +my $cxlfile="/dev/cxl/afu$cardN.0m"; +$rc=-9; + +while ($timer--) +{ + print "."; + sleep 1; + if (-e $cxlfile) + { + $basedir=`ls -d /sys/devices/*/*/$target/cxl/card* 2>/dev/null`; + chomp $basedir; + if (-e "$basedir/image_loaded") + { + $image_loaded=`cat $basedir/image_loaded`; + chomp $image_loaded; + if ($image_loaded ne $image) + { + print "image requested was $image, but image_loaded is $image_loaded\n"; + $rc=-5; + last; + } + print "$target successfully loaded $image\n"; + sleep 1; + $rc=0; + last; + } + } +} + +if ($rc == -9) {print "timeout waiting for $target $cxlfile\n";} +else {runcmd("echo 1 > $basedir/perst_reloads_same_image");} +runcmd("rm /etc/modprobe.d/blacklist-cxlflash.conf"); + +exit 0; + diff --git a/src/build/install/resources/cflash_stick.pl b/src/build/install/resources/cflash_stick.pl new file mode 100755 index 00000000..d39a329b --- /dev/null +++ b/src/build/install/resources/cflash_stick.pl @@ -0,0 +1,193 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_stick.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +## +use strict; +use warnings; +use Fcntl; +use Fcntl ':seek'; +use Getopt::Long; + +#------------------------------------------------------------------------------- +# Variables +#------------------------------------------------------------------------------- +my $list="0"; +my $filename=""; +my $target=""; +my $id="2"; +my $prthelp=0; +my $verbose; +my $v=""; + +sub usage() +{ + print "\n"; + print "Usage: cflash_stick.pl [-l] [-f fw_file] [-t ] \n"; + print " -l : List capi sticks available for download \n"; + print " -f : Firmware file \n"; + print " -t or --target : Target device to flash \n"; + print " -h or --help : Help - Print this message \n"; + print "\n ex: cflash_stick.pl -f fw -t /dev/sgX \n\n"; + exit 0; +} + +#------------------------------------------------------------------------------- +# Parse Options +#------------------------------------------------------------------------------- +if ($#ARGV<0) {usage();} + +if (! GetOptions ("f=s" => \$filename, + "l" => \$list, + "t|target=s" => \$target, + "id=s" => \$id, + "v" => \$verbose, + "h|help!" => \$prthelp + )) +{ + usage(); +} +if ($ARGV[0]) +{ + print "\nUnknown Command Line Options:\n"; + foreach(@ARGV) + { + print " $_\n"; + } + print "\n"; + usage(); +} +if ($prthelp) {usage();} + +#check sudo permissions +if (`id -u` != 0) {print "Run with sudo permissions\n"; exit 0;} + +#------------------------------------------------------------------------------- +# Make stdout autoflush +#------------------------------------------------------------------------------- +select(STDOUT); +$| = 1; + +#------------------------------------------------------------------------------- +# list Devices +#------------------------------------------------------------------------------- +my @cards = `lspci |grep 0601`; +my $cmd=""; +my @devices; +my $stick; + +for my $cardN (@cards) +{ + my @Acard=split / /, $cardN; + my $card=$Acard[0]; + chomp $card; + my $afustr=`ls -d /sys/devices/*/*/$Acard[0]/cxl/card*/afu*.0/afu*m 2>/dev/null`; + my @afu=split /\//, $afustr; + my $afu=pop @afu; + if (length $afu) {chomp $afu;} + else {$afu="";} + $list && print "\nFound $Acard[5] $Acard[0] $afu\n"; + my $cardstr=`ls -d /sys/devices/*/*/$Acard[0]/cxl/card* 2>/dev/null`; + chomp $cardstr; + my $cardN=substr $cardstr,-1; + + for (my $i=0; $i<2; $i++) + { + my $Astick=`ls -d /sys/devices/*/*/$Acard[0]/*/*/host*/target*:$i:*/*:*:*:*/scsi_generic/* 2>/dev/null`; + if ($? != 0) {next;} + my @sdev=split /\//, $Astick; + my $dev=$sdev[12]; + chomp $dev; + $list && print " /dev/$dev\n"; + $list && system("/opt/ibm/capikv/afu/cxl_afu_dump /dev/cxl/afu$cardN.0m -v 2>/dev/null | grep NVMe$i"); + } +} + +#------------------------------------------------------------------------------- +# exit if (-l) list option specified +#------------------------------------------------------------------------------- +$list && exit 0; + +#ensure sg_write_buffer is available +my $ubuntu=0; +my $redhat=0; + +#determine distro +if ( `cat /etc/redhat-release 2>/dev/null` =~ "Red Hat" ) {$redhat=1;} +if ( `lsb_release -a 2>/dev/null` =~ "Ubuntu" ) {$ubuntu=1;} + +#ensure sg_write_buffer is available +if ( $redhat ) {$cmd="rpm -qa | grep -v udev|grep -v libs | grep sg3_utils >/dev/null "; } +if ( $ubuntu ) {$cmd="dpkg -l|grep -v udev|grep sg3-utils >/dev/null ";} + +system($cmd); +if ($? != 0) {print "ERROR: package sg3-utils is not installed\n"; exit -1;} + +#------------------------------------------------------------------------------- +# do the sg_write_buffer +#------------------------------------------------------------------------------- +if (! -e $filename) +{ + if (! ($filename =~ /images/)) + { + if (-e "/opt/ibm/capikv/afu/images/$filename") + { + $filename = "/opt/ibm/capikv/afu/images/$filename"; + } + } +} +if (! -e $filename) {die "ERROR: bad filename: $filename\n";} +if (! -e $target) {die "ERROR: bad target: $target\n";} +my $flen= -s $filename; +if ($flen==0) {die "ERROR: zero length file: $filename\n";} + +if ($verbose) {$v=" -v ";} + +# write fw +my $rc=0; +my $_64k=64*1024; +my $wlen=0; +my $off=0; +my $tot=0; +print "sg_write_buffer $flen bytes\n"; +do +{ + if ($tot+$_64k > $flen) {$wlen=$flen-$tot;} + else {$wlen=$_64k;} + $tot+=$wlen; + $cmd="sg_write_buffer -l $wlen -s $off -o $off -I $filename -m dmc_offs_defer -i $id $v $target"; + system($cmd); + $rc=$?; + ($rc != 0) && die " write failure, rc=$rc\n"; + $off+=$wlen; +} while ($tot<$flen); + +# activate firmware +$cmd="sg_write_buffer -m activate_mc -i $id $v $target"; +print "$cmd\n"; +system($cmd); +$rc=$?; +($rc != 0) && die " activate failure, rc=$rc\n"; +exit 0; + diff --git a/src/build/install/resources/cflash_temp.pl b/src/build/install/resources/cflash_temp.pl new file mode 100755 index 00000000..95994905 --- /dev/null +++ b/src/build/install/resources/cflash_temp.pl @@ -0,0 +1,87 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_temp.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +## +use strict; +use warnings; +use Fcntl; +use Fcntl ':seek'; +use Getopt::Long; + +my $cmd=""; +my $out; +my $ubuntu=0; +my $redhat=0; + +#determine distro +if ( `cat /etc/redhat-release 2>/dev/null` =~ "Red Hat" ) {$redhat=1;} +if ( `lsb_release -a 2>/dev/null` =~ "Ubuntu" ) {$ubuntu=1;} + +#check sudo permissions +(`id -u` == 0) || die "Run with sudo permissions"; + +#ensure sg_write_buffer is available +if ( $redhat ) {$cmd="rpm -qa | grep -v udev|grep -v libs | grep sg3_utils >/dev/null "; } +if ( $ubuntu ) {$cmd="dpkg -l|grep -v udev|grep sg3-utils >/dev/null ";} + +system($cmd); +if ($? != 0) {print "ERROR: package sg3-utils is not installed\n"; exit -1;} + +# Make stdout autoflush +select(STDOUT); +$| = 1; + +# print temps +my @cards = `lspci |grep 0601`; + +for my $adap (@cards) +{ + my @Acard=split / /, $adap; + my $card=$Acard[0]; + chomp $card; + my $afustr=`ls -d /sys/devices/*/*/$card/cxl/card*/afu*.0/afu*m 2>/dev/null`; + my @afu=split /\//, $afustr; + my $afu=pop @afu; + if (length $afu) {chomp $afu;} + else {$afu="";} + my $cardstr=`ls -d /sys/devices/*/*/$card/cxl/card* 2>/dev/null`; + chomp $cardstr; + my $cardN=substr $cardstr,-1; + print "\nFound $Acard[5] $card $afu\n"; + $out=`/opt/ibm/capikv/afu/flashgt_temp $card $cardN`; + print " $out"; + + for (my $i=0; $i<2; $i++) + { + my $Astick=`ls -d /sys/devices/*/*/$card/*/*/host*/target*:$i:*/*:*:*:*/scsi_generic/* 2>/dev/null`; + if ($? != 0) {next;} + my @sdev=split /\//, $Astick; + my $dev=$sdev[12]; + chomp $dev; + $out=`sg_logs -p 0xd /dev/$dev |grep "Current temp"`; + print " $dev: $out"; + } +} + diff --git a/src/build/install/resources/cflash_version b/src/build/install/resources/cflash_version new file mode 100755 index 00000000..4838c3fa --- /dev/null +++ b/src/build/install/resources/cflash_version @@ -0,0 +1,277 @@ +#!/bin/bash +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_version $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +# check permissions +if [[ $(id -u) != 0 ]]; then echo "use sudo"; exit 1; fi + +export PATH=$PATH:/opt/ibm/capikv/bin:/opt/ibm/capikv/afu + +#determine distro +if [[ $(lsb_release -a 2>/dev/null) =~ Ubuntu ]]; then ubuntu=1; fi +if [[ $(cat /etc/redhat-release 2>/dev/null) =~ Red ]]; then redhat=1; fi +if [[ $(cat /proc/cpuinfo|grep platform 2>/dev/null) =~ pSeries ]]; then VM=1; fi + +unset CF_VER +if [[ -e ver.txt ]] +then + CF_VER=$(cat ver.txt) +fi + +CORSA_VER=0512D1 +GT_VER=1114N1 +err_rc=0 + +if [[ $ubuntu ]] +then + OS_VER=16.04.1 + K_VER=4.4.0-36.55 + K_VER_GRUB=4.4.0-36-generic +elif [[ $redhat ]] +then + #TODO: Check the Red Hat Versions correct to targetted release + OS_VER=7.3; #VERSION_ID from /etc/os-release + K_VER=3.10.0 + K_VER_GRUB=3.10.0 +fi + +#TODO: version checks for VM systems + +if [[ $ubuntu ]] +then + if [[ -z $CF_VER ]] + then + CF_VER=$(dpkg -l |grep -v test|grep "cxlflash "|awk '{print $3}') + fi + + EXISTS_OS_VER=$(lsb_release -a 2>&1|grep -v LSB|grep Description|awk '{print $3}') + if [[ ! $EXISTS_OS_VER =~ $OS_VER ]] + then + err_rc=1 + echo "ERROR: OS: Installed($EXISTS_OS_VER) != Required($OS_VER)" + else + echo "INFO: OS: $EXISTS_OS_VER" + fi + + if [[ ! -z $1 ]]; then K_VER=$1; fi + + EXISTS_CF_TEST_VER=$(dpkg -l|grep cxlflash-test|awk '{print $3}') + if [[ -z $EXISTS_CF_TEST_VER && ! $EXISTS_CF_TEST_VER =~ $CF_VER ]] + then + err_rc=2 + echo "ERROR: cxlflash-test: Installed($EXISTS_CF_TEST_VER) != Required($CF_VER)" + fi + + EXISTS_CF_VER=$(dpkg -l|egrep -v "image|test"|grep cxlflash|awk '{print $3}') + if [[ -z $EXISTS_CF_VER ]] + then + err_rc=3 + echo "ERROR: cxlflash: missing" + elif [[ ! $EXISTS_CF_VER =~ $CF_VER ]] + then + err_rc=3 + echo "ERROR: cxlflash: Installed($EXISTS_CF_VER) != Required($CF_VER)" + else + echo "INFO: cxlflash: $EXISTS_CF_VER" + fi + + EXISTS_CF_IMAGE_VER=$(dpkg -l|grep cxlflashimage|awk '{print $3}') + if [[ -z $EXISTS_CF_IMAGE_VER ]] + then + err_rc=4 + echo "ERROR: cxlflashimage: missing" + elif [[ ! $EXISTS_CF_IMAGE_VER =~ $CF_VER ]] + then + err_rc=4 + echo "ERROR: cxlflashimage: Installed($EXISTS_CF_IMAGE_VER) Required($CF_VER)" + fi + + EXISTS_K_VER=$(dpkg -l|grep linux-image) + if [[ ! $EXISTS_K_VER =~ $K_VER ]] + then + err_rc=5 + echo "ERROR: Kernel: Booted($(uname -a|awk '{print $3}')) != Required($K_VER)" + else + echo "INFO: Kernel: $K_VER" + fi + + EXISTS_GRUB_VER=$(grep initrd /boot/grub/grub.cfg|head -1|awk -Finitrd.img- '{print $2}') + if [[ ! $EXISTS_GRUB_VER =~ $K_VER_GRUB ]] + then + err_rc=6 + echo "ERROR: Grub: grub.cfg($EXISTS_GRUB_VER) != Required($K_VER_GRUB)" + fi + + if [[ ! -z $EXISTS_CF_IMAGE_VER ]] + then + IFS=$'\n' + adaps=($(capi_flash.pl -l)) + unset IFS + for line in "${adaps[@]}" + do + if [[ $VM ]] + then + type=$(echo $line|awk '{print $5}') + adapt=$(echo $line|awk '{print $4}') + adap="$adapt " + fw=$(echo $line|awk -F\> '{print $2}' ) + else + type=$(echo $line|awk '{print $4}') + adap=$(echo $line|awk '{print $6}') + fw=$(echo $line|awk -F\> '{print $2}') + fi + + if [[ $line =~ 0601 ]] + then + if [[ ! $line =~ $GT_VER ]] + then + err_rc=7 + echo "ERROR: $adap $type Loaded($fw) != Required($GT_VER)" + else + echo "INFO: $adap $type $fw" + fi + elif [[ $line =~ 04cf ]] + then + if [[ ! $line =~ $CORSA_VER ]] + then + err_rc=7 + echo "ERROR: $adap $type Loaded($fw) != Required($CORSA_VER)" + else + echo "INFO: $adap $type $fw" + fi + fi + done + fi + +elif [[ $redhat ]] +then + if [[ -z $CF_VER ]] + then + CF_VER=$(rpm -aq | egrep -v "test|image"| grep cxlflash | tail -1 | sed 's,_,-,g'| awk -F'-' '{print $3"-"$4}') + fi + + EXISTS_OS_VER=$(grep "VERSION_ID" /etc/os-release | awk -F'"' '{print $2}') + if [[ ! $EXISTS_OS_VER =~ $OS_VER ]] + then + err_rc=1 + echo "ERROR: OS: Installed($EXISTS_OS_VER) != Required($OS_VER)" + else + echo "INFO: OS: $EXISTS_OS_VER" + fi + + if [[ ! -z $1 ]]; then K_VER=$1; fi + + EXISTS_CF_TEST_VER=$(rpm -qa | grep cxlflash-test| tail -1| sed 's,_,-,g'| awk -F'-' '{print $2"-"$3"-"$4}') + if [[ -z $EXISTS_CF_TEST_VER && ! $EXISTS_CF_TEST_VER =~ $CF_VER ]] + then + err_rc=2 + echo "ERROR: cxlflash-test: Installed($EXISTS_CF_TEST_VER) != Required($CF_VER)" + fi + + EXISTS_CF_VER=$(rpm -qa |egrep -v "image|test"|grep cxlflash|tail -1| sed 's,_,-,g'| awk -F'-' '{print $2"-"$3"-"$4}') + if [[ -z $EXISTS_CF_VER ]] + then + err_rc=3 + echo "ERROR: cxlflash: missing" + elif [[ ! $EXISTS_CF_VER =~ $CF_VER ]] + then + err_rc=3 + echo "ERROR: cxlflash: Installed($EXISTS_CF_VER) != Required($CF_VER)" + else + echo "INFO: cxlflash: $EXISTS_CF_VER" + fi + + EXISTS_CF_IMAGE_VER=$(rpm -qa |grep cxlflashimage|tail -1| sed 's,_,-,g'| awk -F'-' '{print $2"-"$3"-"$4}') + if [[ -z $EXISTS_CF_IMAGE_VER ]] + then + err_rc=4 + echo "ERROR: cxlflashimage: missing" + elif [[ ! $EXISTS_CF_IMAGE_VER =~ $CF_VER ]] + then + err_rc=4 + echo "ERROR: cxlflashimage: Installed($EXISTS_CF_IMAGE_VER) Required($CF_VER)" + fi + + EXISTS_K_VER=$(rpm -q kernel) + if [[ ! $EXISTS_K_VER =~ $K_VER ]] + then + err_rc=5 + echo "ERROR: Kernel: Booted($(uname -a|awk '{print $3}')) != Required($K_VER)" + else + echo "INFO: Kernel: $K_VER" + fi + + EXISTS_GRUB_VER=$(grep initrd /boot/grub2/grub.cfg | head -1 | awk -Finitramfs- '{print $2}') + if [[ ! $EXISTS_GRUB_VER =~ $K_VER_GRUB ]] + then + err_rc=6 + echo "ERROR: Grub: grub.cfg($EXISTS_GRUB_VER) != Required($K_VER_GRUB)" + fi + + if [[ ! -z $EXISTS_CF_IMAGE_VER ]] + then + IFS=$'\n' + adaps=($(capi_flash.pl -l)) + unset IFS + for line in "${adaps[@]}" + do + if [[ $VM ]] + then + type=$(echo $line|awk '{print $5}') + adapt=$(echo $line|awk '{print $4}') + adap="$adapt " + fw=$(echo $line|awk -F\> '{print $2}' ) + else + type=$(echo $line|awk '{print $4}') + adap=$(echo $line|awk '{print $6}') + fw=$(echo $line|awk -F\> '{print $2}') + fi + + if [[ $line =~ 0601 ]] + then + if [[ ! $line =~ $GT_VER ]] + then + err_rc=7 + echo "ERROR: $adap $type Loaded($fw) != Required($GT_VER)" + else + echo "INFO: $adap $type $fw" + fi + elif [[ $line =~ 04cf ]] + then + if [[ ! $line =~ $CORSA_VER ]] + then + err_rc=7 + echo "ERROR: $adap $type Loaded($fw) != Required($CORSA_VER)" + else + echo "INFO: $adap $type $fw" + fi + fi + done + fi +fi + +#TODO: add version checks for VM + +exit $err_rc diff --git a/src/build/install/resources/cflash_wear.pl b/src/build/install/resources/cflash_wear.pl new file mode 100755 index 00000000..b73690f7 --- /dev/null +++ b/src/build/install/resources/cflash_wear.pl @@ -0,0 +1,85 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cflash_wear.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +## +use strict; +use warnings; +use Fcntl; +use Fcntl ':seek'; +use Getopt::Long; + +my $cmd=""; +my $out; + +my $ubuntu=0; +my $redhat=0; + +#determine distro +if ( `cat /etc/redhat-release 2>/dev/null` =~ "Red Hat" ) {$redhat=1;} +if ( `lsb_release -a 2>/dev/null` =~ "Ubuntu" ) {$ubuntu=1;} + +#check sudo permissions +(`id -u` == 0) || die "Run with sudo permissions"; + +#ensure sg_write_buffer is available +if ( $redhat ) {$cmd="rpm -qa | grep -v udev | grep -v libs | grep sg3_utils >/dev/null "; } +if ( $ubuntu ) {$cmd="dpkg -l|grep -v udev|grep sg3-utils >/dev/null ";} + +system($cmd); +if ($? != 0) {print "ERROR: package sg3-utils is not installed\n"; exit -1;} + +# Make stdout autoflush +select(STDOUT); +$| = 1; + +my @cards = `lspci |grep 0601`; + +for my $adap (@cards) +{ + my @Acard=split / /, $adap; + my $card=$Acard[0]; + chomp $card; + my $afustr=`ls -d /sys/devices/*/*/$card/cxl/card*/afu*.0/afu*m 2>/dev/null`; + my @afu=split /\//, $afustr; + my $afu=pop @afu; + if (length $afu) {chomp $afu;} + else {$afu="";} + my $cardstr=`ls -d /sys/devices/*/*/$card/cxl/card* 2>/dev/null`; + chomp $cardstr; + my $cardN=substr $cardstr,-1; + print "\nFound $Acard[5] $card $afu\n"; + + for (my $i=0; $i<2; $i++) + { + my $Astick=`ls -d /sys/devices/*/*/$card/*/*/host*/target*:$i:*/*:*:*:*/scsi_generic/* 2>/dev/null`; + if ($? != 0) {next;} + my @sdev=split /\//, $Astick; + my $dev=$sdev[12]; + chomp $dev; + $out=`sg_logs -p 0x11 /dev/$dev |grep "endurance"`; + print " $dev: $out"; + } +} + diff --git a/src/build/install/resources/capikvutils.sh b/src/build/install/resources/cflashutils.sh similarity index 79% rename from src/build/install/resources/capikvutils.sh rename to src/build/install/resources/cflashutils.sh index ea7b1dc9..84a98f12 100755 --- a/src/build/install/resources/capikvutils.sh +++ b/src/build/install/resources/cflashutils.sh @@ -1,8 +1,8 @@ -#!/bin/bash -e +#!/bin/bash # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # -# $Source: src/build/install/resources/capikvutils.sh $ +# $Source: src/build/install/resources/cflashutils.sh $ # # IBM Data Engine for NoSQL - Power Systems Edition User Library Project # @@ -38,6 +38,7 @@ EIO=5; EACCES=13; EINVAL=22; +_VM=$(cat /proc/cpuinfo|grep platform|grep pSeries) die() { @@ -188,26 +189,59 @@ ctrlblockdevmap() fi } - #@desc print out a status table of the current LUN modes / mappings #@returns sets rc to 0 on success, OR sets return code to non-zero value on error printstatus() { - #list of all known SG devices - local to prevent this from causing side effects / problems in udev handler - local _SGDEVS=`ls /sys/module/cxlflash/drivers/pci:cxlflash/*:*:*.*/host*/target*:*:*/*:*:*:*/scsi_generic | grep sg` - local lunid=0; - local lunmode=0; - local blockdev=0; - local scsitopo=0; echo "CXL Flash Device Status" - printf "%10s: %10s %5s %9s %32s\n" "Device" "SCSI" "Block" "Mode" "LUN WWID"; - for dev in $_SGDEVS; do - getlunid lunid $dev; - getmode lunmode $dev; - getblockdev blockdev $dev || blockdev="n/a"; - getscsitopo scsitopo $dev; - printf "%10s: %10s, %5s, %9s, %32s\n" "$dev" "$scsitopo" "$blockdev" "$lunmode" "$lunid"; + if [[ -z $_VM ]] + then + _capis=$(lspci|egrep "0601|04cf" 2>/dev/null|awk '{print $1}') + else + _capis=$(ls -d /sys/bus/pci/drivers/cxlflash/*:* 2>/dev/null|awk -F/ '{print $7}') + fi + + for capi in $_capis + do + if [[ -z $_VM ]] + then + devid=$(lspci|grep $capi |awk -F" " '{print $6}') + else + grep_id=$(echo $capi|cut -c6-) + devid=$(lspci|grep $grep_id |awk -F" " '{print $7}') + fi + local devspec=$(cat /sys/bus/pci/devices/$capi/devspec) + local loccode=$(cat /proc/device-tree$devspec/ibm,loc-code 2>/dev/null) + + printf "\nFound $devid $capi $loccode\n" + + #list of all known SG devices - local to prevent this from causing side effects / problems in udev handler + if [[ -z $_VM ]] + then + _SGDEVS=$(ls -d /sys/devices/pci*/*/$capi/pci*/*/host*/target*/*/scsi_generic/sg* 2>/dev/null|awk -F/ '{print $13}') + else + _SGDEVS=$(ls -d /sys/devices/platform/*ibm,coherent-platform-facility/*/$capi/*/*/*/scsi_generic/sg* 2>/dev/null |awk -F/ '{print $12}') + fi + + local lunid=0; + local lunmode=0; + local blockdev=0; + local scsitopo=0; + if [[ -z $_SGDEVS ]] + then + continue + fi + printf "%10s: %10s %5s %9s %32s\n" "Device" "SCSI" "Block" "Mode" "LUN WWID"; + for dev in $_SGDEVS + do + getlunid lunid $dev; + getmode lunmode $dev; + getblockdev blockdev $dev || blockdev="n/a"; + getscsitopo scsitopo $dev; + printf "%10s: %10s, %5s, %9s, %32s\n" "$dev" "$scsitopo" "$blockdev" "$lunmode" "$lunid"; + done done + printf "\n" } #@desc set the mode for a given block device @@ -253,8 +287,19 @@ getluntablestate() #@returns rc to 0 on success, OR sets return code to non-zero value on error dotableupdate() { + #check sudo permissions + SUDO=$(id -u) + if [[ $SUDO != 0 ]]; then echo "run with sudo permissions to refresh"; return; fi + + #attempt to rescan for cxl scsi devices + local host; + for host in $(ls /sys/module/cxlflash/drivers/pci:cxlflash/*:*:*.*/ 2>/dev/null| grep host) + do + echo "- - -" > /sys/class/scsi_host/$host/scan + done + #list of all known SG devices - local to prevent this from causing side effects / problems in udev handler - local _SGDEVS=`ls /sys/module/cxlflash/drivers/pci:cxlflash/*:*:*.*/host*/target*:*:*/*:*:*:*/scsi_generic | grep sg` + local _SGDEVS=`ls /sys/module/cxlflash/drivers/pci:cxlflash/*:*:*.*/host*/target*:*:*/*:*:*:*/scsi_generic 2>/dev/null| grep sg` date >> $LOGFILE; if [[ ! -f $SIOTABLE ]]; then echo "Unable to access '$SIOTABLE'"; diff --git a/src/build/install/resources/cxl_flash_vm b/src/build/install/resources/cxl_flash_vm new file mode 100755 index 00000000..e69de29b diff --git a/src/build/install/resources/cxlffdc b/src/build/install/resources/cxlffdc index c2799349..055432a0 100755 --- a/src/build/install/resources/cxlffdc +++ b/src/build/install/resources/cxlffdc @@ -23,42 +23,211 @@ # permissions and limitations under the License. # # IBM_PROLOG_END_TAG -set -e -DESTDIR=/tmp/cxlffdc -TARBALL=cxlffdc.tgz + +DATADIR=/var/log/cflash/cxlffdc +PREVDIR=/var/log/cflash/prev +MISCDIR=/var/log/cflash/misc +TARBALL=/var/log/cflash/cxlffdc."$(date|awk '{print $4"_"$2"_"$3"_"$6}'|sed 's/:/_/'g)".tgz AFUDIR=/opt/ibm/capikv/afu -die() +################## +#Determine distro to differentiate dpkg | rpm +################## +if [[ $(lsb_release -a 2>/dev/null) =~ Ubuntu ]]; then ubuntu=1; fi +if [[ $(cat /etc/redhat-release 2>/dev/null) =~ Red ]]; then redhat=1; fi +if [[ $(cat /proc/cpuinfo|grep platform 2>/dev/null) =~ pSeries ]]; then VM=1; fi + + +################## +# wait 5 seconds for the pid to finish, then kill it if still running +################## +check_pid() { - echo "$1" 1>&2; - exit $2; + local cnt=0 + while [[ $cnt -lt 5 ]] + do + sleep 1 + if [[ ! $(ps -p $1) =~ $1 ]]; then break; fi + cnt=$((cnt+1)) + echo "cxlffdc: waiting for slow process $1" + done + if [[ $(ps -p $1) =~ $1 ]] + then + kill -9 $1 + fi } +################## +# print debug help +################## +if [[ $1 =~ help ]] +then + echo "export CFLSH_BLK_TRC_VERBOSITY=4" + echo "export CFLSH_BLK_TRC_SYSLOG=off" + echo "export KV_TRC_VERBOSITY=2" + echo "" + echo "echo \"module cxlflash +p\" > /sys/kernel/debug/dynamic_debug/control" + if [[ $redhat ]] + then + echo "uncomment these two lines in /etc/rsyslog.conf" + echo "kern.* /var/log/kern.log" + echo "\$ModLoad imklog" + echo "" + echo "Restart the service, run \"service rsyslog restart\"" + echo "run \"journalctl\" to extract debug traces" + fi + exit 0 +fi + +################## +# main +################## if [[ $EUID -ne 0 ]]; then - echo "This script must be run as root" 1>&2 + echo "This script must be run with sudo" 1>&2 exit 1 fi -mkdir $DESTDIR || die "ERROR: Delete previous FFDC at $DESTDIR and try again." 1 -pushd $DESTDIR +ret=$(pwd) +if [[ -e $DATADIR ]]; then rm -rf $DATADIR; fi +if [[ -e $PREVDIR ]]; then rm -rf $PREVDIR; fi +mkdir $DATADIR +cd $DATADIR + cardnums=`ls -d /sys/class/cxl/card* | awk -F"/sys/class/cxl/card" '{ print $2 }'` -for i in $cardnums; do - mkdir card$i; - $AFUDIR/psl_trace_dump card$i $i || echo "WARNING: Error occcurred dumping card$i"; #psl_trace_dump needs to be on our path... +for i in $cardnums +do + #psl_trace_dump needs to be on our path + mkdir card$i; + if [[ $VM ]] + then + printf "\n ERROR: psl_trace_dump on VM Not Supported! \n\n" + else + echo "Dumping PSL data for card$i" + $AFUDIR/psl_trace_dump card$i $i >/dev/null + fi +done + +AFUs=`ls /dev/cxl/afu*m` +for afu_m in $AFUs +do + $AFUDIR/cxl_afu_dump $afu_m > $(basename $afu_m).txt done +lspci -d 1014:0601 -vv | egrep "0601|LnkSta:" >> link_status.txt +echo "" >> link_status.txt + +for gt in $(lspci|grep 0601|awk '{print $1}') +do + card=$(ls -d /sys/devices/*/*/$gt/cxl/card* 2>/dev/null) + cardN=${card: -1} + echo "afu$cardN:" >> cxl_afu_status.txt + $AFUDIR/cxl_afu_status -d afu$cardN.0m >> cxl_afu_status.txt + printf "\n" >> cxl_afu_status.txt +done + +ps_fields="user,group,ruser,rgroup,pid,tid,ppid,pgid,uid,tgid,nlwp,sgi_p,nice,pri,\ +etime,pcpu,pmem,lim,rss,s,size,sz,vsz,trs,stat,f,stime,time,tty,comm,args" + +machine_info > $DATADIR/machine_info +ulimit -a > $DATADIR/ulimits +top -n 1 -b > $DATADIR/top +ps -Tdaeo $ps_fields > $DATADIR/ps +env > $DATADIR/env +uptime > $DATADIR/uptime +date >> $DATADIR/uptime +lspci -vvv > $DATADIR/lspci_vvv +if [[ $ubuntu ]] +then + lsb_release -a > $DATADIR/codelevels 2>/dev/null +elif [[ $redhat ]] +then + cat /etc/*-release | egrep "PRETTY_NAME|^NAME|VERSION_ID|^VERSION=" > $DATADIR/codelevels 2>/dev/null +fi +uname -a >> $DATADIR/codelevels +if [[ $ubuntu ]] +then + dpkg -l|egrep "cflash|cflashimage" >> $DATADIR/codelevels +elif [[ $redhat ]] +then + rpm -qa|egrep "cflash|cflashimage" >> $DATADIR/codelevels +fi + +/opt/ibm/capikv/afu/capi_flash.pl -l >> $DATADIR/codelevels +/opt/ibm/capikv/bin/cflash_version >> $DATADIR/codelevels + +/opt/ibm/capikv/afu/cflash_temp.pl >> $DATADIR/cflash_temp.txt& +check_pid $! +/opt/ibm/capikv/afu/cflash_wear.pl >> $DATADIR/cflash_wear.txt& +check_pid $! +/opt/ibm/capikv/bin/cxlfstatus > $DATADIR/cxlfstatus& +check_pid $! + +if [[ ! -z $(ls -A $MISCDIR) ]] +then + mkdir $DATADIR/misc + cp $MISCDIR/* $DATADIR/misc + truncate --size=0 $MISCDIR/* +fi + dmesg > dmesg.txt -cp /var/log/syslog $DESTDIR -cp /sys/firmware/opal/msglog $DESTDIR/opal_msglog +if [[ -e /var/log/syslog ]] +then + cp /var/log/syslog $DATADIR + truncate --size=0 /var/log/syslog +fi + +if [[ $redhat ]] +then + journalctl -n 50000 > /$DATADIR/syslog +fi + +#redhat version of syslog +if [[ -e /var/log/messages ]] +then + cp /var/log/messages $DATADIR + truncate --size=0 /var/log/messages +fi -mcontexts=`ls /dev/cxl/afu*m` -for mcontext in $mcontexts; do - $AFUDIR/cxl_afu_dump $mcontext > $(basename $mcontext).txt +if [[ -e /sys/firmware/opal/msglog ]] +then + cp /sys/firmware/opal/msglog $DATADIR/opal_msglog +fi + +for file in $(ls -1 /var/log/kern.log) +do + lines=$(wc -l $file | awk '{print $1}') + if [[ $lines -gt 0 ]] + then + tail -1000000 $file > $DATADIR/${file##*/} + truncate --size=0 $file + fi +done + +for file in $(ls -1 /tmp/*.cflash_block_trc) +do + lines=$(wc -l $file | awk '{print $1}') + if [[ $lines -gt 0 ]] + then + tail -1000000 $file > $DATADIR/${file##*/} + truncate --size=0 $file + fi done -popd -tar -cvzf $TARBALL $DESTDIR -rm -rf $DESTDIR +for file in $(ls -1 /tmp/*.arkdb.kv.log) +do + lines=$(wc -l $file | awk '{print $1}') + if [[ $lines -gt 0 ]] + then + tail -1000000 $file > $DATADIR/${file##*/} + truncate --size=0 $file + fi +done + +cd .. +tar -cvzf $TARBALL cxlffdc +mv $DATADIR $PREVDIR + echo "INFO: FFDC Collected below." ls -l $TARBALL +cd $ret diff --git a/src/build/install/resources/cxlffdc.cleardebug b/src/build/install/resources/cxlffdc.cleardebug new file mode 100644 index 00000000..33692506 --- /dev/null +++ b/src/build/install/resources/cxlffdc.cleardebug @@ -0,0 +1,29 @@ +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cxlffdc.cleardebug $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +export CFLSH_BLK_TRC_VERBOSITY=1 +unset CFLSH_BLK_TRC_SYSLOG +export KV_TRC_VERBOSITY=1 +echo "module cxlflash -p" > /sys/kernel/debug/dynamic_debug/control diff --git a/src/build/install/resources/cxlffdc.debug b/src/build/install/resources/cxlffdc.debug new file mode 100644 index 00000000..439dfb6c --- /dev/null +++ b/src/build/install/resources/cxlffdc.debug @@ -0,0 +1,29 @@ +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/cxlffdc.debug $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +export CFLSH_BLK_TRC_VERBOSITY=4 +export CFLSH_BLK_TRC_SYSLOG=off +export KV_TRC_VERBOSITY=2 +echo "module cxlflash +p" > /sys/kernel/debug/dynamic_debug/control diff --git a/src/build/install/resources/cxlfrefreshluns b/src/build/install/resources/cxlfrefreshluns index 800239d8..4727ef98 100755 --- a/src/build/install/resources/cxlfrefreshluns +++ b/src/build/install/resources/cxlfrefreshluns @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # @@ -25,7 +25,7 @@ # IBM_PROLOG_END_TAG CAPIKV=/opt/ibm/capikv -source $CAPIKV/bin/capikvutils.sh +source $CAPIKV/bin/cflashutils.sh showhelp() diff --git a/src/build/install/resources/cxlfsetlunmode b/src/build/install/resources/cxlfsetlunmode index 65ec6e85..2ad4631f 100755 --- a/src/build/install/resources/cxlfsetlunmode +++ b/src/build/install/resources/cxlfsetlunmode @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # @@ -25,7 +25,7 @@ # IBM_PROLOG_END_TAG CAPIKV=/opt/ibm/capikv -source $CAPIKV/bin/capikvutils.sh +source $CAPIKV/bin/cflashutils.sh showhelp() diff --git a/src/build/install/resources/cxlfstatus b/src/build/install/resources/cxlfstatus index 6bcea04f..9f881200 100755 --- a/src/build/install/resources/cxlfstatus +++ b/src/build/install/resources/cxlfstatus @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # @@ -25,7 +25,7 @@ # IBM_PROLOG_END_TAG CAPIKV=/opt/ibm/capikv -source $CAPIKV/bin/capikvutils.sh +source $CAPIKV/bin/cflashutils.sh showhelp() diff --git a/src/build/install/resources/flash_all_adapters b/src/build/install/resources/flash_all_adapters index 35fb4cd5..97645fad 100755 --- a/src/build/install/resources/flash_all_adapters +++ b/src/build/install/resources/flash_all_adapters @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # @@ -23,41 +23,29 @@ # permissions and limitations under the License. # # IBM_PROLOG_END_TAG -#Constants + KERNEL_MOD="cxlflash" -CAPI_FLASH=capi_flash.pl -CAPI_IMAGE=1410f0041410f004.00000001151029D1 +CAPI_FLASH_PL=capi_flash.pl +PERST_PL=cflash_perst.pl +CORSA_IMAGE=1410f0041410f004.00000001160512D1 +GT_F_IMAGE=1410000614100006.00000001160903N0 +GT_IMAGE=1410000614100006.00000001161114N1 IMAGES_DIR=images #relative to the cwd of the script FLASH_ROOT=/opt/ibm/capikv/afu - MINCAPMASK="0003" CAPABILITYKW="V3" -#Globals -factory="" - -#allow us to die with a specific error code -die() -{ - echo "AFUFLASHERR$2: $1" 1>&2; - exit $2; -} - -if [ "$1" == "--factory" ]; then - factory="-p0" -fi - +e_rc=0 +do_perst=0 +load_factory=0 +opt="" checkidle() { - echo "INFO: Checking for CAPI Master Context..." - master_offline=true - lsmod | grep $KERNEL_MOD && master_offline=false #if the cxlflash module is found, we set the master_offline flag to false -#otherwise, shows the module info - - if $master_offline ; then - echo "INFO: CXL Flash Module is not running. Device is assumed to be idle."; - else - die "CXL Flash is running (see above). Cannot flash while the device is active. Run \"rmmod cxlflash\" to unload it." 1; + mods=$(lsmod) + if [[ $mods == *"cxlflash"* ]] + then + printf " ERROR, Cannot flash with cxlflash loaded: \"rmmod cxlflash\" and retry\n\n" + exit 1 fi } @@ -65,66 +53,231 @@ readvpd() { local pciid=$1 local keyword=$2 - local value=`lspci -s $pciid -vv | sed -n -e '/Vital Product Data/,/End/ p' | grep $keyword | awk -F": " '{print $2}'` - if [[ -z "$value" ]]; then - die "Unable to read PCI device $pciid, VPD keyword $keyword. This afu image is not compatible with this adapter." 2; + vpd_value=$(lspci -s $pciid -vv | sed -n -e '/Vital Product Data/,/End/ p' | grep $keyword | awk -F": " '{print $2}') + if [[ -z "$vpd_value" ]] + then + printf " ERROR, PCI device %s, VPD keyword: %s not found\n\n" $pciid $keyword + return 2 fi - echo $value + return 0 } findloc() { - devspecpath="/sys/bus/pci/devices/$1/devspec" + local devspecpath="/sys/bus/pci/devices/$1/devspec" if [[ ! -f "$devspecpath" ]]; then - die "Unable to find devspec for device $1. Could not find '$devspecpath'" 3; + echo " ERROR, Unable to find devspec for device %s. Could not find '%s'\n\n" $1 $devspecpath + return 3 fi - local devspec=`cat $devspecpath` + local devspec=$(cat $devspecpath) local loccodepath="/proc/device-tree$devspec/ibm,loc-code" if [[ ! -f "$loccodepath" ]]; then - die "Unable to find device tree special file path for PCI device $1: $loccodepath" 4; + printf " ERROR, Unable to find device tree special file path for PCI device %s: %s\n\n" $1 $loccodepath + return 4 fi - local loccode=`cat $loccodepath`; - if [[ -z "$loccode" ]]; then - die "Location code is invalid for PCI device $1" 5; + loccode=$(cat $loccodepath) + if [[ -z "$loccode" ]] + then + printf " ERROR, Location code is invalid for PCI device %s\n\n" $1 + return 5 fi - echo $loccode; - } + return 0 +} checkcompat() { - echo "INFO: Checking card / image compatibility for image $CAPI_IMAGE" - #enumerate all adapters, get the PCI id (e.g. "0000:01:00.0" from below) - #parse something like "Found CAPI Adapter : /sys/bus/pci/devices/0000:01:00.0" - for adapter in `$FLASH_ROOT/$CAPI_FLASH -l | awk -F/ '{print $NF}'`; + line=$(lspci -s $1| egrep "04cf|0601") + if [ $? -ne 0 ] + then + printf " ERROR, unknown capi adapter\n\n" + rc=7 + fi + + if [[ -z $2 ]]; then return 0; fi + + readvpd $1 $2; rc=$? + if [ $rc -ne 0 ]; then return $rc; fi + resultmask=$(($MINCAPMASK & $vpd_value)) + if [[ "$resultmask" -ne "$MINCAPMASK" ]] + then + printf " ERROR, The available firmware is incompatible with this adapter. %s=0x%s\n\n" $2 $vpd_value + rc=6 + fi + return $rc +} + +flashadapters() +{ + $FLASH_ROOT/$CAPI_FLASH_PL -l | while read adapStr do - echo "INFO: Checking adapter / firmware compatibility for $(findloc $adapter)"; - adaptermask=$(readvpd $adapter $CAPABILITYKW); - local resultmask=$(($MINCAPMASK & $adaptermask)) - if [[ "$resultmask" -ne "$MINCAPMASK" ]]; then - die "Adapter $(findloc $adapter) firmware is incompatible with this adapter. Please use a different firmware image. Minimum capability mask for this firmware is 0x$MINCAPMASK and adapter VPD capability is 0x$adaptermask." 6; + adap=$(echo $adapStr|awk '{print $6}') + afu=$(echo $adapStr|awk '{print $5}') + #verify location code + findloc $adap; rc=$? + if [[ $rc -ne 0 ]]; then e_rc=$rc; continue; fi + + #get device ID + devid=$(lspci -s $adap| awk 'BEGIN{FS="IBM Device "}; {print $NF}'|awk -F\( '{print $1}') + + if [[ $devid =~ 04cf && $load_factory -eq 1 ]]; then continue; fi + + echo "INFO: Processing: $adap $afu $loccode" + + #check for a healthy adap + if [ ! -f /sys/devices/*/*/$adap/cxl/card*/image_loaded ] + then + echo " ERROR, Skipping, card is in a bad state" + echo + continue + fi + + #query FW level + afuver=$(/opt/ibm/capikv/afu/cxl_afu_dump /dev/cxl/${afu}.0m -v 2>/dev/null| grep "AFU Version"|awk -F" " '{print $NF}') + if [[ -z $afuver ]] + then + afuver=__UNKNOWN_FW_LEVEL__ fi + + if [ $devid = "04cf" ] #CORSA + then + #verify VPD + checkcompat $adap $CAPABILITYKW; rc=$? + if [[ $rc -ne 0 ]] + then + e_rc=$rc; + continue; + fi + + if [[ $CORSA_IMAGE =~ $afuver ]] + then + printf " Firmware is up to date (%s)\n\n" $afuver + continue + fi + printf " Replacing %s with %s\n\n" $afuver $CORSA_IMAGE + $FLASH_ROOT/$CAPI_FLASH_PL -t $adap -f $FLASH_ROOT/$IMAGES_DIR/$CORSA_IMAGE + rc=$? + if [[ $rc -ne 0 ]] + then + e_rc=$rc + echo " ERROR: Failed flashing $adap" + else + if [[ $do_perst -eq 1 ]] + then + $FLASH_ROOT/$PERST_PL -t $adap + rc=$? + if [[ $rc -ne 0 ]] + then + e_rc=$rc + echo " ERROR: Failed PERSTing $adap" + fi + fi + fi + + elif [ $devid = "0601" ] #FLASH_GT + then + #verify VPD + checkcompat $adap; rc=$? + if [[ $rc -ne 0 ]]; then e_rc=$rc; continue; fi + + if [[ $GT_IMAGE =~ $afuver ]] + then + printf " Firmware is up to date (%s)\n\n" $afuver + continue + fi + if [[ $load_factory -eq 1 ]] + then + printf " Flashing %s\n\n" $GT_IMAGE + else + if [[ $GT_F_IMAGE =~ $afuver ]] + then + printf " Flashing %s\n\n" $GT_IMAGE + else + printf " Replacing %s with %s\n\n" $afuver $GT_IMAGE + fi + fi + $FLASH_ROOT/$CAPI_FLASH_PL $opt -t $adap -f $FLASH_ROOT/$IMAGES_DIR/$GT_IMAGE + rc=$? + if [[ $rc -ne 0 ]] + then + e_rc=$rc + echo " ERROR: Failed flashing $adap" + else + if [[ $do_perst -eq 1 ]] + then + $FLASH_ROOT/$PERST_PL -t $adap + rc=$? + if [ $rc -ne 0 ] + then + e_rc=$rc + echo " ERROR: Failed PERSTing $adap" + fi + fi + fi + + else + echo " ERROR, Download is not supported for the Device ID: $devid" + e_rc=8 + fi + echo "" done - echo "INFO: All present IBM DataEngine for NoSQL adapters are capable of loading this image." - } - -flashadapters() +flashadapters_VM() { - echo "INFO: Now flashing all CAPI devices..." + $FLASH_ROOT/$CAPI_FLASH_PL -l | while read adap + do + if [[ $adap =~ 04cf ]]; then fw=$CORSA_IMAGE; + else fw=$GT_IMAGE; + fi + afuN=$(echo $adap|cut -c19-24)m + echo $adap + afuver=$(/opt/ibm/capikv/afu/cxl_afu_dump /dev/cxl/$afuN -v 2>/dev/null| grep "AFU Version"|awk -F" " '{print $NF}') - #enumerate all adapters, get the path (item #5 from the awk command) - #parse something like "Found CAPI Adapter : /sys/bus/pci/devices/0000:01:00.0" - for adapter in `$FLASH_ROOT/$CAPI_FLASH -l | awk '{print $5}'`; - do - echo "INFO: Flashing adapter: $adapter"; - $FLASH_ROOT/$CAPI_FLASH $factory -t $adapter -f $FLASH_ROOT/$IMAGES_DIR/$CAPI_IMAGE || die "Unable to flash. Halting script." 2; - done + echo "INFO: Processing: $afuN" + if [[ -z $afuver ]]; then afuver=__UNKNOWN_FW_LEVEL__; fi + + if [[ $fw =~ $afuver ]] + then + printf " Firmware is up to date (%s)\n\n" $afuver + continue + fi + printf " Replacing %s with %s\n\n" $afuver $fw + + $FLASH_ROOT/$CAPI_FLASH_PL -t $afuN -f $fw + if [ $rc -ne 0 ] + then + e_rc=$rc + echo " ERROR: Failed flashing $adap" + fi + done } -#Main Code -checkidle; -checkcompat; -flashadapters; +######### +# main +######### +grep pSeries /proc/cpuinfo >/dev/null 2>&1 && _VM=1 +checkidle +echo "module cxlflash +p" > /sys/kernel/debug/dynamic_debug/control + +if [[ $1 =~ reload || $2 =~ reload ]]; then do_perst=1; fi +if [[ $1 =~ factory || $2 =~ factory ]] +then + opt=" -p0 " + load_factory=1 + GT_IMAGE=$GT_F_IMAGE +fi + +if [[ $_VM -eq 1 ]]; then flashadapters_VM +else flashadapters +fi + +echo "" +if [[ $e_rc -eq 0 ]] +then + echo "INFO: Adapter update success" +else + echo "INFO: review msgs for errors" +fi -echo "INFO: Adapter update complete. To restart the adapters on the new image, please run ${FLASH_ROOT}/reload_all_adapters" +echo "module cxlflash -p" > /sys/kernel/debug/dynamic_debug/control +exit $e_rc diff --git a/src/build/install/resources/flashgt_nvme_override b/src/build/install/resources/flashgt_nvme_override new file mode 100755 index 00000000..e923eac2 Binary files /dev/null and b/src/build/install/resources/flashgt_nvme_override differ diff --git a/src/build/install/resources/license/LICENSE b/src/build/install/resources/license/LICENSE deleted file mode 100644 index 68c771a0..00000000 --- a/src/build/install/resources/license/LICENSE +++ /dev/null @@ -1,176 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - diff --git a/src/build/install/resources/machine_info b/src/build/install/resources/machine_info new file mode 100755 index 00000000..3462eeb9 --- /dev/null +++ b/src/build/install/resources/machine_info @@ -0,0 +1,38 @@ +#!/bin/bash +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/machine_info $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +echo "cores enabled = $(ls /proc/device-tree/cpus/ | grep -c PowerPC)" +echo "SMT mode: $(ppc64_cpu --smt)" +echo "CPUs online (based on SMT) = $(grep -c proc /proc/cpuinfo)" +echo "CPU speed:" +ppc64_cpu --frequency +echo "total system memory and free memory available:" +grep Mem /proc/meminfo +echo "NUMA node balance:" +cat /sys/devices/system/node/node*/meminfo | grep MemTotal +cat /sys/devices/system/node/node*/meminfo | grep MemFree +echo "HugePage usage:" +grep Huge /proc/meminfo diff --git a/src/build/install/resources/postafuinstall b/src/build/install/resources/postafuinstall index 7f9bc17b..f96eefd5 100644 --- a/src/build/install/resources/postafuinstall +++ b/src/build/install/resources/postafuinstall @@ -28,6 +28,3 @@ USERNAME=cxl GROUPNAME=$USERNAME FLASH_ROOT=/opt/ibm/capikv/afu DATA_DIR=$CAPIKV_ROOT/data -INIFILE=$DATA_DIR/capikv.ini - -echo "INFO: Please remove the \"cxlflash\" module and run \"$FLASH_ROOT/flash_all_adapters\" to perform the CAPI accelerator microcode update." diff --git a/src/build/install/resources/postinstall b/src/build/install/resources/postinstall index c20e4a98..9e57a262 100755 --- a/src/build/install/resources/postinstall +++ b/src/build/install/resources/postinstall @@ -23,20 +23,22 @@ # permissions and limitations under the License. # # IBM_PROLOG_END_TAG -set -e #halt on error + + USERNAME=cxl GROUPNAME=$USERNAME CAPIKV_ROOT=/opt/ibm/capikv +BIN_DIR=$CAPIKV_ROOT/bin ETC_DIR=$CAPIKV_ROOT/etc SIO_FILE=$ETC_DIR/sioluntable.ini + #Creating empty SIO lun table and etc dir echo "INFO: Ensuring SIO LUN Table exists..." echo "INFO: Creating system cxl user / group..." -adduser --system --group $USERNAME -if ! egrep -i "^$GROUPNAME" /etc/group >/dev/null; then - echo "INFO: Creating cxl group..." - groupadd -r $GROUPNAME -fi +useradd --system --user-group $USERNAME 2>/dev/null + +set -e #halt on error + echo "INFO: setting permissions on cxl and cxlflash device nodes." udevadm trigger -s cxl udevadm trigger -s cxlflash @@ -47,6 +49,8 @@ touch $SIO_FILE chown $USERNAME:$GROUPNAME $SIO_FILE chmod 660 $SIO_FILE +if [[ -e $BIN_DIR/cflash_logs_cron ]]; then cp $BIN_DIR/cflash_logs_cron /etc/cron.hourly; fi + echo "INFO: enabling cxlfd service for LUN Management" systemctl enable cxlfd || echo "WARNING: Unable to enable the cxlfd service via systemctl. Please enable the cxlfd daemon for LUN management." systemctl start cxlfd || echo "WARNING: Unable to start the cxlfd service via systemctl. Please enable the cxlfd daemon for LUN management." diff --git a/src/build/install/resources/postremove b/src/build/install/resources/postremove new file mode 100755 index 00000000..6a81a55c --- /dev/null +++ b/src/build/install/resources/postremove @@ -0,0 +1,29 @@ +#!/bin/bash +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/install/resources/postremove $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +CRON_FILE=/etc/cron.hourly/cflash_logs_cron + +if [[ -e $CRON_FILE ]]; then rm -f $CRON_FILE; fi diff --git a/src/build/install/resources/readme.resources.txt b/src/build/install/resources/readme.resources.txt index fe3e32f0..dadd7be7 100644 --- a/src/build/install/resources/readme.resources.txt +++ b/src/build/install/resources/readme.resources.txt @@ -38,8 +38,8 @@ psl_trace_dump Purpose: collected PSL FFDC / traces on a failed system -capikvutils.sh -Purpose: Utility code for cxlflash and ibmcapikv tooling +cflashutils.sh +Purpose: Utility code for cflash tooling cxlfrefreshluns cxlfsetlunmode cxlfstatus Purpose: suite of utilities for manipulating the cxlflash driver diff --git a/src/build/install/resources/reload_all_adapters b/src/build/install/resources/reload_all_adapters index 3b8cae1b..c7fe7070 100755 --- a/src/build/install/resources/reload_all_adapters +++ b/src/build/install/resources/reload_all_adapters @@ -23,138 +23,13 @@ # permissions and limitations under the License. # # IBM_PROLOG_END_TAG -#Constants -KERNEL_MOD="cxlflash" -FLASH_ROOT=/opt/ibm/capikv/afu -RELOADDELAY=0.5 -MAXRELOADCOUNT=120 #120 .5 second intervals -MODPROBEDIR=/etc/modprobe.d -BLACKLISTFILE=blacklist-${KERNEL_MOD}.conf -#allow us to die with a specific error code -die() -{ - echo "AFUFLASHERR$2: $1" 1>&2; - exit $2; -} +grep pSeries /proc/cpuinfo >/dev/null 2>&1 && _VM=1 +if [[ $_VM -eq 1 ]]; then echo "Not Supported on VM"; exit 0; fi -checkidle() -{ - echo "INFO: Checking for CAPI Master Context..." - master_offline=true - lsmod | grep $KERNEL_MOD && master_offline=false #if the cxlflash module is found, we set the master_offline flag to false -#otherwise, shows the module info - - if $master_offline ; then - echo "INFO: CXL Flash Module is not running. Device is assumed to be idle."; - else - die "CXL Flash is running (see above). Cannot flash while the device is active. Run \"rmmod cxlflash\" to unload it." 1; - fi -} - -disablemodule() -{ - if [ ! -e $MODPROBEDIR/$BLACKLISTFILE ]; then - echo "INFO: Temporarily blacklisting ${KERNEL_MOD}" - cp $FLASH_ROOT/$BLACKLISTFILE $MODPROBEDIR/$BLACKLISTFILE - fi -} - -enablemodule() -{ - if [ -e $MODPROBEDIR/$BLACKLISTFILE ]; then - echo "INFO: Removing ${KERNEL_MOD} from blacklist" - rm $MODPROBEDIR/$BLACKLISTFILE - fi -} - - -findloc() -{ - devspecpath="/sys/bus/pci/devices/$1/devspec" - if [[ ! -f "$devspecpath" ]]; then - die "Unable to find devspec for device $1. Could not find '$devspecpath'" 3; - fi - local devspec=`cat $devspecpath` - local loccodepath="/proc/device-tree$devspec/ibm,loc-code" - if [[ ! -f "$loccodepath" ]]; then - die "Unable to find device tree special file path for PCI device $1: $loccodepath" 4; - fi - local loccode=`cat $loccodepath`; - if [[ -z "$loccode" ]]; then - die "Location code is invalid for PCI device $1" 5; - fi - echo $loccode; - } - -writeTo_perst_reloads_same_image() -{ - local cxlsysfs="/sys/class/cxl" - local val=$1 - - for adapter in `ls $cxlsysfs | grep card`; do - echo "Writing $val to perst_reloads_same_image for $adapter" - echo "$val" > $cxlsysfs/$adapter/perst_reloads_same_image; - done -} - -reloadadapters() -{ - local rebootrequired="false" - local cxlsysfs="/sys/class/cxl" - - # Disable perst_reloads_same_image for all cards - writeTo_perst_reloads_same_image "0"; - - #Disable the cxlflash module temporarily - disablemodule; - - #enumerate all cxl cards, and issue a PERST to each one - #should result in card0 card1 card2.. etc. - for adapter in `ls $cxlsysfs | grep card`; do - if [[ (-f $cxlsysfs/$adapter/load_image_on_perst) && (-f $cxlsysfs/$adapter/reset) ]]; then - echo "Attempting to reload $adapter" - echo "user" > $cxlsysfs/$adapter/load_image_on_perst; - echo "1" > $cxlsysfs/$adapter/reset; - local pollcount=0; - while [ ! -f $cxlsysfs/$adapter/image_loaded ] - do - if [[ "$pollcount" -eq "$MAXRELOADCOUNT" ]]; then - #re-enable the cxlflash module - enablemodule; - # Re-enable perst_reloads_same_image for all cards - writeTo_perst_reloads_same_image "1"; - die "Unable to reload $adapter" 100; - fi - pollcount=$((pollcount+1)) - echo -n "."; - sleep $RELOADDELAY; - done - echo "done" - - local imgselected=`cat $cxlsysfs/$adapter/image_loaded`; - if [[ "$imgselected" -ne "user" ]]; then - echo "Card $adapter did not boot on the user image. Image selected was $imgselected."; - rebootrequired="true"; - fi - else - rebootrequired="true"; - fi - done - - #re-enable the cxlflash module - enablemodule; - # Re-enable perst_reloads_same_image for all cards - writeTo_perst_reloads_same_image "1"; - - if [[ "$rebootrequired" == "true" ]]; then - echo "WARNING: This kernel does not support a warm reload of the CAPI adapter firmware. A cold restart of the system is required to reinitialize the adapter. Please power off, then power on (do not \"reboot\")." - else - echo "INFO: All present IBM DataEngine for NoSQL adapters have reloaded on this image." - fi - -} - -#Main Code -checkidle; -reloadadapters; +for dev in $(lspci |egrep "04cf|0601"|awk '{print $1}') +do + echo "reloading $dev" + /opt/ibm/capikv/afu/cflash_perst.pl -t $dev + if [[ $? -eq -2 || $? -eq -3 ]]; then exit -1; fi +done diff --git a/src/build/packaging/makefile b/src/build/packaging/makefile index 4829c35a..03007673 100644 --- a/src/build/packaging/makefile +++ b/src/build/packaging/makefile @@ -26,60 +26,90 @@ UNAME=$(shell uname) ROOTPATH = ../../.. ifeq ($(UNAME),AIX) -all: aixcapikv aixcapikv-test aixafuimage +all: aixcxlflash aixcxlflash-test aixcxlflashimage else -all: ibmcapikv ibmcapikv-test afuimage + +#only add tarpkgs if we are in the DISABLED dir path +DISABLED=$(shell pwd|grep DISABLED) +ifeq "$(DISABLED)" "" +all: cxlflash cxlflash-test cxlflashimage +else +all: cxlflash cxlflash-test cxlflashimage tarpkgs +endif + endif -VERSIONMAJOR=2 -VERSIONMINOR=0 +VERSIONMAJOR=4 +VERSIONMINOR=1 +SHORT_VERSION=${VERSIONMAJOR}.${VERSIONMINOR} VERSION=${VERSIONMAJOR}.${VERSIONMINOR}-${GITREVISION} -ibmcapikv: +cxlflash: mkdir -p ${PKGDIR} cd ${PKGDIR}; \ - fpm -f -s dir -t rpm -n $@ -v ${VERSION} \ - --depends 'advance-toolchain-at7.1-runtime' \ - --depends 'libudev1' \ + fpm -f -s dir -t rpm -n $@ -v ${VERSION} --iteration=1 \ + --depends 'advance-toolchain-at10.0-runtime' \ + --depends 'systemd-devel' \ -C ./install_root \ - --after-install ${SURELOCKROOT}/src/build/install/resources/postinstall .; \ + --after-install ${SURELOCKROOT}/src/build/install/resources/postinstall \ + --after-remove ${SURELOCKROOT}/src/build/install/resources/postremove \ + --conflicts ibmcapikv --replaces ibmcapikv .; \ fpm -f -s dir -t deb -n $@ -v ${VERSION} \ - --depends 'advance-toolchain-at7.1-runtime' \ + --depends 'advance-toolchain-at10.0-runtime' \ --depends 'libudev1' \ -C ./install_root \ - --after-install ${SURELOCKROOT}/src/build/install/resources/postinstall .; \ + --after-install ${SURELOCKROOT}/src/build/install/resources/postinstall \ + --after-remove ${SURELOCKROOT}/src/build/install/resources/postremove \ + --conflicts ibmcapikv --replaces ibmcapikv .; \ tar -cvzf $@-${GITREVISION}.tar.gz -C ./install_root . -ibmcapikv-test: +cxlflash-test: mkdir -p ${PKGDIR} cd ${PKGDIR}; \ - fpm -f -s dir -t rpm -n $@ -v ${VERSION} -C ./test_root \ - --depends 'ibmcapikv = ${VERSION}' \ - --depends 'libudev-dev' .; \ + fpm -f -s dir -t rpm -n $@ -v ${VERSION} -C ./test_root --iteration=1 \ + --depends 'cxlflash = ${VERSION}-1' \ + --depends 'systemd-devel' \ + --conflicts ibmcapikv-test --replaces ibmcapikv-test .; \ fpm -f -s dir -t deb -n $@ -v ${VERSION} -C ./test_root \ - --depends 'ibmcapikv = ${VERSION}' \ - --depends 'libudev-dev' .; \ + --depends 'cxlflash = ${VERSION}' \ + --depends 'libudev-dev' \ + --conflicts ibmcapikv-test --replaces ibmcapikv-test .; \ tar -cvzf $@-${GITREVISION}.tar.gz -C ./test_root . -afuimage: +cxlflashimage: mkdir -p ${PKGDIR} cd ${PKGDIR}; \ + fpm -f -s dir -t rpm -a all -n $@ -v ${VERSION} -C ./afu_root --iteration=1 \ + --after-install ${SURELOCKROOT}/src/build/install/resources/postafuinstall \ + --conflicts afuimage --replaces afuimage .; \ fpm -f -s dir -t deb -a all -n $@ -v ${VERSION} -C ./afu_root \ - --after-install ${SURELOCKROOT}/src/build/install/resources/postafuinstall . + --after-install ${SURELOCKROOT}/src/build/install/resources/postafuinstall \ + --conflicts afuimage --replaces afuimage .; \ + +tarpkgs: + cd ${PKGDIR}; \ + echo ${VERSION} > ver.txt; \ + tar -cvzf pkg.${VERSION}.rpm.tar.gz *${SHORT_VERSION}*.rpm cflash_configure ver.txt; \ + tar -cvzf pkg.${VERSION}.deb.tar.gz *${SHORT_VERSION}*.deb cflash_configure ver.txt; \ + cp pkg.${VERSION}*deb.tar.gz /gsa/ausgsa/projects/s/surelock/surelock-sw/releases/GA4/latest_jenkins/pkg.deb.tar.gz; \ + cp pkg.${VERSION}*rpm.tar.gz /gsa/ausgsa/projects/s/surelock/surelock-sw/releases/GA4/latest_jenkins/pkg.rpm.tar.gz -aixcapikv: +aixcxlflash: mkdir -p ${PKGDIR} cd ${PKGDIR}; \ - tar -cvf ibmcapikv_${VERSION}.ppc64be.tar -C ./install_root . + tar -cvf cxlflash_${VERSION}.ppc64be.tar -C ./install_root .; \ + gzip -f cxlflash_${VERSION}.ppc64be.tar -aixcapikv-test: +aixcxlflash-test: mkdir -p ${PKGDIR} cd ${PKGDIR}; \ - tar -cvf ibmcapikv-test_${VERSION}.ppc64be.tar -C ./test_root . + tar -cvf cxlflash-test_${VERSION}.ppc64be.tar -C ./test_root .; \ + gzip -f cxlflash-test_${VERSION}.ppc64be.tar -aixafuimage: +aixcxlflashimage: mkdir -p ${PKGDIR} cd ${PKGDIR}; \ - tar -cvf afuimage_${VERSION}.ppc64be.tar -C ./afu_root . + tar -cvf cxlflashimage_${VERSION}.ppc64be.tar -C ./afu_root .; \ + gzip -f cxlflashimage_${VERSION}.ppc64be.tar include ${ROOTPATH}/config.mk diff --git a/src/build/tools/build_x86 b/src/build/tools/build_x86 new file mode 100755 index 00000000..d182a3a1 --- /dev/null +++ b/src/build/tools/build_x86 @@ -0,0 +1,147 @@ +#!/bin/bash +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/tools/build_x86 $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +function check_rc +{ + if [[ $1 -ne 0 ]] + then + __exit $1 + fi +} + +function __exit +{ + exit $1 +} + +if [[ $1 = "help" ]] +then + echo "options: cleanall tests run_unit run_fvt" + echo "default: build only shipped targets" + exit 0 +fi + +if [[ $(uname) = "Linux" ]] +then + if [[ $(basename $PWD) = "surelock-sw" ]] + then + source env.bash + else + if [[ -z $SURELOCKROOT ]] + then + echo "set SURELOCKROOT or source env.bash before running" + exit 1 + fi + fi + + cd $SURELOCKROOT + LINUX=1 + MAKE=make + + UNAME=$(uname --all) + if [[ $UNAME =~ "ppc64le" ]] + then + LE=1 + else + LE=0 + fi +else + #AIX + if [[ ! -z $SURELOCKROOT ]] + then + cd $SURELOCKROOT + fi + if [[ $(basename $PWD) != "surelock-sw" ]] + then + echo "must be in surelock-sw dir to execute" + exit 1 + fi + if [[ -z $SURELOCKROOT ]] + then + SURELOCKROOT= + fi + LINUX=0 + LE=0 + MAKE=gmake +fi + +unset CUSTOMFLAGS +unset BLOCK_FILEMODE_ENABLED +unset BLOCK_MC_ENABLED +unset TARGET_PLATFORM +unset BLOCK_KERNEL_MC_ENABLED + +if [[ -h customrc ]] +then + rm -f customrc +fi + +if [[ $(uname -m) =~ x86_64 ]] +then + ln -s customrc.x86_64 customrc + source env.bash +fi + +if [[ -z $1 ]] +then + $MAKE -j16 + __exit $? +fi + +if [[ $1 = "cleanall" && -z $2 ]] +then + $MAKE cleanall + $MAKE -j16 + __exit $? +fi + +if [[ $1 = "cleanall" || $2 = "cleanall" || $3 = "cleanall" ]] +then + $MAKE cleanall +fi + +if [[ $1 = "tests" || $2 = "tests" || $3 = "tests" ]] +then + $MAKE tests -j16 + rc=$? + check_rc $rc +fi + +if [[ $1 = "run_unit" || $2 = "run_unit" || $3 = "run_unit" ]] +then + $MAKE tests -j16 + $MAKE run_unit + __exit $? +fi + +if [[ $1 = "run_fvt" || $2 = "run_fvt" || $3 = "run_fvt" ]] +then + $MAKE tests -j16 + $MAKE run_fvt + __exit $? +fi + +__exit $? diff --git a/src/build/tools/promoteafu b/src/build/tools/promoteafu new file mode 100755 index 00000000..6373fa24 --- /dev/null +++ b/src/build/tools/promoteafu @@ -0,0 +1,80 @@ +#!/bin/bash -e +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/build/tools/promoteafu $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +set -x +#name of our new AFU +NEWAFU=$1 +#source directory +AFURELDIR=/gsa/ausgsa/projects/s/surelock/images +#destination dir +RESOURCES=$SURELOCKROOT/src/build/install/resources +#adapter flash script from installer +FLASH_ADAPTER_SCRIPT=$RESOURCES/flash_all_adapters + +if [[ -z $SURELOCKROOT ]]; then echo "source env.bash first"; exit; fi + +cd $SURELOCKROOT + +#print usage +if [ -z "$AFURELDIR/$NEWAFU" ]; then + echo "$0 afuname.ext" + echo "Usage: invoke this program with an AFU name. It will be copied from the afu release directory and committed into surelock." + exit 1 +fi + +#delete any old ones +# Check image for Surelock card +if [[ -f src/build/install/corsa*.bin && $1 =~ bin ]]; then + echo "removing current image file corsa*.bin" + git rm src/build/install/corsa*.bin + +# Check for image for Surelock Capi card with PAPR header +elif [[ -f src/build/install/1410f004*.* && $1 =~ 1410f004 ]]; then + echo "removing current image file 1410f004*.* present" + git rm src/build/install/1410f0041410f004.* + +# Check image for NVMe card +elif [[ -f src/build/install/14100106*.* && $1 =~ 14100106 ]]; then + echo "removing current image file 14100106*.* present" + git rm src/build/install/14100106* +fi + + +#make sure AFU has group-readable permissions and world-readable permissions +#this is necessary because we don't always force a user to be gsa-logged-in +chmod 755 $AFURELDIR/$NEWAFU + +#create the new sym link +mkdir -p $RESOURCES +ln -s $AFURELDIR/$NEWAFU $RESOURCES/$NEWAFU + + +#add it to the flash script +sed -i "s/^CORSA_IMAGE=.*/CORSA_IMAGE=$NEWAFU/" $FLASH_ADAPTER_SCRIPT + +#make a git commit with the necessary changes +git add $RESOURCES/$NEWAFU +git add $FLASH_ADAPTER_SCRIPT +git commit -m "Promote new AFU - $NEWAFU" diff --git a/src/cflash/cxlfcommon.c b/src/cflash/cxlfcommon.c index ddc666f1..3e659392 100644 --- a/src/cflash/cxlfcommon.c +++ b/src/cflash/cxlfcommon.c @@ -56,13 +56,13 @@ int set_lun_mode(lun_table_entry_t* lun_entry_ptr, MODE_T mode) */ -bool cxlf_set_mode(char* target_device, uint8_t target_mode, uint8_t* wwid) +int cxlf_set_mode(char* target_device, uint8_t target_mode, uint8_t* wwid) { /*------------------------------------------------------------------------*/ /* Local Variables */ /*------------------------------------------------------------------------*/ int fd = 0; - bool rc = false; + int rc = false; struct dk_cxlflash_manage_lun manage_lun = {{0}}; int retrycount=0; uint8_t empty_wwid[DK_CXLFLASH_MANAGE_LUN_WWID_LEN] = {0}; @@ -97,9 +97,9 @@ bool cxlf_set_mode(char* target_device, uint8_t target_mode, uint8_t* wwid) TRACEV("WWID: "); for(i=0; i < DK_CXLFLASH_MANAGE_LUN_WWID_LEN; i++) { - printf("%02x", wwid[i]); + TRACED("%02x", wwid[i]); } - printf("\n"); + TRACED("\n"); } @@ -114,11 +114,9 @@ bool cxlf_set_mode(char* target_device, uint8_t target_mode, uint8_t* wwid) fd = open(target_device, (O_EXCL|O_NONBLOCK|O_RDWR)); retrycount++; } while((fd < 0) && (retrycount < MAX_FOPEN_RETRIES)); - int errsv = errno; if (fd < 0) { - TRACED("Unable to open device / special file.: '%s' (errno %d)\n", target_device, errsv); - rc = false; + rc = EBUSY; break; } @@ -140,20 +138,30 @@ bool cxlf_set_mode(char* target_device, uint8_t target_mode, uint8_t* wwid) memcpy(manage_lun.wwid, wwid, sizeof(manage_lun.wwid)); - + errno=0; int rslt = ioctl(fd, DK_CXLFLASH_MANAGE_LUN, &manage_lun); - TRACEV("ioctl result: %d\n",rslt); if(rslt == 0) { //only set the "true" return code if we were able to successfully make the call to the DD rc=true; } + else + { + TRACED("MANAGE_LUN ioctl ("); + for(i = 0; i /sys/kernel/debug/powerpc/eeh_max_freezes"); CHECK_RC(rc, "Failed to make max_freezes"); @@ -83,9 +95,43 @@ int do_perst() // reload_all_adapters will perform the PESRT - rc=system("/opt/ibm/capikv/afu/reload_all_adapters"); - CHECK_RC(rc, "Failed to reload updated afu image"); - + fileP = fopen("/tmp/perstinfo_file", "r"); + if (NULL == fileP) + { + /* if perstinfo_file does not present; + User will do PERST in all adapter + */ + rc=system("/opt/ibm/capikv/afu/reload_all_adapters"); + CHECK_RC(rc, "Failed to reload updated afu image"); + + } + else + { + + while (fgets(tmpBuff,MAXBUFF, fileP) != NULL) + { + iTer = 0; + + while (iTer < MAXBUFF) + { + if (tmpBuff[iTer] =='\n') + { + tmpBuff[iTer]='\0'; + break; + } + iTer++; + } + + sprintf(blockCheckP," /opt/ibm/capikv/afu/cflash_perst.pl -t %s ", tmpBuff ); + printf("......... command : %s \n",blockCheckP); + rc = system(blockCheckP); + if ( rc ) + { + printf(".................... do_perst() - PERST-ing failed for %s ........ \n", tmpBuff ); + return rc; + } + } // End of While (fgets()) + } // End of else rc=system("modprobe -v cxlflash"); CHECK_RC(rc, "Failed to load cxlflash driver"); @@ -117,6 +163,13 @@ int ioctl_7_1_188( int flag ) // just for sake of cleanup ! set_spio_mode(); +#ifndef _AIX + char WWID[MAXBUFF]; + + rc = diskToWWID ( WWID ); + CHECK_RC(rc, "diskToWWID failed"); +#endif + //ctx_init with default flash disk & devno rc = ctx_init(p_ctx); CHECK_RC(rc, "Context init failed"); @@ -193,6 +246,10 @@ int ioctl_7_1_188( int flag ) rc = do_microcode_update(); CHECK_RC(rc, "do_microcode_update failed"); +#ifndef _AIX + rc = WWIDtoDisk ( WWID); + CHECK_RC(rc, "WWIDToDisk failed"); +#endif //ctx_init with default flash disk & devno rc = ctx_init(p_ctx); CHECK_RC(rc, "Context init failed"); diff --git a/src/cflash/test/196.c b/src/cflash/test/196.c index 87f005d6..953dad04 100644 --- a/src/cflash/test/196.c +++ b/src/cflash/test/196.c @@ -127,7 +127,7 @@ int ioctl_7_1_196() #else //TBD for linux #endif - p_ctx[0]->work.num_interrupts = p_ctx[1]->work.num_interrupts = 4; + p_ctx[0]->work.num_interrupts = p_ctx[1]->work.num_interrupts = cflash_interrupt_number(); rc=ioctl_dk_capi_attach_reuse(p_ctx[0],p_ctx[1],LUN_DIRECT); diff --git a/src/cflash/test/203.c b/src/cflash/test/203.c index c9f40d3d..3899a809 100644 --- a/src/cflash/test/203.c +++ b/src/cflash/test/203.c @@ -406,6 +406,15 @@ int ioctl_7_1_203() int rc=0; rc = setenv("NCHAN_VALUE", "0", true); CHECK_RC(rc, "NCHAN_VALUE env value setting failed \n"); + +#ifndef _AIX + if( diskSizeCheck( cflash_path, 256)) + { + TESTCASE_SKIP("DISK SIZE is less than required \n"); + return 0; + } +#endif + MAX_RES_HANDLE=get_max_res_hndl_by_capacity(cflash_path); if (MAX_RES_HANDLE <= 0) { diff --git a/src/cflash/test/cflash_test.h b/src/cflash/test/cflash_test.h index 9a8b8b1b..9ef3f3e5 100755 --- a/src/cflash/test/cflash_test.h +++ b/src/cflash/test/cflash_test.h @@ -1,4 +1,4 @@ - /* IBM_PROLOG_BEGIN_TAG */ +/* IBM_PROLOG_BEGIN_TAG */ /* This is an automatically generated prolog. */ /* */ /* $Source: src/cflash/test/cflash_test.h $ */ @@ -29,6 +29,23 @@ // For jenkins run, we won't build manual tests by default ! //#define MANUAL +// AIX_MANUAL : under this tag test cases are MANUAL in AIX but automated in Linux +// Like - EEH test cases. +#ifndef _AIX + +#define AIX_MANUAL + +#endif + +// Incase we define MANUAL then all the test case either MANUAL or AIX_MANUAL will be +/// enabled and available to user. + +#ifdef MANUAL + +#define AIX_MANUAL + +#endif + #include #include #ifdef _AIX @@ -77,7 +94,6 @@ typedef __u64 dev64_t; //no use in Linux, its dummy #include #include #include - #include #define MC_PATHLEN 64 @@ -93,6 +109,8 @@ typedef __u64 dev64_t; //no use in Linux, its dummy #define CFLASH_ADAP_POLL_INDX 0 #define CFLASH_DISK_POLL_INDX 1 +#define CFLSH_NUM_INTERRUPTS 4 + // below flags are going to be used in do_write_or_read() // depending on the user need @@ -139,11 +157,17 @@ typedef __u64 dev64_t; //no use in Linux, its dummy #define MAX_OPENS 494 #else #define MAX_OPENS 508 +#define MAX_OPENS_PVM 502 #endif //max allowd VLUNS #define MAX_VLUNS 1024 + +#ifndef _AIX +#define RECO_DISK_SIZE 500 +#endif + #ifndef _AIX #define B_DONE 0x01 #define B_ERROR 0x02 @@ -198,6 +222,7 @@ typedef struct eehCmd int diskToPCIslotConv( char *, char * ); int prepEEHcmd(char *, char * ); void * do_trigger_eeh_cmd( void * ); +int is_UA_device( char * disk_name ); //Context structure. @@ -209,6 +234,7 @@ struct ctx char rbuf[NUM_CMDS][0x1000]; // 4K read data buffer (page aligned) char wbuf[NUM_CMDS][0x1000]; // 4K write data buffer (page aligned) __u64 rrq_entry[NUM_RRQ_ENTRY]; // 128B RRQ (page aligned) + sisl_ioarcb_t sq_entry[NUM_RRQ_ENTRY+1]; struct afu_cmd { @@ -235,6 +261,12 @@ struct ctx __u64 *p_hrrq_start; __u64 *p_hrrq_end; volatile __u64 *p_hrrq_curr; + + //SQ support + sisl_ioarcb_t *p_sq_start; + sisl_ioarcb_t *p_sq_end; + volatile sisl_ioarcb_t *p_sq_curr; + unsigned int toggle; @@ -266,6 +298,8 @@ struct ctx uint64_t exceptions; char sense_data[512]; int dummy_sense_flag; + int close_adap_fd_flag; /* set when attach returns APP_CLOSE_ADAP_FD */ + int sq_mode_flag; /* flag will be set when AFU is in SQ mode */ uint64_t adap_except_type; uint64_t adap_except_time; uint64_t adap_except_data; @@ -341,6 +375,18 @@ struct flash_disk }; +#ifndef _AIX +typedef +enum { + CFLASH_HOST_UNKNOWN = 0, /* Unknown host type */ + CFLASH_HOST_NV = 1, /* Bare Metal (or No virtualization */ + CFLASH_HOST_PHYP = 2, /* pHyp host type */ + CFLASH_HOST_KVM = 3, /* KVM host type */ +} cflash_host_type_t; + +cflash_host_type_t host_type ; + +#endif typedef enum @@ -642,6 +688,7 @@ int ctx_reinit(struct ctx *p_ctx); int ctx_close(struct ctx *p_ctx); void ctx_close_thread(void *); int get_fvt_dev_env(); +int cflash_interrupt_number(void ); int test_init(struct ctx *p_ctx); void *ctx_rrq_rx(void *arg); @@ -748,6 +795,7 @@ int create_resource(struct ctx *p_ctx, __u64 nlba, __u64 flags, __u16 lun_type); int vlun_resize(struct ctx *p_ctx, __u64 nlba); int wait4all(); +int wait4allOnlyRC(); int do_io(struct ctx *p_ctx, __u64 stride); int do_io_nocompare(struct ctx *p_ctx, __u64 stride); int do_large_io(struct ctx *p_ctx, struct rwlargebuf *rwbuf, __u64 size); @@ -762,6 +810,14 @@ int ioctl_dk_capi_query_path_check_flag(struct ctx *p_ctx, int setRUnlimited(); char * diskWithoutDev(char * source , char * destination ); #endif +#ifndef _AIX +int turnOffTestCase( char * ); +void setupFVT_ENV( void ); +int diskSizeCheck( char * , float ); +void identifyHostType( void ); +int diskToWWID ( char * WWID); +int WWIDtoDisk ( char * WWID); +#endif int test_spio_vlun(int); int test_spio_plun(); int test_fc_port_reset_vlun(); diff --git a/src/cflash/test/cflash_test2.c b/src/cflash/test/cflash_test2.c index 40c93285..20cd836c 100644 --- a/src/cflash/test/cflash_test2.c +++ b/src/cflash/test/cflash_test2.c @@ -709,7 +709,13 @@ int mc_test_chunk_regress(int cmd) int i; //pid_t pid1; int max_p= MAX_OPENS; - +#ifndef _AIX + /* By default max_p will be set to BML max user context value */ + if ( host_type == CFLASH_HOST_PHYP ) + { + max_p = MAX_OPENS_PVM ; + } +#endif pid = getpid(); MAX_RES_HANDLE=get_max_res_hndl_by_capacity(cflash_path); if (MAX_RES_HANDLE <= 0) @@ -931,8 +937,8 @@ int test_mc_good_error_afu_dev() cfdisk = get_flash_disks(fldisks, FDISKS_ALL); if (cfdisk < 2) { - fprintf(stderr, "Failed,need 2 flash disks \n"); - return -1; + TESTCASE_SKIP("Failed,need 2 flash disks "); + return 0; } char *str1 = getenv("LONG_RUN"); diff --git a/src/cflash/test/cflash_test_engine.c b/src/cflash/test/cflash_test_engine.c index e10a1a7f..73a4bb77 100644 --- a/src/cflash/test/cflash_test_engine.c +++ b/src/cflash/test/cflash_test_engine.c @@ -24,6 +24,11 @@ /* IBM_PROLOG_END_TAG */ #include "cflash_test.h" +#ifndef _AIX +extern int g_error; +extern int g_errno; +#endif + int test1() { return -1; @@ -39,9 +44,15 @@ int mc_test_engine(mc_test_t test_name) if (DEBUG) system("ctctrl -c cflashdd -r -q"); #else + // setting errno,g_error,g_errno to zero ; want to avoid any error set before test really start + errno=0; + g_error =0; + g_errno=0; + if (DEBUG) system("echo \"module cxlflash +p\" > /sys/kernel/debug/dynamic_debug/control"); displayBuildinfo(); + setupFVT_ENV(); #endif if (DEBUG) diff --git a/src/cflash/test/cflash_test_error2.c b/src/cflash/test/cflash_test_error2.c index 9b872641..14e1ca36 100644 --- a/src/cflash/test/cflash_test_error2.c +++ b/src/cflash/test/cflash_test_error2.c @@ -30,6 +30,9 @@ #include #define KB 1024 +extern int manEEHonoff; +extern int quickEEHonoff; + // do_io() will use this pid. extern pid_t pid; extern int g_error; @@ -62,7 +65,21 @@ int test_vSpio_eehRecovery(int cmd) // pid used to create unique data patterns & logging from util ! pid = getpid(); +#ifndef _AIX + if (turnOffTestCase("PVM") && cmd == 1 && manEEHonoff == 0) + { + TESTCASE_SKIP("Test case not supported in PowerVM env"); + return 0; + } + + if ( quickEEHonoff == 1 && cmd == 1 ) + { + TESTCASE_SKIP(" Test case will be skipped as user requested LIMTED EEH run"); + return 0; + } + +#endif //ctx_init with default flash disk & devno rc = ctx_init(p_ctx); CHECK_RC(rc, "Context init failed"); @@ -113,9 +130,17 @@ int test_vSpio_eehRecovery(int cmd) #endif rc = do_io(p_ctx, stride); - if ( rc == 2) rc=0; - else CHECK_RC(rc, "1st IO attempt didn't fail"); - + //SW356037: First IO will not get UA for FlashGT + if ( is_UA_device( p_ctx->dev ) == TRUE ) + { + if ( rc == 2 ) rc=0; + else CHECK_RC(1, "1st IO attempt didn't fail"); + } + else + { + CHECK_RC(rc, "do_io() failed"); + p_ctx->dummy_sense_flag = 1; + } #ifdef _AIX last_lba = p_ctx->last_phys_lba; #else @@ -186,6 +211,21 @@ int test_dSpio_eehRecovery(int cmd) // pid used to create unique data patterns & logging from util ! pid = getpid(); +#ifndef _AIX + if (turnOffTestCase("PVM") && cmd == 1 && manEEHonoff == 0) + { + TESTCASE_SKIP("Test case not supported in PowerVM env"); + return 0; + } + + if ( quickEEHonoff == 1 && cmd == 1 ) + { + TESTCASE_SKIP(" Test case will be skipped as user requested LIMTED EEH run"); + return 0; + } + +#endif + // ctx_init with default flash disk & devno rc = ctx_init(p_ctx); CHECK_RC(rc, "Context init failed"); @@ -234,9 +274,17 @@ int test_dSpio_eehRecovery(int cmd) #endif rc = do_io(p_ctx, stride); - if ( rc == 2) rc=0; - else CHECK_RC(rc, "1st IO attempt didn't fail"); - + //SW356037: First IO will not get UA for FlashGT + if ( is_UA_device( p_ctx->dev ) == TRUE ) + { + if ( rc == 2 ) rc=0; + else CHECK_RC(1, "1st IO attempt didn't fail"); + } + else + { + CHECK_RC(rc, "do_io() failed"); + p_ctx->dummy_sense_flag = 1; + } #ifdef _AIX last_lba = p_ctx->last_phys_lba; #else @@ -327,7 +375,7 @@ int test_ioctl_fcp() // Prepare for attach ioctl p_ctx->flags = DK_AF_ASSIGN_AFU; - p_ctx->work.num_interrupts = 4; // use num_interrupts from AFU desc + p_ctx->work.num_interrupts = cflash_interrupt_number(); // use num_interrupts from AFU desc #ifdef _AIX p_ctx->devno = fc_devno; #endif /*_AIX */ @@ -750,7 +798,7 @@ int test_ioctl_spio_errcase() #ifdef _AIX p_ctx->work.num_interrupts = 5; // use num_interrupts from AFU desc #else - p_ctx->work.num_interrupts = 4; // use num_interrupts from AFU desc + p_ctx->work.num_interrupts = cflash_interrupt_number(); // use num_interrupts from AFU desc #endif /*_AIX*/ p_ctx->context_id = 0x1; diff --git a/src/cflash/test/cflash_test_ioctl.c b/src/cflash/test/cflash_test_ioctl.c index 53af8737..5698b641 100644 --- a/src/cflash/test/cflash_test_ioctl.c +++ b/src/cflash/test/cflash_test_ioctl.c @@ -37,6 +37,11 @@ static int threadRC; extern int irPFlag ; #endif +#ifndef _AIX +extern int manEEHonoff; +extern int quickEEHonoff; +#endif + char *diskList[MC_PATHLEN]; int diskCount = 0 ; @@ -301,7 +306,7 @@ int test_dca_ioctl(int flag) // func@DK_CAPI_ATTACH { case 1: //TEST_DCA_VALID_ALL_VALUES TCN@7.1.10 p_ctx->flags = p_ctx->work.flags; - p_ctx->work.num_interrupts = 4; + p_ctx->work.num_interrupts = cflash_interrupt_number(); rc = ioctl_dk_capi_attach(p_ctx); CHECK_RC(rc, "DK_CAPI_ATTACH failed"); break; @@ -400,8 +405,23 @@ int test_dca_error_ioctl(int flag) // func@DK_CAPI_ATTACH error path { fprintf(stderr,"%d:Attention:System doesn't fullfil test req,Need 2 disks from a same adapter\n",pid); TESTCASE_SKIP("Need disk from same adapter"); +#ifndef _AIX + if(flag == 4) + return 1; +#endif return 0; } + +#ifndef _AIX + + if (turnOffTestCase("PVM") && ( flag == 10 || flag == 15 || flag == 16 ) && manEEHonoff == 0) + { + TESTCASE_SKIP("Test case not supported in PowerVM env"); + return 0; + } + +#endif + //open CAPI Flash disk device strcpy(p_ctx->dev,disks[0].dev); strcpy(p_ctx_1->dev,disks[1].dev); @@ -424,7 +444,7 @@ int test_dca_error_ioctl(int flag) // func@DK_CAPI_ATTACH error path #else //TBD for linux #endif - p_ctx->work.num_interrupts = p_ctx_1->work.num_interrupts = 4; + p_ctx->work.num_interrupts = p_ctx_1->work.num_interrupts = cflash_interrupt_number(); switch ( flag ) { @@ -615,6 +635,25 @@ int test_dcrc_ioctl(int flag) // func@DK_CAPI_RECOVER_CTX pid = getpid(); +#ifndef _AIX + + if (turnOffTestCase("PVM") && manEEHonoff == 0) + { + if ( flag == 3 || flag == 4 || flag == 7 || flag == 12 || flag == 13 || flag == 8 ) + { + TESTCASE_SKIP("Test case not supported in PowerVM env"); + return 0; + } + } + + if (quickEEHonoff == 1 && ( flag == 4 || flag == 7 || flag == 8 || flag == 12 || flag == 13)) + { + TESTCASE_SKIP(" Test case will be skipped as user requested LIMITED EEH run"); + return 0; + } + +#endif + if ( flag == 4 ) { prepDiskList(cflash_path); @@ -851,9 +890,17 @@ int test_dcrc_ioctl(int flag) // func@DK_CAPI_RECOVER_CTX #endif rc = do_io(p_ctx, stride); - if ( rc == 2) rc=0; - else CHECK_RC(rc, "1st IO attempt didn't fail"); - + //SW356037: First IO will not get UA for FlashGT + if ( is_UA_device( p_ctx->dev ) == TRUE ) + { + if ( rc == 2 ) rc=0; + else CHECK_RC(1, "1st IO attempt didn't fail"); + } + else + { + CHECK_RC(rc, "do_io() failed"); + p_ctx->dummy_sense_flag = 1; + } #ifdef _AIX p_ctx->flags = DK_VF_HC_TUR; p_ctx->hint = DK_HINT_SENSE; @@ -883,8 +930,17 @@ int test_dcrc_ioctl(int flag) // func@DK_CAPI_RECOVER_CTX #endif rc=do_io(p_array_ctx[0],stride); - if ( rc == 2) rc=0; - else CHECK_RC(rc, "1st IO attempt didn't fail"); + + if ( is_UA_device( p_array_ctx[0]->dev ) == TRUE ) + { + if ( rc == 2 ) rc=0; + else CHECK_RC(1, "1st IO attempt didn't fail"); + } + else + { + CHECK_RC(rc, "do_io() failed"); + p_array_ctx[0]->dummy_sense_flag = 1; + } #ifdef _AIX p_array_ctx[0]->flags = DK_VF_HC_TUR; @@ -1101,8 +1157,16 @@ int test_dcrc_ioctl(int flag) // func@DK_CAPI_RECOVER_CTX pthread_create(&thread2, NULL, ctx_rrq_rx, p_ctx); #endif rc=do_io(p_ctx, stride); - if ( rc == 2 ) rc=0; - else CHECK_RC(1, "1st IO attempt didn't fail"); + //SW356037: First IO will not get UA for FlashGT + if ( is_UA_device( p_ctx->dev ) == TRUE ) + { + if ( rc == 2 ) rc=0; + else CHECK_RC(1, "1st IO attempt didn't fail"); + } + else + { + CHECK_RC(rc, "do_io() failed"); + } rc=do_io(p_ctx, stride); CHECK_RC(rc, "do_io() failed"); @@ -1145,8 +1209,16 @@ int test_dcrc_ioctl(int flag) // func@DK_CAPI_RECOVER_CTX pthread_create(&thread2, NULL, ctx_rrq_rx, p_ctx); #endif rc=do_io(p_ctx, stride); - if ( rc == 2 ) rc=0; - else CHECK_RC(1, "1st IO attempt didn't fail"); + //SW356037: First IO will not get UA for FlashGT + if ( is_UA_device( p_ctx->dev ) == TRUE ) + { + if ( rc == 2 ) rc=0; + else CHECK_RC(1, "1st IO attempt didn't fail"); + } + else + { + CHECK_RC(rc, "do_io() failed"); + } rc=do_io(p_ctx, stride); CHECK_RC(rc, "do_io() failed"); @@ -1812,7 +1884,7 @@ int test_invalid_version_ioctl(int flag) rc = ioctl_dk_capi_query_path(p_ctx); CHECK_RC(rc, "DK_CAPI_QUERY_PATHH failed"); #endif - p_ctx->work.num_interrupts = 4; + p_ctx->work.num_interrupts = cflash_interrupt_number(); switch ( flag ) { #ifdef _AIX @@ -2022,7 +2094,7 @@ int test_dcd_ioctl( int flag ) // func@DK_CAPI_DETACH CHECK_RC(rc, "DK_CAPI_QUERY_PATH failed"); p_ctx->work.num_interrupts = 5; #else - p_ctx->work.num_interrupts = 4; + p_ctx->work.num_interrupts = cflash_interrupt_number(); #endif /*_AIX*/ p_ctx->flags = DK_AF_ASSIGN_AFU; @@ -2486,6 +2558,20 @@ int test_dcv_ioctl( int flag ) // func@DK_CAPI_VERIFY uint64_t exp_last_lba; //__u64 stride= 0x1000; +#ifndef _AIX + if (turnOffTestCase("PVM") && ( flag == 12 || flag == 1 || flag == 3 ) && manEEHonoff == 0) + { + TESTCASE_SKIP("Test case not supported in PowerVM env"); + return 0; + } + + if (quickEEHonoff == 1 && ( flag == 1 || flag == 3 )) + { + TESTCASE_SKIP(" Test case will be skipped as user requested LIMITED EEH run"); + return 0; + } + +#endif pid = getpid(); rc = ctx_init(p_ctx); CHECK_RC(rc, "Context init failed"); @@ -2904,10 +2990,18 @@ int test_dcv_ioctl( int flag ) // func@DK_CAPI_VERIFY CHECK_RC(rc, "ctx_reinit() failed"); - rc = do_io(p_ctx, 0x5); - if ( rc == 0 ) + rc = do_io(p_ctx, 0x100); + + //SW356037: First IO will not get UA for FlashGT + if ( is_UA_device( p_ctx->dev ) == TRUE ) + { + if ( rc == 2 ) rc=0; + else CHECK_RC(1, "1st IO attempt didn't fail"); + } + else { - CHECK_RC(1, "IO did not fail\n"); + CHECK_RC(rc, "do_io() failed"); + p_ctx->dummy_sense_flag = 1; } // Sleep for a while before issuing verify ioctl. diff --git a/src/cflash/test/cflash_test_ioctl_io.c b/src/cflash/test/cflash_test_ioctl_io.c index 44627e25..f6d564c8 100644 --- a/src/cflash/test/cflash_test_ioctl_io.c +++ b/src/cflash/test/cflash_test_ioctl_io.c @@ -33,6 +33,11 @@ extern pid_t ppid; extern bool afu_reset; extern bool long_run_enable; extern char cflash_path[MC_PATHLEN]; + +#ifndef _AIX +extern int manEEHonoff; +#endif + static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; static __u64 chunks[]={ 4,8,16,32,24,16,20,2,12,1 }; static int imLastContext = 0; @@ -312,11 +317,12 @@ int test_spio_direct_virtual() printf("LONG_RUN enabled...\n"); count = 10; } + cfdisk = get_flash_disks(fldisks, FDISKS_ALL); if (cfdisk < 2) { - fprintf(stderr,"Must have 2 flash disks..\n"); - return -1; + TESTCASE_SKIP("Must have 2 flash disks.."); + return 0; } while (count-- >0) @@ -395,7 +401,7 @@ int create_ctx_process(char *dev, dev64_t devno, __u64 chunk) pthread_t intr_thread; //__u64 flags; int i; - + rc = ctx_init2(p_ctx, dev, DK_AF_ASSIGN_AFU, devno); CHECK_RC(rc, "Context init failed"); // interrupt handler per context @@ -432,6 +438,15 @@ int max_ctx_max_res(int cmd) struct ctx myctx; struct ctx *p_ctx=&myctx; int max_p = MAX_OPENS; // we can change MAX_OPEN Value in cflash_tests.h + +#ifndef _AIX + /* By default max_p will be set to BML max user context value */ + if ( host_type == CFLASH_HOST_PHYP ) + { + max_p = MAX_OPENS_PVM ; + } +#endif + memset(p_ctx, 0, sizeof(struct ctx)); __u64 chunk = 0; __u64 lba; @@ -525,10 +540,12 @@ int do_attach_detach(char *dev, dev64_t devno, __u16 lun_type) if (LUN_VIRTUAL == lun_type) { chunk = rand()%16; + debug("%d: initial chunk size=%ld \n",pid,chunk); //create 0 vlun size & later call resize ioctl rc = create_resource(p_ctx, chunk, DK_UVF_ALL_PATHS, lun_type); CHECK_RC(rc, "create LUN_VIRTUAL failed"); chunk = rand()%32; + debug("%d: chunk before vlun_resize=%ld \n",pid,chunk); nlba = chunk * p_ctx->chunk_size; rc = vlun_resize(p_ctx, nlba); CHECK_RC(rc, "vlun_resize failed"); @@ -558,9 +575,18 @@ int test_spio_attach_detach(int cmd) cfdisk = get_flash_disks(disks, FDISKS_ALL); if (cfdisk < 2) { - fprintf(stderr,"Must have 2 flash disks..\n"); - return -1; + TESTCASE_SKIP("Must have 2 flash disks.."); + return 0; } + +#ifndef _AIX + if( diskSizeCheck( cflash_path , RECO_DISK_SIZE )) + { + TESTCASE_SKIP("DISK SIZE is less than required \n"); + return 0; + } +#endif + if (fork() == 0) { //child process @@ -760,7 +786,6 @@ int create_res_hndl_afu_reset(bool do_recover, bool last) #else rc = do_poll_eeh(p_ctx); #endif - g_error=0; //reset any prev error might caught while EEH if ( noIOP == NULL ) { pthread_join(ioThreadId, NULL); @@ -791,6 +816,7 @@ int create_res_hndl_afu_reset(bool do_recover, bool last) p_ctx->hint = DK_HINT_SENSE; #endif fflush(stdout); + g_error=0; //reset any prev error might caught while EEH ctx_reinit(p_ctx); #ifdef _AIX p_ctx->hint=DK_HINT_SENSE; @@ -852,6 +878,7 @@ int create_res_hndl_afu_reset(bool do_recover, bool last) sleep(5); // Don't let child exit to keep max ctx alive rc |= ctx_close(p_ctx); CHECK_RC(rc,"ctx close or close_res failed\n"); + rc |= g_error ; // if g_error is set here inform caller return rc; } @@ -867,6 +894,13 @@ int max_ctx_rcvr_except_last_one() char tmpBuff[MAXBUFF]; char cmdToRun[MAXBUFF]; const char *configCmdP = "echo 10000000 > /sys/kernel/debug/powerpc/eeh_max_freezes"; + + if (turnOffTestCase("PVM") && manEEHonoff == 0) + { + TESTCASE_SKIP("Test case not supported in PowerVM env"); + return 0; + } + #endif system("ipcrm -Q 0x1234 >/dev/null 2>&1"); @@ -892,9 +926,12 @@ int max_ctx_rcvr_except_last_one() #ifdef _AIX printf("%d:.....MAIN: BOSS do EEH now manually.......\n",getpid()); #else - char * manualEehP = getenv("MANUAL_EEH"); - if ( NULL == manualEehP ) + if (manEEHonoff != 0) + { + printf("%d:.....MAIN: BOSS do EEH now manually.......\n",getpid()); + } + else { printf("%d:.....MAIN: doing EEH now .......\n",getpid()); @@ -909,6 +946,7 @@ int max_ctx_rcvr_except_last_one() { g_error = -1; fprintf(stderr,"%d: Failed in %s \n",pid,configCmdP); + exit(1); } debug("%d ---------- Command : %s----------------\n",pid,cmdToRun); @@ -917,14 +955,11 @@ int max_ctx_rcvr_except_last_one() { g_error = -1; fprintf(stderr,"%d: Failed in %s \n",pid,cmdToRun); + exit(1); } } - else - { - printf("%d:.....MAIN: BOSS do EEH now manually.......\n",getpid()); - } #endif sleep(10); // Let the max ctx recover before we proceed. @@ -952,7 +987,7 @@ int max_ctx_rcvr_except_last_one() rc = create_res_hndl_afu_reset(true, true); exit(rc); } - rc = wait4all(); + rc = wait4allOnlyRC(); printf("%d: rc for wait4all(): %d\n", getpid(), rc); system("ipcrm -Q 0x1234"); return rc; @@ -1056,6 +1091,13 @@ int max_ctx_rcvr_last_one_no_rcvr() char tmpBuff[MAXBUFF]; char cmdToRun[MAXBUFF]; const char *configCmdP = "echo 10000000 > /sys/kernel/debug/powerpc/eeh_max_freezes"; + + if (turnOffTestCase("PVM") && manEEHonoff == 0) + { + TESTCASE_SKIP("Test case not supported in PowerVM env"); + return 0; + } + #endif for (i = 0; i < max_p-1; i++) @@ -1089,8 +1131,11 @@ int max_ctx_rcvr_last_one_no_rcvr() #ifdef _AIX printf("%d:.....MAIN: BOSS do EEH now manually.......\n",getpid()); #else - char * manualEehP = getenv("MANUAL_EEH"); - if ( NULL != manualEehP ) + if( manEEHonoff != 0) + { + printf("%d:.....MAIN: BOSS do EEH now manually.......\n",getpid()); + } + else { printf("%d:.....MAIN: doing EEH now .......\n",getpid()); rc = diskToPCIslotConv(cflash_path, tmpBuff ); @@ -1104,6 +1149,7 @@ int max_ctx_rcvr_last_one_no_rcvr() { g_error = -1; fprintf(stderr,"%d: Failed in %s \n",pid,configCmdP); + exit(1); } printf("%d ---------- Command : %s----------------\n",pid,cmdToRun); @@ -1113,16 +1159,13 @@ int max_ctx_rcvr_last_one_no_rcvr() { g_error = -1; fprintf(stderr,"%d: Failed in %s \n",pid,cmdToRun); + exit(1); } } - else - { - printf("%d:.....MAIN: BOSS do EEH now manually.......\n",getpid()); - } #endif - rc = wait4all(); + rc = wait4allOnlyRC(); printf("%d: rc for wait4all(): %d\n", pid, rc); return rc; } @@ -1144,6 +1187,14 @@ int test_clone_ioctl(int cmd) uint64_t RES_CLOSED=-1; int cl_index[5]={ 1,7,10,12,15 }; pid = getpid(); + +#ifndef _AIX + if( diskSizeCheck( cflash_path , 128 )) + { + TESTCASE_SKIP("DISK SIZE is less than required \n"); + return 0; + } +#endif rc =ctx_init(p_ctx); CHECK_RC(rc, "Context init failed"); pthread_create(&thread, NULL, ctx_rrq_rx, p_ctx); @@ -1282,28 +1333,37 @@ int max_ctx_cross_limit() struct ctx myctx; struct ctx *p_ctx=&myctx; pid = getpid(); - int max_p = MAX_OPENS; - //int max_p = 1; - for (i=0; iflags = flags; - p_ctx->work.num_interrupts = 4; // use num_interrupts from AFU desc + + p_ctx->work.num_interrupts = cflash_interrupt_number(); // use num_interrupts from AFU desc p_ctx->devno = devno; //do context attach @@ -203,6 +353,13 @@ int ctx_init_internal(struct ctx *p_ctx, return -1; } + /* AFU is running with SQ mode */ + if ( p_ctx->sq_mode_flag == TRUE ) + { + p_ctx->p_sq_start = &p_ctx->sq_entry[0]; + p_ctx->p_sq_end = &p_ctx->sq_entry[NUM_RRQ_ENTRY]; + p_ctx->p_sq_curr = p_ctx->p_sq_start; + } // initialize RRQ pointers p_ctx->p_hrrq_start = &p_ctx->rrq_entry[0]; p_ctx->p_hrrq_end = &p_ctx->rrq_entry[NUM_RRQ_ENTRY - 1]; @@ -229,8 +386,16 @@ int ctx_init_internal(struct ctx *p_ctx, } else { + write_64(&p_ctx->p_host_map->rrq_start, (__u64) p_ctx->p_hrrq_start); write_64(&p_ctx->p_host_map->rrq_end, (__u64) p_ctx->p_hrrq_end); + + if ( p_ctx->sq_mode_flag == TRUE ) + { + write_64(&p_ctx->p_host_map->sq_start, (__u64) p_ctx->p_sq_start ); + write_64(&p_ctx->p_host_map->sq_end, (__u64) p_ctx->p_sq_end ); + } + } mypid = getpid(); @@ -822,6 +987,11 @@ int send_write(struct ctx *p_ctx, __u64 start_lba, p_ctx->cmd[i].sa.host_use[0] = 0; // 0 means active p_ctx->cmd[i].sa.ioasc = 0; + + if ( p_ctx->sq_mode_flag == TRUE ) + { + p_ctx->cmd[i].rcb.sq_ioasa_ea = (uint64_t)&(p_ctx->cmd[i].sa); + } hexdump(&p_ctx->wbuf[i][0],0x20,"Writing"); } //send_single_cmd(p_ctx); @@ -857,45 +1027,73 @@ int send_cmd(struct ctx *p_ctx) asm volatile( "lwsync" : : ); while (cnt) { - room = read_64(&p_ctx->p_host_map->cmd_room); - debug_2("%d:room =0X%"PRIX64"\n",pid,room); + + if ( p_ctx->sq_mode_flag == TRUE ) + { + debug_2("%d:Placing IOARCB =0X%"PRIX64" into SQ \n", + pid,(__u64)&p_ctx->cmd[p_cmd].rcb); + + hexdump((void *)(&p_ctx->cmd[p_cmd].rcb), sizeof(p_ctx->cmd[p_cmd].rcb), + "RCB data writing in SQ........"); + + bcopy((void*)&p_ctx->cmd[p_cmd++].rcb,(void*)p_ctx->p_sq_curr, + sizeof(sisl_ioarcb_t)); + + p_ctx->p_sq_curr++; + + if ( p_ctx->p_sq_curr > p_ctx->p_sq_end ) + { + p_ctx->p_sq_curr = p_ctx->p_sq_start; + } + + write_64(&p_ctx->p_host_map->sq_tail, (__u64)p_ctx->p_sq_curr); + + cnt--; + } + else + { + + room = read_64(&p_ctx->p_host_map->cmd_room); + debug_2("%d:room =0X%"PRIX64"\n",pid,room); #ifdef _AIX //not to break anything on Linux #ifndef __64BIT__ if (0XFFFFFFFF == room) #else - if ( -1 == room) + if ( -1 == room) #endif - { - fprintf(stderr,"%d:Failed:cmd_room=-1 afu_reset done/not recovered...\n",pid); - usleep(1000); - return -1; - } + { + fprintf(stderr,"%d:Failed:cmd_room=-1 afu_reset done/not recovered...\n",pid); + usleep(1000); + return -1; + } #endif - if (0 == room) - { - usleep(MC_BLOCK_DELAY_ROOM); - wait_try--; - debug("%d:still pending cmd:%d\n",pid,cnt); - } - if (0 == wait_try) - { - fprintf(stderr, "%d : send cmd wait over %d cmd remain\n", + if (0 == room) + { + usleep(MC_BLOCK_DELAY_ROOM); + wait_try--; + debug("%d:still pending cmd:%d\n",pid,cnt); + } + if (0 == wait_try) + { + fprintf(stderr, "%d : send cmd wait over %d cmd remain\n", pid, cnt); - return -1; - } - for (i = 0; i < room; i++) - { - // add a usleep here if room=0 ? - // write IOARRIN - debug_2("%d:Placing IOARCB =0X%"PRIX64" into ioarrin\n", + return -1; + } + for (i = 0; i < room; i++) + { + // add a usleep here if room=0 ? + // write IOARRIN + debug_2("%d:Placing IOARCB =0X%"PRIX64" into ioarrin\n", pid,(__u64)&p_ctx->cmd[p_cmd].rcb); - hexdump((void *)(&p_ctx->cmd[p_cmd].rcb), sizeof(p_ctx->cmd[p_cmd].rcb), + hexdump((void *)(&p_ctx->cmd[p_cmd].rcb), sizeof(p_ctx->cmd[p_cmd].rcb), "RCB data writing in ioarrin........"); - write_64(&p_ctx->p_host_map->ioarrin, + write_64(&p_ctx->p_host_map->ioarrin, (__u64)&p_ctx->cmd[p_cmd++].rcb); - wait_try = MAX_TRY_WAIT; //each cmd give try max time - if (cnt-- == 1) break; - } + wait_try = MAX_TRY_WAIT; //each cmd give try max time + if (cnt-- == 1) break; + } + + } } rw_cmd_loaded=1; return 0; @@ -1048,6 +1246,12 @@ int send_read(struct ctx *p_ctx, __u64 start_lba, p_ctx->cmd[i].sa.host_use[0] = 0; // 0 means active p_ctx->cmd[i].sa.ioasc = 0; + + if ( p_ctx->sq_mode_flag == TRUE ) + { + p_ctx->cmd[i].rcb.sq_ioasa_ea = (uint64_t)&(p_ctx->cmd[i].sa ); + } + } //send_single_cmd(p_ctx); @@ -1500,9 +1704,9 @@ int send_rw_lsize(struct ctx *p_ctx, struct rwlargebuf *p_rwb, buf_len)) { sprintf(buf,"read.%d",pid); - rfd = open(buf,O_RDWR|O_CREAT); + rfd = open(buf,O_RDWR|O_CREAT, 0600); sprintf(buf,"write.%d",pid); - wfd = open(buf,O_RDWR|O_CREAT); + wfd = open(buf,O_RDWR|O_CREAT, 0600); write(rfd,p_rwb->rbuf[i],buf_len); write(wfd,p_rwb->rbuf[i],buf_len); close(rfd); @@ -1763,6 +1967,22 @@ int ioctl_dk_capi_attach(struct ctx *p_ctx) p_ctx->last_phys_lba,p_ctx->block_size,p_ctx->chunk_size,p_ctx->max_xfer); #endif +#ifdef DK_CXLFLASH_APP_CLOSE_ADAP_FD + // if context attach returns DK_CXLFLASH_APP_CLOSE_ADAP_FD flag + if (p_ctx->return_flags & DK_CXLFLASH_APP_CLOSE_ADAP_FD) + { + p_ctx->close_adap_fd_flag = TRUE; + } + +#endif + +#ifdef DK_CXLFLASH_CONTEXT_SQ_CMD_MODE + if ( p_ctx->return_flags & DK_CXLFLASH_CONTEXT_SQ_CMD_MODE ) + { + p_ctx->sq_mode_flag = TRUE ; + } +#endif + debug("%d:adap_fd=%d return_flag=0X%"PRIX64"\n",pid,p_ctx->adap_fd,p_ctx->return_flags); debug("%d:------------- End DK_CAPI_ATTACH -------------\n", pid); @@ -1817,6 +2037,12 @@ int ioctl_dk_capi_detach(struct ctx *p_ctx) #else p_ctx->return_flags = capi_detach.hdr.return_flags; #endif + + // if DK_CXLFLASH_APP_CLOSE_ADAP_FD flag returns during attach + if (p_ctx->close_adap_fd_flag == TRUE) + { + close(p_ctx->adap_fd); + } debug("%d:return_flag=0X%"PRIX64"\n",pid,p_ctx->return_flags); debug("%d:--------------- End DK_CAPI_DETACH -------------\n",pid); @@ -2179,6 +2405,7 @@ int ioctl_dk_capi_log(struct ctx *p_ctx, char *s_data) int ioctl_dk_capi_recover_ctx(struct ctx *p_ctx) { int rc; + debug("%d:--------Start DK_CAPI_RECOVER_CTX ---------\n",pid); #ifdef _AIX struct dk_capi_recover_context recv_ctx; @@ -2205,9 +2432,13 @@ int ioctl_dk_capi_recover_ctx(struct ctx *p_ctx) p_ctx->mmio_size = recv_ctx.mmio_size; p_ctx->return_flags = recv_ctx.return_flags; #else + + int old_adap_fd ; + struct dk_cxlflash_recover_afu recv_ctx; memset(&recv_ctx, 0, sizeof(recv_ctx)); + old_adap_fd = p_ctx->adap_fd; recv_ctx.hdr.version = p_ctx->version; recv_ctx.hdr.flags = p_ctx->flags; recv_ctx.context_id = p_ctx->context_id; @@ -2242,6 +2473,13 @@ int ioctl_dk_capi_recover_ctx(struct ctx *p_ctx) CHECK_RC(1,"mmap failed"); } else debug("%d: New mmap() returned success..\n", pid); + + // if context attach returns DK_CXLFLASH_APP_CLOSE_ADAP_FD flag + if(p_ctx->close_adap_fd_flag == TRUE) + { + close(old_adap_fd); + } + } else { @@ -2624,6 +2862,69 @@ int wait4all() return rc; } +int wait4allOnlyRC() +{ + int rc = 0; + int rc1 = 0; + pid_t mypid; + int nerror = 0; + int rc_fail_cnt=0; + char * noIOP = getenv("NO_IO"); + + while ((mypid = waitpid(-1, &rc, 0))) + { + debug("pid %d waitpid() status =%d\n", mypid,rc); + if (mypid == -1) + { + break; + } + if (WIFEXITED(rc)) + { + rc1 |= WEXITSTATUS(rc); + if (rc1) + { + /* It should start failing whenver it hits the first fail */ + fprintf(stderr, "%d : Returned Failure RC = %d\n", mypid,rc); + nerror = -1; + rc_fail_cnt = rc_fail_cnt +1; + // IO can fail with UA only one time after Recover. + // if rc_fail_cnt = 1 ; then rc_fail_cnt has the authority to change + // nerror = 0 else test case will be marked as failed. + // First IO failure scenario can only happen if we are running with + // IO. In case of NO IO and non-SurelockFC , we should expect no failure + if ( rc_fail_cnt == 1 && noIOP == NULL && is_UA_device(cflash_path) == TRUE ) + { + debug("pid %d Reset cnt %d for nerror status =%d\n", mypid,rc_fail_cnt,nerror); + nerror = 0; + rc1 = 0; + } + } + + } + else + { + fprintf(stderr, "%d : abnormally terminated\n", mypid); + nerror =-1; + } + debug("pid %d exited with rc=%d\n", mypid,rc); + fflush(stdout); + + } + + // Ideally rc_fail_cnt should be 1 ; if it is zero. that IO never failed + // with UA. which is not expected case. + // In case of NO IO, we should expect no failure. rc_fail_cnt should be 0 + if ( rc_fail_cnt == 0 && noIOP == NULL && is_UA_device(cflash_path) == TRUE ) + { + debug("pid %d First IO did not fail with UA, So test failed \n", getpid()); + nerror = 255; + } + + rc = nerror; + nerror = 0; + return rc; +} + int do_internal_io(struct ctx *p_ctx, __u64 stride, bool iocompare) { __u64 st_lba= p_ctx->st_lba; @@ -2709,7 +3010,7 @@ int get_max_res_hndl_by_capacity(char *dev) #ifdef _AIX rc=ioctl_dk_capi_query_path(p_ctx); #endif - p_ctx->work.num_interrupts =4; + p_ctx->work.num_interrupts =cflash_interrupt_number(); rc = ioctl_dk_capi_attach(p_ctx); if (rc) { @@ -2747,7 +3048,7 @@ __u64 get_disk_last_lba(char *dev, dev64_t devno, uint64_t *chunk_size) strcpy(myctx.dev, dev); myctx.devno = devno; myctx.flags = DK_AF_ASSIGN_AFU; - myctx.work.num_interrupts =4; + myctx.work.num_interrupts =cflash_interrupt_number(); rc = ioctl_dk_capi_attach(&myctx); last_lba = myctx.last_phys_lba; *chunk_size = myctx.chunk_size; @@ -2846,8 +3147,7 @@ int do_eeh(struct ctx *p_ctx) pthread_condattr_t cattrVar; char tmpBuff[MAXBUFF]; - char * manualEehP = getenv("MANUAL_EEH"); - if ( NULL != manualEehP ) + if ( manEEHonoff != 0 ) { while (1) { @@ -3287,6 +3587,12 @@ int ioctl_dk_capi_clone(struct ctx *p_ctx,uint64_t src_ctx_id,int src_adap_fd) rc =ioctl(p_ctx->fd,DK_CXLFLASH_VLUN_CLONE, &clone); if (rc) CHECK_RC(errno, "DK_CXLFLASH_VLUN_CLONE failed with errno\n"); + + // if context attach returns DK_CXLFLASH_APP_CLOSE_ADAP_FD flag + if ( p_ctx->close_adap_fd_flag == TRUE ) + { + close(src_adap_fd); + } debug("%d:----------- Done DK_CXLFLASH_VLUN_CLONE ----------\n", pid); #endif return rc; @@ -3947,7 +4253,7 @@ int ioctl_dk_capi_attach_reuse_all_disk( ) #ifdef _AIX new_ctx[i]->work.num_interrupts = 5; #else - new_ctx[i]->work.num_interrupts = 4; + new_ctx[i]->work.num_interrupts = cflash_interrupt_number(); //TBD for linux #endif @@ -4369,6 +4675,7 @@ int keep_doing_eeh_test(struct ctx *p_ctx) int rc; pthread_t ioThreadId; pthread_t thread; + pthread_t thread2; do_io_thread_arg_t ioThreadData; do_io_thread_arg_t * p_ioThreadData=&ioThreadData; p_ioThreadData->p_ctx=p_ctx; @@ -4384,11 +4691,16 @@ int keep_doing_eeh_test(struct ctx *p_ctx) CHECK_RC(rc, "do_io_thread() pthread_create failed"); //Trigger EEH do_eeh(p_ctx); - rc = ioctl_dk_capi_recover_ctx(p_ctx); - CHECK_RC(rc, "ctx reattached failed"); + + // Wait for IO thread to complete + pthread_join(ioThreadId, NULL); + #ifndef _AIX pthread_cancel(thread); #endif + + rc = ioctl_dk_capi_recover_ctx(p_ctx); + CHECK_RC(rc, "ctx reattached failed"); #ifdef _AIX if (!(p_ctx->return_flags & DK_RF_REATTACHED)) CHECK_RC(1, "recover ctx, expected DK_RF_REATTACHED"); @@ -4397,7 +4709,10 @@ int keep_doing_eeh_test(struct ctx *p_ctx) #endif ctx_reinit(p_ctx); usleep(1000); -#ifdef _AIX + +#ifndef _AIX + pthread_create(&thread2, NULL, ctx_rrq_rx, p_ctx); +#endif //better to use io(get failed with UA) rather than verify //otherwise do call verify ioctl on all paths rc=do_io(p_ctx,0x10000); @@ -4405,10 +4720,15 @@ int keep_doing_eeh_test(struct ctx *p_ctx) { fprintf(stderr,"%d:expected to fail for UA, dont worry....\n",pid); } -#endif rc = ioctl_dk_capi_verify(p_ctx); CHECK_RC(rc, "ioctl_dk_capi_verify failed"); usleep(1000); + + rc=do_io(p_ctx,0x1000); + CHECK_RC(rc, "2nd IO request failed\n"); +#ifndef _AIX + pthread_cancel(thread2); +#endif } return 0; } @@ -4562,7 +4882,6 @@ int allDiskToArray( char ** allDiskArrayP, int * diskCountP) int diskInSameAdapater( char * p_file ) { - int rc =0; #ifndef _AIX int iCount =0; @@ -4572,7 +4891,6 @@ int diskInSameAdapater( char * p_file ) char tmpBuff[MAXBUFF]; char npBuff[MAXNP][MAXBUFF]; - char blockCheckP[MAXBUFF]; char * allDiskArray [MAXBUFF]; char sameAdapDisk[MAXNP][MAXBUFF]; @@ -4580,7 +4898,7 @@ int diskInSameAdapater( char * p_file ) int smCnt = 0; int allCnt = 0; - const char *initCmdP = "lspci -v | grep \"Processing accelerators\" | awk '{print $1}' > /tmp/trashFile"; + const char *initCmdP = "/opt/ibm/capikv/bin/cxlfstatus | grep superpipe | awk '{print $2}' | cut -d: -f1 | sort -u > /tmp/trashFile"; rc = system(initCmdP); if ( rc != 0) @@ -4622,33 +4940,20 @@ int diskInSameAdapater( char * p_file ) smCnt = 0; - // only supporting for scsi_generic device now - - sprintf(blockCheckP,"ls -l /sys/bus/pci/devices/" - "%s/pci***:**/***:**:**.*/host*/" - "target*:*:*/*:*:*:*/ | grep -w \"scsi_generic\" >/dev/null 2>&1",tmpBuff); - rc = system(blockCheckP); - - if ( rc == 0 ) + for ( allCnt=0; allCnt < diskCount ;allCnt++) { + sprintf(npBuff[iCount]," /opt/ibm/capikv/bin/cxlfstatus | grep %s |" + "awk '{print $2}' | cut -d: -f1 | grep %s >/dev/null 2>&1",allDiskArray[allCnt],tmpBuff); - for ( allCnt=0; allCnt < diskCount ;allCnt++) + rc = system(npBuff[iCount]); + if ( rc == 0 ) { - - sprintf(npBuff[iCount],"ls -l /sys/bus/pci/devices/" - "%s/pci***:**/***:**:**.*/host*/" - "target*:*:*/*:*:*:*/scsi_generic | grep %s >/dev/null 2>&1",tmpBuff,allDiskArray[allCnt]); - - rc = system(npBuff[iCount]); - if ( rc == 0 ) - { - strcpy(sameAdapDisk[smCnt], allDiskArray[allCnt] ); - smCnt++; - } + strcpy(sameAdapDisk[smCnt], allDiskArray[allCnt] ); + smCnt++; } - - iCount++; } + + iCount++; } if ( fclose(fileP) == EOF ) @@ -4673,7 +4978,6 @@ int diskInSameAdapater( char * p_file ) rc = EINVAL; goto xerror; } - } xerror: @@ -4867,4 +5171,241 @@ char * diskWithoutDev(char * source , char * destination ) } +#endif + +#ifndef _AIX + +int turnOffTestCase( char * reqEnv) +{ + char *tmP = NULL; + int rc =0; + + char *envName = (char *)getenv("FVT_ENV"); + + if ( NULL != envName ) + { + tmP = strtok( envName, "," ); + + while ( tmP != NULL ) + { + if (!strcmp(tmP, reqEnv)) + { + rc = 1 ; + break; + } + + tmP=strtok(NULL,","); + } + } + + return rc; +} + +int diskSizeCheck(char * diskName , float recDiskSize) +{ + float diskSize = 0.0 ; + int rc = 0 ; + FILE *filePtr = NULL; + char tmBuff[1024]; + + sprintf(tmBuff,"lsscsi -sg | grep %s | grep GB >/dev/null 2>&1",diskName); + if( system( tmBuff )) + { + sprintf(tmBuff," lsscsi -sg | grep %s | awk -F' ' '{print $NF}' | sed 's/TB//g'", diskName); + filePtr = popen(tmBuff, "r"); + fscanf(filePtr, "%f", &diskSize); + diskSize = diskSize * 1024; // convert to GB + } + else + { + sprintf(tmBuff," lsscsi -sg | grep %s | awk -F' ' '{print $NF}' | sed 's/GB//g'", diskName); + filePtr = popen(tmBuff, "r"); + fscanf(filePtr, "%f", &diskSize); + } + + pclose(filePtr); + + if( diskSize < recDiskSize) + { + rc = 1; // User will be warned as disk size is less or lsscsi package is not installed + printf(" *********WARNING : Recommended disk size for this test %f GB and used disk size is %f" + " OR SYSTEM does not have lsscsi package installed**********************\n",recDiskSize,diskSize); + } + + debug("***** disk size required for this test %f GB and used disk size is %f GB****\n",recDiskSize,diskSize); + + return rc ; +} + +#endif + +int is_UA_device( char * diskName ) +{ +#ifndef _AIX + int rc =0; + int iTer =0; + int iKey =0; + int found = FALSE ; + FILE *fileP; + char tmpBuff[MAXBUFF]; + char blockCheckP[MAXBUFF]; + + /* + is_UA_device: this function will look for devices which can recieve Unit attention. + Right now, Surelock FC is only one to get Unit attention ; however we can add more + device ids - lspci -v | grep -E " | " to inform the caller of this function + */ + + const char *initCmdP = "lspci -v | grep -E \"04cf\" | awk '{print $1}' > /tmp/adapFile"; + + rc = system(initCmdP); + if ( rc != 0) + { + fprintf(stderr,"%d: Failed in lspci \n",pid); + return FALSE; + } + + fileP = fopen("/tmp/adapFile", "r"); + if (NULL == fileP) + { + fprintf(stderr,"%d: Error opening file /tmp/adapFile \n", pid); + return FALSE ; + } + + while (fgets(tmpBuff,MAXBUFF, fileP) != NULL) + { + iTer = 0; + while (iTer < MAXBUFF) + { + if (tmpBuff[iTer] =='\n') + { + tmpBuff[iTer]='\0'; + break; + } + iTer++; + } + // only supporting for scsi_generic device + + iKey = strlen(diskName)-4 ; + sprintf(blockCheckP,"ls -d /sys/devices/*/*/" + "%s/*/*/host*/target*/*:*:*:*/scsi_generic/* | grep -w %s >/dev/null 2>&1",tmpBuff,&diskName[iKey]); + rc = system(blockCheckP); + if ( rc == 0 ) + { + found = TRUE; + printf(".................... is_UA_device() : %s is UA device ........ \n", diskName); + fclose(fileP); + break; + } + } + + return found; +#else + //TBD: AIX related changes will be done in seprate code drop + return TRUE; +#endif +} + +#ifndef _AIX + +#define CXLSTAT "/opt/ibm/capikv/bin/cxlfstatus" + +int diskToWWID (char * WWID) +{ + char * d_name; + char cmdToRun[MAXBUFF]; + int rc = 0; + FILE *fileP = NULL; + int i =0; + + d_name = strrchr( cflash_path, '/') + 1; + + sprintf(cmdToRun,"%s | grep %s | awk -F \" \" '{ print $5 }' > /tmp/WWID", CXLSTAT, d_name); + rc = system(cmdToRun); + if (rc) + return rc; + + fileP = fopen("/tmp/WWID", "r"); + if (NULL == fileP) + { + fprintf(stderr,"%d: Error opening file /tmp/WWID \n", pid); + rc = EINVAL; + return rc; + } + + if ( NULL == fgets(WWID,MAXBUFF, fileP) ) + { + fprintf(stderr,"%d: Error in file /tmp/WWID \n", pid); + rc = EINVAL; + fclose(fileP); + return rc; + } + + while( i < MAXBUFF ) + { + if ( WWID[i]=='\n' ) + { + WWID[i]='\0'; + break; + } + + i++; + } + + fclose(fileP); + + return rc; + +} + +int WWIDtoDisk (char * WWID) +{ + char diskNameBuff[MAXBUFF]; + char cmdToRun[MAXBUFF]; + FILE *fileP = NULL; + int rc = 0; + int i =0; + + sprintf(cmdToRun,"%s | grep %s | awk '{ print $1 }' | head -1 | sed -e 's/://g' > /tmp/cflash_disk", + CXLSTAT, WWID); + + rc = system(cmdToRun); + + if (rc) + return rc; + + fileP = fopen("/tmp/cflash_disk", "r"); + if (NULL == fileP) + { + fprintf(stderr,"%d: Error opening file /tmp/cflash_disk \n", pid); + rc = EINVAL; + return rc; + } + + if ( NULL == fgets(diskNameBuff,MAXBUFF, fileP) ) + { + fprintf(stderr,"%d: Error in file /tmp/cflash_disk \n", pid); + rc = EINVAL; + fclose(fileP); + return rc; + } + + while( i < MAXBUFF ) + { + if ( diskNameBuff[i]=='\n' ) + { + diskNameBuff[i]='\0'; + break; + } + + i++; + } + + sprintf(cflash_path,"/dev/%s", diskNameBuff); + + fclose(fileP); + return rc; + +} + #endif diff --git a/src/cflash/test/fvt_cflash.C b/src/cflash/test/fvt_cflash.C index 35d2cdf6..4ad2d3d9 100644 --- a/src/cflash/test/fvt_cflash.C +++ b/src/cflash/test/fvt_cflash.C @@ -35,6 +35,14 @@ extern "C" #include "cflash_test.h" } + +#ifndef _AIX +TEST(Cflash_FVT_Suite,G_dk_capi_clone) +{ + ASSERT_EQ(0,mc_test_engine(TEST_DK_CAPI_CLONE)); +} +#endif + TEST(Cflash_FVT_Suite, E_MC_test_Ioctl_Invalid_Versions) { #ifdef _AIX @@ -719,15 +727,18 @@ TEST(Cflash_FVT_Suite, E_bad_ioarcb_address) TEST(Cflash_FVT_Suite, E_bad_ioasa_address) { + int res = test_mc_invalid_ioarcb(101); #ifdef _AIX - ASSERT_EQ(255, test_mc_invalid_ioarcb(101)); + if ( res == 255) #else - int res = test_mc_invalid_ioarcb(101); if ( res == 10 || res == 255 ) - res = 1; - - ASSERT_EQ(1, res); #endif + res = 1; + //sanity check + int ret_flag = test_spio_vlun(2); + if( ret_flag ) + res = 2; // if sanity check fails set res to 2 + ASSERT_EQ(1, res); } TEST(Cflash_FVT_Suite, E_cmd_room_violation) @@ -887,6 +898,9 @@ TEST(Cflash_FVT_Suite, E_test_detach_diff_proc) ASSERT_EQ(0,mc_test_engine(TEST_DETACH_DIFF_PROC)); } +#ifdef _AIX + + TEST(Cflash_FVT_Suite, E_test_vlun_verify) { ASSERT_EQ(0,mc_test_engine(EXCP_VLUN_VERIFY)); @@ -897,7 +911,6 @@ TEST(Cflash_FVT_Suite, E_test_plun_verify) ASSERT_EQ(0,mc_test_engine(EXCP_PLUN_VERIFY)); } - TEST(Cflash_FVT_Suite,E_test_inval_devno) { ASSERT_EQ(0,mc_test_engine(EXCP_INVAL_DEVNO)); @@ -912,13 +925,6 @@ TEST(Cflash_FVT_Suite,E_test_inval_rschndl) ASSERT_EQ(0,mc_test_engine(EXCP_INVAL_RSCHNDL)); } -#ifndef _AIX -TEST(Cflash_FVT_Suite,G_dk_capi_clone) -{ - ASSERT_EQ(0,mc_test_engine(TEST_DK_CAPI_CLONE)); -} -#endif -#ifdef _AIX TEST(Cflash_FVT_Suite, G_MC_test_MRC_MC_VLUN) { ASSERT_EQ(0,mc_test_engine(G_ioctl_7_1_119)); @@ -1074,26 +1080,8 @@ TEST(Cflash_FVT_Suite, G_7_1_203) { ASSERT_EQ(0,mc_test_engine(G_ioctl_7_1_203)); } -#ifdef MANUAL -TEST(Cflash_FVT_Suite, M_7_5_13_1) -{ - ASSERT_EQ(0,mc_test_engine(M_TEST_7_5_13_1)); -} - -TEST(Cflash_FVT_Suite, M_7_5_13_2) -{ - ASSERT_EQ(0,mc_test_engine(M_TEST_7_5_13_2)); -} - -TEST(Cflash_FVT_Suite, E_test_fc_reset_vlun) -{ - ASSERT_EQ(0,mc_test_engine(TEST_FC_PR_RESET_VLUN)); -} -TEST(Cflash_FVT_Suite, E_test_fc_reset_plun) -{ - ASSERT_EQ(0,mc_test_engine(TEST_FC_PR_RESET_PLUN)); -} +#ifdef AIX_MANUAL TEST(Cflash_FVT_Suite, E_MC_test_DCA_EEH_Flag_Set_Reuse_CTX_On_New_Disk) { @@ -1104,6 +1092,7 @@ TEST(Cflash_FVT_Suite, G_MC_test_DCRC_EEH_of_VLUN) { ASSERT_EQ(0,mc_test_engine(TEST_DCRC_EEH_VLUN)); } + TEST(Cflash_FVT_Suite, G_MC_test_DCRC_EEH_Vlun_Resuse_Ctx) { ASSERT_EQ(0,mc_test_engine(TEST_DCRC_EEH_VLUN_RESUSE_CTX)); @@ -1123,29 +1112,70 @@ TEST(Cflash_FVT_Suite, G_MC_test_DCRC_EEH_Vlun_Release) ASSERT_EQ(0,mc_test_engine(TEST_DCRC_EEH_VLUN_RELEASE)); } -TEST(Cflash_FVT_Suite, G_MC_test_DCRC_IO_Eeh_vlun) +TEST(Cflash_FVT_Suite, G_MC_test_DCRC_IO_EEH_vlun) { ASSERT_EQ(0,mc_test_engine(TEST_DCRC_IO_EEH_VLUN)); } -TEST(Cflash_FVT_Suite, G_MC_test_DCRC_IO_Eeh_Plun) +TEST(Cflash_FVT_Suite, G_MC_test_DCRC_IO_EEH_Plun) { ASSERT_EQ(0,mc_test_engine(TEST_DCRC_IO_EEH_PLUN)); } -TEST(Cflash_FVT_Suite, G_MC_test_DCV_Unexpected_error) +TEST(Cflash_FVT_Suite, G_MC_test_DCV_Plun_Rst_flag_EEH) +{ + ASSERT_EQ(0,mc_test_engine(TEST_DCV_PLUN_RST_FlAG_EEH)); +} + +TEST(Cflash_FVT_Suite, G_MC_test_DCRC_EEH_Plun_Mutli_Vlun) +{ + ASSERT_EQ(0,mc_test_engine(TEST_DCRC_EEH_PLUN_MULTI_VLUN)); +} + +TEST(Cflash_FVT_Suite, E_test_vSpio_EEH_Recovery) +{ + ASSERT_EQ(0,mc_test_engine(TEST_VSPIO_EEHRECOVERY)); +} + +TEST(Cflash_FVT_Suite, E_test_dSpio_EEH_Recovery) +{ + ASSERT_EQ(0,mc_test_engine(TEST_DSPIO_EEHRECOVERY)); +} + +TEST(Cflash_FVT_Suite, G_MC_test_DCV_Unexpected_error_EEH) { ASSERT_EQ(0,mc_test_engine(TEST_DCV_UNEXPECTED_ERR)); } -TEST(Cflash_FVT_Suite, G_MC_test_DCV_Unexpected_error_vlun) +TEST(Cflash_FVT_Suite, G_MC_test_DCV_Unexpected_error_vlun_EEH) { ASSERT_EQ(0,mc_test_engine(TEST_DCV_UNEXPECTED_ERR_VLUN)); } -TEST(Cflash_FVT_Suite, G_MC_test_DCV_Plun_Rst_flag_EEH) + +#endif + +#ifdef MANUAL + +TEST(Cflash_FVT_Suite, M_7_5_13_1) { - ASSERT_EQ(0,mc_test_engine(TEST_DCV_PLUN_RST_FlAG_EEH)); + ASSERT_EQ(0,mc_test_engine(M_TEST_7_5_13_1)); +} + +TEST(Cflash_FVT_Suite, M_7_5_13_2) +{ + ASSERT_EQ(0,mc_test_engine(M_TEST_7_5_13_2)); +} + +TEST(Cflash_FVT_Suite, E_test_fc_reset_vlun) +{ + ASSERT_EQ(0,mc_test_engine(TEST_FC_PR_RESET_VLUN)); } +TEST(Cflash_FVT_Suite, E_test_fc_reset_plun) +{ + ASSERT_EQ(0,mc_test_engine(TEST_FC_PR_RESET_PLUN)); +} + + /*** DK_CAPI_LOG_EVENT ****/ #ifdef _AIX @@ -1179,24 +1209,14 @@ TEST(Cflash_FVT_Suite, E_test_ioctl_fcp) } #endif -TEST(Cflash_FVT_Suite, E_test_vSpio_eehRecovery) -{ - ASSERT_EQ(0,mc_test_engine(TEST_VSPIO_EEHRECOVERY)); -} - -TEST(Cflash_FVT_Suite, E_test_dSpio_eehRecovery) -{ - ASSERT_EQ(0,mc_test_engine(TEST_DSPIO_EEHRECOVERY)); -} - -TEST(Cflash_FVT_Suite, E_test_vSpio_eehRecovery_long) +TEST(Cflash_FVT_Suite, E_test_vSpio_EEH_Recovery_long) { int rc=get_fvt_dev_env(); if(rc) ASSERT_EQ(0,rc); ASSERT_EQ(0,test_vSpio_eehRecovery(2)); } -TEST(Cflash_FVT_Suite, E_test_dSpio_eehRecovery_long) +TEST(Cflash_FVT_Suite, E_test_dSpio_EEH_Recovery_long) { int rc=get_fvt_dev_env(); if(rc) @@ -1233,7 +1253,7 @@ TEST(Cflash_FVT_Suite, E_max_ctx_rcvr_last_one_no_rcvr) ASSERT_EQ(0,mc_test_engine(MAX_CTX_RCVR_LAST_ONE_NO_RCVR)); } -TEST(Cflash_FVT_Suite,E_test_eeh_simulation) +TEST(Cflash_FVT_Suite,E_test_EEH_simulation) { ASSERT_EQ(0,mc_test_engine(EXCP_EEH_SIMULATION)); } @@ -1282,10 +1302,6 @@ TEST(Cflash_FVT_Suite, E_test_cfdisk_ctxs_diff_devno) ASSERT_EQ(0,mc_test_engine(TEST_CFDISK_CTXS_DIFF_DEVNO)); } #endif -TEST(Cflash_FVT_Suite, G_MC_test_DCRC_EEH_Plun_Mutli_Vlun) -{ - ASSERT_EQ(0,mc_test_engine(TEST_DCRC_EEH_PLUN_MULTI_VLUN)); -} TEST(Cflash_FVT_Suite,E_test_vlun_uattention) { ASSERT_EQ(0,mc_test_engine(EXCP_VLUN_UATTENTION)); diff --git a/src/cflash/test/makefile b/src/cflash/test/makefile index 56a64b0e..5fa8daab 100644 --- a/src/cflash/test/makefile +++ b/src/cflash/test/makefile @@ -67,10 +67,9 @@ endif DEPS=$(addprefix $(TESTDIR)/, $(run_cflash_fvt_OFILES:.o=.dep)) -CFLAGS += \ - -g \ - -D__FVT__\ - -I$(ROOTPATH)/src/test/framework/googletest/googletest/include +CFLAGS += -g -Wno-unused-result\ + -D__FVT__\ + -I$(ROOTPATH)/${GTESTINC} CXXFLAGS+=$(CFLAGS) include ${ROOTPATH}/config.mk diff --git a/src/include/capiblock.h b/src/include/capiblock.h index 2abfec6b..71064f8a 100644 --- a/src/include/capiblock.h +++ b/src/include/capiblock.h @@ -39,8 +39,12 @@ #define NULL_CHUNK_ID -1 +#define NULL_CHUNK_CG_ID NULL_CHUNK_ID typedef int chunk_id_t; +typedef int chunk_cg_id_t; +typedef int chunk_r0_id_t; + typedef uint64_t chunk_ext_arg_t; /************************************************************************/ @@ -51,119 +55,119 @@ typedef struct chunk_stats_s { uint32_t block_size; /* Block size of this chunk. */ uint32_t num_paths; /* Number of paths of this chunk. */ uint64_t max_transfer_size; /* Maximum transfer size in */ - /* blocks of this chunk. */ + /* blocks of this chunk. */ uint64_t num_reads; /* Total number of reads issued */ - /* via cblk_read interface */ + /* via cblk_read interface */ uint64_t num_writes; /* Total number of writes issued */ - /* via cblk_write interface */ + /* via cblk_write interface */ uint64_t num_areads; /* Total number of async reads */ - /* issued via cblk_aread interface */ + /* issued via cblk_aread interface */ uint64_t num_awrites; /* Total number of async writes */ - /* issued via cblk_awrite interface*/ + /* issued via cblk_awrite interface*/ uint32_t num_act_reads; /* Current number of reads active */ - /* via cblk_read interface */ + /* via cblk_read interface */ uint32_t num_act_writes; /* Current number of writes active */ - /* via cblk_write interface */ + /* via cblk_write interface */ uint32_t num_act_areads; /* Current number of async reads */ - /* active via cblk_aread interface */ + /* active via cblk_aread interface */ uint32_t num_act_awrites; /* Current number of async writes */ - /* active via cblk_awrite interface*/ + /* active via cblk_awrite interface*/ uint32_t max_num_act_writes; /* High water mark on the maximum */ - /* number of writes active at once */ + /* number of writes active at once */ uint32_t max_num_act_reads; /* High water mark on the maximum */ - /* number of reads active at once */ + /* number of reads active at once */ uint32_t max_num_act_awrites; /* High water mark on the maximum */ - /* number of asyync writes active */ - /* at once. */ + /* number of asyync writes active */ + /* at once. */ uint32_t max_num_act_areads; /* High water mark on the maximum */ - /* number of asyync reads active */ - /* at once. */ + /* number of asyync reads active */ + /* at once. */ uint64_t num_blocks_read; /* Total number of blocks read */ uint64_t num_blocks_written; /* Total number of blocks written */ uint64_t num_errors; /* Total number of all error */ - /* responses seen */ + /* responses seen */ uint64_t num_aresult_no_cmplt; /* Number of times cblk_aresult */ - /* returned with no command */ - /* completion */ + /* returned with no command */ + /* completion */ uint64_t num_retries; /* Total number of all commmand */ - /* retries. */ + /* retries. */ uint64_t num_timeouts; /* Total number of all commmand */ - /* time-outs. */ + /* time-outs. */ uint64_t num_fail_timeouts; /* Total number of all commmand */ - /* time-outs that led to a command */ - /* failure. */ + /* time-outs that led to a command */ + /* failure. */ uint64_t num_no_cmds_free; /* Total number of times we didm't */ - /* have free command available */ + /* have free command available */ uint64_t num_no_cmd_room ; /* Total number of times we didm't */ - /* have room to issue a command to */ - /* the AFU. */ + /* have room to issue a command to */ + /* the AFU. */ uint64_t num_no_cmds_free_fail; /* Total number of times we didn't */ - /* have free command available and */ - /* failed a request because of this*/ + /* have free command available and */ + /* failed a request because of this*/ uint64_t num_fc_errors; /* Total number of all FC */ - /* error responses seen */ + /* error responses seen */ uint64_t num_port0_linkdowns; /* Total number of all link downs */ - /* seen on port 0. */ + /* seen on port 0. */ uint64_t num_port1_linkdowns; /* Total number of all link downs */ - /* seen on port 1. */ + /* seen on port 1. */ uint64_t num_port0_no_logins; /* Total number of all no logins */ - /* seen on port 0. */ + /* seen on port 0. */ uint64_t num_port1_no_logins; /* Total number of all no logins */ - /* seen on port 1. */ + /* seen on port 1. */ uint64_t num_port0_fc_errors; /* Total number of all general FC */ - /* errors seen on port 0. */ + /* errors seen on port 0. */ uint64_t num_port1_fc_errors; /* Total number of all general FC */ - /* errors seen on port 1. */ + /* errors seen on port 1. */ uint64_t num_cc_errors; /* Total number of all check */ - /* condition responses seen */ + /* condition responses seen */ uint64_t num_afu_errors; /* Total number of all AFU error */ - /* responses seen */ + /* responses seen */ uint64_t num_capi_false_reads; /* Total number of all times */ - /* poll indicated a read was ready */ - /* but there was nothing to read. */ + /* poll indicated a read was ready */ + /* but there was nothing to read. */ uint64_t num_capi_read_fails; /* Total number of all */ - /* CXL_EVENT_READ_FAIL responses */ - /* seen. */ + /* CXL_EVENT_READ_FAIL responses */ + /* seen. */ uint64_t num_capi_adap_resets; /* Total number of all adapter */ - /* reset errors. */ + /* reset errors. */ uint64_t num_capi_adap_chck_err;/* Total number of all check */ - /* adapter errors. */ + /* adapter errors. */ uint64_t num_capi_reserved_errs;/* Total number of all */ - /* CXL_EVENT_RESERVED responses */ - /* seen. */ + /* CXL_EVENT_RESERVED responses */ + /* seen. */ uint64_t num_capi_data_st_errs; /* Total number of all */ - /* CAPI data storage event */ - /* responses seen. */ + /* CAPI data storage event */ + /* responses seen. */ uint64_t num_capi_afu_errors; /* Total number of all */ - /* CAPI error responses seen */ + /* CAPI error responses seen */ uint64_t num_capi_afu_intrpts; /* Total number of all */ - /* CAPI AFU interrupts for command */ - /* responses seen. */ + /* CAPI AFU interrupts for command */ + /* responses seen. */ uint64_t num_capi_unexp_afu_intrpts; /* Total number of all of */ - /* unexpected AFU interrupts */ + /* unexpected AFU interrupts */ uint64_t num_success_threads; /* Total number of pthread_creates */ - /* that succeed. */ + /* that succeed. */ uint64_t num_failed_threads; /* Total number of pthread_creates */ - /* that failed. */ + /* that failed. */ uint64_t num_canc_threads; /* Number of threads we had to */ - /* cancel, which succeeded. */ + /* cancel, which succeeded. */ uint64_t num_fail_canc_threads; /* Number of threads we had to */ - /* cancel, but the cancel failed */ + /* cancel, but the cancel failed */ uint64_t num_fail_detach_threads;/* Number of threads we detached */ - /* but the detach failed */ + /* but the detach failed */ uint64_t num_active_threads; /* Current number of threads */ - /* running. */ + /* running. */ uint64_t max_num_act_threads; /* Maximum number of threads */ - /* running simultaneously. */ + /* running simultaneously. */ uint64_t num_cache_hits; /* Total number of cache hits */ - /* seen on all reads */ + /* seen on all reads */ uint64_t num_reset_contexts; /* Total number of reset contexts */ - /* done */ + /* done */ uint64_t num_reset_contxt_fails;/* Total number of reset context */ - /* failures */ + /* failures */ uint32_t primary_path_id; /* Primary path id */ uint64_t num_path_fail_overs; /* Total number of times a request */ - /* has failed over to another path.*/ + /* has failed over to another path.*/ } chunk_stats_t; @@ -181,7 +185,7 @@ typedef offset_t cflash_offset_t; typedef off_t cflash_offset_t; #endif /************************************************************************/ -/* Open flags */ +/* Open flags */ /************************************************************************/ #define CBLK_OPN_SCRUB_DATA CBLK_SCRUB_DATA_FLG @@ -189,19 +193,28 @@ typedef off_t cflash_offset_t; #define CBLK_OPN_VIRT_LUN 2 /* Use a virtual lun */ #define CBLK_OPN_NO_INTRP_THREADS 4 /* Do not use back threads for */ - /* handling interrupts processing */ + /* handling interrupts processing */ #define CBLK_OPN_SHARE_CTXT 8 /* Share context in same process */ #ifdef _AIX #define CBLK_OPN_RESERVE 0x10 /* Tell master context to use */ - /* reservations on this lun. */ + /* reservations on this lun. */ #define CBLK_OPN_FORCED_RESERVE 0x20 /* Tell master context to break */ - /* reservations for this lun and */ - /* establish a new reservation */ -#define CBLK_OPN_MPIO_FO 0x40 /* Use multi-path I/O fail over */ - /* this lun. */ + /* reservations for this lun and */ + /* establish a new reservation */ #endif /* _AIX */ +#define CBLK_OPN_MPIO_FO 0x40 /* Use multi-path I/O fail over */ + /* this lun. */ + +#define CBLK_OPN_GROUP 0x100 /* Use cblk_cg_open() */ + +/************************************************************************/ +/* Common flag for non-open APIs */ +/************************************************************************/ +#define CBLK_GROUP_ID 0x100 /* id passed is a chunk group id */ +#define CBLK_GROUP_RAID0 0x200 /* id passed is a RAID0 group id */ +#define CBLK_GROUP_MASK (CBLK_GROUP_ID|CBLK_GROUP_RAID0) /************************************************************************/ @@ -210,13 +223,13 @@ typedef off_t cflash_offset_t; #define CBLK_ARW_WAIT_CMD_FLAGS 1 /* Wait for commmand for cblk_aread */ - /* or cblk_awrite. */ + /* or cblk_awrite. */ #define CBLK_ARW_USER_TAG_FLAG 2 /* The caller is specifying a user */ - /* defined tag for this request. */ + /* defined tag for this request. */ #define CBLK_ARW_USER_STATUS_FLAG 4 /* The caller has set the status */ - /* parameter to the address which it */ - /* expects command completion status */ - /* to be posted. */ + /* parameter to the address which it */ + /* expects command completion status */ + /* to be posted. */ typedef enum { CBLK_ARW_STATUS_PENDING = 0, /* Command has not completed */ @@ -229,11 +242,11 @@ typedef enum { typedef struct cblk_arw_status_s { cblk_status_type_t status; /* Status of command */ /* See errno field for additional */ - /* details on failure. */ + /* details on failure. */ size_t blocks_transferred; /* Number of blocks transferred for */ - /* this request. */ + /* this request. */ int fail_errno; /* Errno when status indicates */ - /* CBLK_ARW_STAT_FAIL. */ + /* CBLK_ARW_STAT_FAIL. */ } cblk_arw_status_t; /************************************************************************/ @@ -241,49 +254,54 @@ typedef struct cblk_arw_status_s { /************************************************************************/ #define CBLK_ARESULT_NEXT_TAG 1 /* cblk_aresult will return the tag */ - /* of the next async I/O to complete */ - /* for this chunk. If this flag is not*/ - /* set then caller should have passed */ - /* the address of the tag for which */ - /* they are waiting to complete. */ + /* of the next async I/O to complete */ + /* for this chunk. If this flag is not*/ + /* set then caller should have passed */ + /* the address of the tag for which */ + /* they are waiting to complete. */ #define CBLK_ARESULT_BLOCKING 2 /* If set then cblk_aresult will block*/ - /* until the specified tag completes. */ - /* Otherwise cblk_aresult will return */ - /* immediately with a value of 1 if */ - /* the specified tag has not yet */ - /* completed */ + /* until the specified tag completes. */ + /* Otherwise cblk_aresult will return */ + /* immediately with a value of 1 if */ + /* the specified tag has not yet */ + /* completed */ #define CBLK_ARESULT_USER_TAG 4 /* If set then the tag parameter */ - /* specifies a user defined tag that */ + /* specifies a user defined tag that */ /* was provided when the cblk_aread */ /* or cblk_awrite call was issued. */ +#define CBLK_ARESULT_NO_HARVEST 8 /* If set then cblk_aresult will */ + /* not pull newly completed cmds from */ + /* the AFU, but will instead check */ + /* the completed queue. */ + /************************************************************************/ /* cblk_listio flags and structure */ /************************************************************************/ #define CBLK_LISTIO_WAIT_ISSUE_CMD 1 /* Wait for commmand for all commands */ - /* in issue_io_list. */ + /* in issue_io_list. */ typedef struct cblk_io { uint8_t version; /* Version of structure */ int flags; /* Flags for the request */ #define CBLK_IO_USER_TAG 0x0001 /* Caller is specifying a user defined*/ - /* tag. */ + /* tag. */ #define CBLK_IO_USER_STATUS 0x0002/* Caller is specifying a status */ - /* location to be updated. */ + /* location to be updated. */ #define CBLK_IO_PRIORITY_REQ 0x0004/* This is a (high) priority request */ - /* that should be expedited vs non- */ - /* priority requests. */ + /* that should be expedited vs non- */ + /* priority requests. */ uint8_t request_type; /* Type of request */ #define CBLK_IO_TYPE_READ 0x01 /* Read data request */ #define CBLK_IO_TYPE_WRITE 0x02 /* Write data request */ void *buf; /* Data buffer for request. */ cflash_offset_t lba; /* Starting logical block address for */ - /* request. */ + /* request. */ size_t nblocks; /* Size of request based on number of */ - /* blocks. */ + /* blocks. */ int tag; /* Tag for request */ cblk_arw_status_t stat; /* Status of request. */ } cblk_io_t; @@ -324,13 +342,74 @@ int cblk_aresult(chunk_id_t chunk_id,int *tag, uint64_t *status, int flags); /* CAPI flash I/O request interface */ int cblk_listio(chunk_id_t chunk_id, - cblk_io_t *issue_io_list[],int issue_items, - cblk_io_t *pending_io_list[], int pending_items, - cblk_io_t *wait_io_list[],int wait_items, - cblk_io_t *completion_io_list[],int *completion_items, - uint64_t timeout,int flags); + cblk_io_t *issue_io_list[],int issue_items, + cblk_io_t *pending_io_list[], int pending_items, + cblk_io_t *wait_io_list[],int wait_items, + cblk_io_t *completion_io_list[],int *completion_items, + uint64_t timeout,int flags); /* Clone a chunk (such as a parent and chilld process' chunk */ int cblk_clone_after_fork(chunk_id_t chunk_id, int mode, int flags); + +typedef struct cflsh_cg_tag_s +{ + chunk_id_t id; + int tag; +} cflsh_cg_tag_t; + +chunk_cg_id_t cblk_cg_open(const char *path, + int max_num_requests, + int mode, + int num_chunks, + chunk_ext_arg_t ext, + int flags); +int cblk_cg_close(chunk_cg_id_t cgid, + int flags); +int cblk_cg_get_stats(chunk_cg_id_t cgid, + chunk_stats_t *stats, + int flags); +int cblk_cg_get_lun_size(chunk_cg_id_t cgid, + size_t *nblocks, + int flags); +int cblk_cg_get_size(chunk_cg_id_t cgid, + size_t *nblocks, + int flags); +int cblk_cg_set_size(chunk_cg_id_t cgid, + size_t nblocks, + int flags); +int cblk_cg_read(chunk_cg_id_t cgid, + void *pbuf, + cflash_offset_t lba, + size_t nblocks, + int flags); +int cblk_cg_write(chunk_cg_id_t cgid, + void *pbuf, + cflash_offset_t lba, + size_t nblocks, + int flags); +int cblk_cg_aread(chunk_cg_id_t cgid, + void *pbuf, + cflash_offset_t lba, + size_t nblocks, + cflsh_cg_tag_t *ptag, + cblk_arw_status_t *p_arwstatus, + int flags); +int cblk_cg_awrite(chunk_cg_id_t cgid, + void *pbuf, + cflash_offset_t lba, + size_t nblocks, + cflsh_cg_tag_t *ptag, + cblk_arw_status_t *p_arwstatus, + int flags); +int cblk_cg_aresult(chunk_cg_id_t cgid, + cflsh_cg_tag_t *ptag, + uint64_t *p_arwstatus, + int flags); +int cblk_cg_clone_after_fork(chunk_cg_id_t cgid, + int mode, + int flags); +int cblk_cg_get_num_chunks(chunk_cg_id_t cgid, + int flags); + #endif /* _H_CFLASH_BLOCK */ diff --git a/src/include/cflash_eras.h b/src/include/cflash_eras.h index ce9eac9d..2a502edd 100644 --- a/src/include/cflash_eras.h +++ b/src/include/cflash_eras.h @@ -40,7 +40,7 @@ typedef uint32_t eye_catch4b_t; #define __EYEC2(__a,__b) (((__a)<< 8) | (__b)) #define __EYEC4(__a,__b,__c,__d) ((__EYEC2(__a,__b) << 16) | __EYEC2(__c,__d)) - +#define __EYEC8(__a,__b,__c,__d,__e,__f,__g,__h) (((unsigned long long)__EYEC4(__a,__b,__c,__d) << 32) | __EYEC4(__e,__f,__g,__h)) #endif /* _H_CFLASH_ERAS_H */ diff --git a/src/include/cflash_scsi_user.h b/src/include/cflash_scsi_user.h index 86174e1b..350c6edf 100644 --- a/src/include/cflash_scsi_user.h +++ b/src/include/cflash_scsi_user.h @@ -271,6 +271,58 @@ struct inqry_data { }; +/* + * Standard SCSI INQUIRY page 83 data format + * +=====-=======-=======-=======-=======-=======-=======-=======-=======+ + * | Bit| 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | + * |Byte | | | | | | | | | + * |=====+=======================+=======================================| + * | 0 | Peripheral qualifier | Peripheral device type | + * |-----+---------------------------------------------------------------| + * | 1 | Page code 0x83 | + * |-----+---------------------------------------------------------------| + * | 2 | (MSB) | + * |- - -+--- Page Length (n-3) ---| + * | 3 | (LSB) | + * |-----+---------------------------------------------------------------| + * | 4 | (MSB) | + * |- - -+--- Descriptor List ---| + * | n | (LSB) | + * +=====================================================================+ + */ +struct inqry_pg83_data { + uint8_t pdevtype; /* Peripherial device/qualifier */ + + uint8_t page_code; + uint16_t page_len; /* Page Length */ +}; + +/* + * Standard SCSI INQUIRY page 83 descriptor format + * +=====-=======-=======-=======-=======-=======-=======-=======-=======+ + * | Bit| 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | + * |Byte | | | | | | | | | + * |=====+=======================+=======================================| + * | 0 | Protocol Identifier | Code Set | + * |-----+---------------------------------------------------------------| + * | 1 | PIV |Reservd|Assoc | Designator Type | + * |-----+---------------------------------------------------------------| + * | 2 | Reserverd | + * |-----+---------------------------------------------------------------| + * | 3 | Designator Length (n-3) | + * |-----+---------------------------------------------------------------| + * | 4 | (MSB) | + * |- - -+--- Designator List ---| + * | n | (LSB) | + * +=====================================================================+ + */ +struct inqry_pg83_descriptor_hdr { + uint8_t protocol_id_code_set; /* Protocol ID and Code set */ + uint8_t flags_type; /* Flags and designator type*/ + uint8_t reserved; /* Reserved for future use */ + uint8_t designator_length; /* Length of designator */ +}; + /************************************************************************/ /* Device ID Code Page defines used for extracting WWID */ diff --git a/src/include/cflash_sisl.h b/src/include/cflash_sisl.h index a8f95098..2ba02b8e 100755 --- a/src/include/cflash_sisl.h +++ b/src/include/cflash_sisl.h @@ -163,7 +163,12 @@ typedef struct sisl_ioarcb_s { __u16 timeout; /* in units specified by req_flags */ __u32 rsvd1; __u8 cdb[16]; /* must be in big endian */ - __u64 rsvd2; + union { + __u64 sq_ioasa_ea; /* When using Submission queues (SQ),*/ + /* this is the effective address of */ + /* ths associated IOASA. */ + __u64 rsvd2; + }; } sisl_ioarcb_t; @@ -347,6 +352,12 @@ __u64 intr_status; /* this sends LISN# programmed in ctx_ctrl. __u64 cmd_room; __u64 ctx_ctrl; /* least signiifcant byte or b56:63 is LISN# */ __u64 mbox_w; /* restricted use */ + __u64 sq_start; /* Submission queue (SQ) start register */ + __u64 sq_end; /* Submission queue (SQ) end register */ + __u64 sq_head; /* Submission queue (SQ) head register */ + __u64 sq_tail; /* Submission queue (SQ) tail register */ + __u64 sq_context_reset; /* Submission queue (SQ) context reset + register. */ }; /* per context provisioning & control MMIO */ diff --git a/src/include/cflash_tools_user.h b/src/include/cflash_tools_user.h index 7146e668..28094608 100644 --- a/src/include/cflash_tools_user.h +++ b/src/include/cflash_tools_user.h @@ -32,6 +32,8 @@ #define CFLASH_LITTLE_ENDIAN_HOST 1 #elif defined TARGET_ARCH_PPC64BE #define CFLASH_BIG_ENDIAN_HOST 1 +#elif defined TARGET_ARCH_x86_64 + #define CFLASH_x86_64_HOST 1 #else #error "Unknown Architecture. This typically indicates a makefile or environment variable error. Check the TARGET_ARCH env var." #endif diff --git a/src/include/cflsh_usfs.h b/src/include/cflsh_usfs.h new file mode 100644 index 00000000..2d1403ae --- /dev/null +++ b/src/include/cflsh_usfs.h @@ -0,0 +1,125 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* bos720 src/bos/usr/ccs/lib/libcflsh_block/cflsh_ufs.h 1.6 */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/* %Z%%M% %I% %W% %G% %U% */ + + +/* + * COMPONENT_NAME: (sysxcflashufs) CAPI Flash user space file system library + * + * FUNCTIONS: API for users of this library. + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +#ifndef _H_CFLASH_USFS +#define _H_CFLASH_USFS + + +#include +#ifdef _AIX +#include +#endif /* AIX */ + + +#define CUSFS_DISK_NAME_DELIMINATOR ":" +/* + * Typical CAPI Flash User Space File System + * API. + */ +int cusfs_init(void *arg,uint64_t flags); +int cusfs_term(void *arg,uint64_t flags); +int cusfs_creat(char *path, mode_t mode); +int cusfs_open(char *path, int flags, mode_t mode); +int cusfs_close(int fd); +ssize_t cusfs_read(int fd, void *buffer,size_t nbytes); +ssize_t cusfs_write(int fd, void *buffer,size_t nbytes); +int cusfs_fsync(int fd); +off_t cusfs_lseek(int fd, off_t Offset, int Whence); +off64_t cusfs_lseek64(int fd, off64_t Offset, int Whence); +#ifdef _AIX +offset_t cusfs_llseek(int fd, offset_t Offset ,int Whence); +#endif /* _AIX */ + +int cusfs_fstat(int fd,struct stat *Buffer); +off_t cusfs_lseek(int fd, off_t Offset, int Whence); + +int cusfs_fstat64(int fd,struct stat64 *Buffer); + +#ifdef _AIX +int cusfs_fstat64x(int fd,struct stat64x *Buffer); +int cusfs_stat64x(char *path,struct stat64x *Buffer); +int cusfs_lstat64x(char *path,struct stat64x *Buffer); +#endif /* _AIX */ + +int cusfs_stat64(char *path,struct stat64 *Buffer); +int cusfs_lstat64(char *path,struct stat64 *Buffer); + +int cusfs_fstat(int fd,struct stat *Buffer); +int cusfs_stat(char *path,struct stat *Buffer); +int cusfs_lstat(char *path,struct stat *Buffer); + +int cusfs_readlink(const char *path, char *buffer, size_t buffer_size); +int cusfs_access(char *path, int mode); +int cusfs_fchmod(int fd, mode_t mode); +int cusfs_fchown(int fd, uid_t uid, gid_t group); +int cusfs_ftruncate(int fd, off_t length); +int cusfs_ftruncate64(int fd, off64_t length); +int cusfs_posix_fadvise(int fd, off_t offset, off_t length, int advise); +#ifndef _AIX +int cusfs_posix_fadvise64(int fd, off64_t offset, off64_t length, int advise); +#endif /* !_AIX */ + +int cusfs_link(char *path1, char *path2); +int cusfs_unlink(char *path); +int cusfs_utime(char *path,struct utimbuf *times); +#ifdef _NOT_YET +int cusfs_futimens(int fd,struct timespec *times) +#endif +int cusfs_aio_read64(struct aiocb64 *aiocbp); +int cusfs_aio_read(struct aiocb *aiocbp); +int cusfs_aio_write64(struct aiocb64 *aiocbp); +int cusfs_aio_write(struct aiocb *aiocbp); +int cusfs_aio_error64(struct aiocb64 *aiocbp); +int cusfs_aio_error(struct aiocb *aiocbp); +int cusfs_aio_return64(struct aiocb64 *aiocbp); +int cusfs_aio_return(struct aiocb *aiocbp); +int cusfs_aio_fsync64(int op,struct aiocb64 *aiocbp); +int cusfs_aio_fsync(int op,struct aiocb *aiocbp); +int cusfs_aio_cancel64(int fd, struct aiocb64 *aiocbp); +int cusfs_aio_cancel(int fd, struct aiocb *aiocbp); +int cusfs_aio_suspend64(const struct aiocb64 *const list[], int nent, + const struct timespec *timeout); +int cusfs_aio_suspend(const struct aiocb *const list[], int nent, + const struct timespec *timeout); +#endif /* _H_CFLASH_USFS */ diff --git a/src/include/cflsh_usfs_admin.h b/src/include/cflsh_usfs_admin.h new file mode 100644 index 00000000..61276e73 --- /dev/null +++ b/src/include/cflsh_usfs_admin.h @@ -0,0 +1,143 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* bos720 src/bos/usr/ccs/lib/libcflsh_block/cflsh_ufs.h 1.6 */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/* %Z%%M% %I% %W% %G% %U% */ + + +/* + * COMPONENT_NAME: (sysxcflashufs) CAPI Flash user space file system library + * + * FUNCTIONS: API for users of this library. + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +#ifndef _H_CFLASH_USFS_ADMIN +#define _H_CFLASH_USFS_ADMIN + +#ifndef _AIX +#define _GNU_SOURCE +#endif /* !_AIX */ + +#include +#include +#include + +/************************************************************************/ +/* cufs_create_fs flags */ +/************************************************************************/ + +#define CUSFS_FORCE_CREATE_FS 0x00000001 + + + +struct cusfs_query_fs { + int version; + int flags; +#define CUSFS_QRY_DIF_ENDIAN 0x1 /* Filesystem has different */ + /* endianess that this host */ + int os_type; + int reserved; + uint64_t fs_block_size; + uint64_t disk_block_size; + uint64_t num_blocks; + uint64_t free_block_table_size; + uint64_t inode_table_size; + time_t create_time; + time_t write_time; + time_t mount_time; + time_t fsck_time; +}; + +/* + * Administrative CAPI Flash User Space File System + * commands. + */ + + +int cusfs_create_fs(char *device_name, int flags); +int cusfs_query_fs(char *device_name, struct cusfs_query_fs *qry, int flags); +int cusfs_statfs(char *pathname, struct statfs *statfs); + +int cusfs_statfs64(char *pathname, struct statfs64 *statfs); + +int cusfs_remove_fs(char *device_name, int flags); +int cusfs_fsck(char *device_name, int flags); +int cusfs_mkdir(char *pathname, mode_t mode_flags); +int cusfs_rmdir(char *pathname); + +#ifdef _AIX +DIR64 *cusfs_opendir64( char *directory_name); + +int cusfs_closedir64(DIR64 *directory_pointer); + +int cusfs_readdir64_r(DIR64 *directory_pointer,struct dirent64 *entry, + struct dirent64 ** result); + +struct dirent64 *cusfs_readdir64(DIR64 *directory_pointer); +offset_t cusfs_telldir64(DIR64 *directory_pointer); + +void cusfs_seekdir64(DIR64 *directory_pointer, offset_t location); +#else +DIR *cusfs_opendir( char *directory_name); + +int cusfs_closedir(DIR *directory_pointer); + +int cusfs_readdir_r(DIR *directory_pointer,struct dirent *entry, + struct dirent ** result); + +off_t cusfs_telldir(DIR *directory_pointer); + +void cusfs_seekdir(DIR *directory_pointer, off_t location); + +#endif /* !_AIX */ + +struct dirent *cusfs_readdir(DIR *directory_pointer); + + + +//cflsh_ufs_data_obj_t *cufs_create_file(cflsh_ufs_t *cufs, char *full_pathname, +// mode_t mode_flags, uid_t uid, gid_t gid, int flags); + +#define CFLSH_USFS_LINK_SYM_FLAG 0x1 + +int cusfs_create_link(char *orig_path, char *path, + mode_t mode_flags, uid_t uid, gid_t gid, + int flags); +int cusfs_remove_file(char *pathname, int flags); + +int cusfs_list_files(char *pathname, int flags); +int cusfs_rename(char *from_pathname, + char *to_pathname); + +#endif /* _H_CFLSH_USFS_ADNIN */ diff --git a/src/include/fuse.h b/src/include/fuse.h new file mode 100644 index 00000000..b6f04216 --- /dev/null +++ b/src/include/fuse.h @@ -0,0 +1,1083 @@ +// IBM_PROLOG_BEGIN_TAG +// This is an automatically generated prolog. +// +// $Source: src/include/fuse.h $ +// +// IBM CONFIDENTIAL +// +// COPYRIGHT International Business Machines Corp. 2017 +// +// p1 +// +// Object Code Only (OCO) source materials +// Licensed Internal Code Source Materials +// IBM Surelock Licensed Internal Code +// +// The source code for this program is not published or other- +// wise divested of its trade secrets, irrespective of what has +// been deposited with the U.S. Copyright Office. +// +// Origin: 30 +// +// IBM_PROLOG_END +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2007 Miklos Szeredi + + This program can be distributed under the terms of the GNU LGPLv2. + See the file COPYING.LIB. +*/ + +#ifndef _FUSE_H_ +#define _FUSE_H_ + +/** @file + * + * This file defines the library interface of FUSE + * + * IMPORTANT: you should define FUSE_USE_VERSION before including this + * header. To use the newest API define it to 26 (recommended for any + * new application), to use the old API define it to 21 (default) 22 + * or 25, to use the even older 1.X API define it to 11. + */ + +#ifndef FUSE_USE_VERSION +#define FUSE_USE_VERSION 21 +#endif + +#include "fuse_common.h" + +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* ----------------------------------------------------------- * + * Basic FUSE API * + * ----------------------------------------------------------- */ + +/** Handle for a FUSE filesystem */ +struct fuse; + +/** Structure containing a raw command */ +struct fuse_cmd; + +/** Function to add an entry in a readdir() operation + * + * @param buf the buffer passed to the readdir() operation + * @param name the file name of the directory entry + * @param stat file attributes, can be NULL + * @param off offset of the next entry or zero + * @return 1 if buffer is full, zero otherwise + */ +typedef int (*fuse_fill_dir_t) (void *buf, const char *name, + const struct stat *stbuf, off_t off); + +/* Used by deprecated getdir() method */ +typedef struct fuse_dirhandle *fuse_dirh_t; +typedef int (*fuse_dirfil_t) (fuse_dirh_t h, const char *name, int type, + ino_t ino); + +/** + * The file system operations: + * + * Most of these should work very similarly to the well known UNIX + * file system operations. A major exception is that instead of + * returning an error in 'errno', the operation should return the + * negated error value (-errno) directly. + * + * All methods are optional, but some are essential for a useful + * filesystem (e.g. getattr). Open, flush, release, fsync, opendir, + * releasedir, fsyncdir, access, create, ftruncate, fgetattr, lock, + * init and destroy are special purpose methods, without which a full + * featured filesystem can still be implemented. + * + * Almost all operations take a path which can be of any length. + * + * Changed in fuse 2.8.0 (regardless of API version) + * Previously, paths were limited to a length of PATH_MAX. + * + * See http://fuse.sourceforge.net/wiki/ for more information. There + * is also a snapshot of the relevant wiki pages in the doc/ folder. + */ +struct fuse_operations { + /** Get file attributes. + * + * Similar to stat(). The 'st_dev' and 'st_blksize' fields are + * ignored. The 'st_ino' field is ignored except if the 'use_ino' + * mount option is given. + */ + int (*getattr) (const char *, struct stat *); + + /** Read the target of a symbolic link + * + * The buffer should be filled with a null terminated string. The + * buffer size argument includes the space for the terminating + * null character. If the linkname is too long to fit in the + * buffer, it should be truncated. The return value should be 0 + * for success. + */ + int (*readlink) (const char *, char *, size_t); + + /* Deprecated, use readdir() instead */ + int (*getdir) (const char *, fuse_dirh_t, fuse_dirfil_t); + + /** Create a file node + * + * This is called for creation of all non-directory, non-symlink + * nodes. If the filesystem defines a create() method, then for + * regular files that will be called instead. + */ + int (*mknod) (const char *, mode_t, dev_t); + + /** Create a directory + * + * Note that the mode argument may not have the type specification + * bits set, i.e. S_ISDIR(mode) can be false. To obtain the + * correct directory type bits use mode|S_IFDIR + * */ + int (*mkdir) (const char *, mode_t); + + /** Remove a file */ + int (*unlink) (const char *); + + /** Remove a directory */ + int (*rmdir) (const char *); + + /** Create a symbolic link */ + int (*symlink) (const char *, const char *); + + /** Rename a file */ + int (*rename) (const char *, const char *); + + /** Create a hard link to a file */ + int (*link) (const char *, const char *); + + /** Change the permission bits of a file */ + int (*chmod) (const char *, mode_t); + + /** Change the owner and group of a file */ + int (*chown) (const char *, uid_t, gid_t); + + /** Change the size of a file */ + int (*truncate) (const char *, off_t); + + /** Change the access and/or modification times of a file + * + * Deprecated, use utimens() instead. + */ + int (*utime) (const char *, struct utimbuf *); + + /** File open operation + * + * No creation (O_CREAT, O_EXCL) and by default also no + * truncation (O_TRUNC) flags will be passed to open(). If an + * application specifies O_TRUNC, fuse first calls truncate() + * and then open(). Only if 'atomic_o_trunc' has been + * specified and kernel version is 2.6.24 or later, O_TRUNC is + * passed on to open. + * + * Unless the 'default_permissions' mount option is given, + * open should check if the operation is permitted for the + * given flags. Optionally open may also return an arbitrary + * filehandle in the fuse_file_info structure, which will be + * passed to all file operations. + * + * Changed in version 2.2 + */ + int (*open) (const char *, struct fuse_file_info *); + + /** Read data from an open file + * + * Read should return exactly the number of bytes requested except + * on EOF or error, otherwise the rest of the data will be + * substituted with zeroes. An exception to this is when the + * 'direct_io' mount option is specified, in which case the return + * value of the read system call will reflect the return value of + * this operation. + * + * Changed in version 2.2 + */ + int (*read) (const char *, char *, size_t, off_t, + struct fuse_file_info *); + + /** Write data to an open file + * + * Write should return exactly the number of bytes requested + * except on error. An exception to this is when the 'direct_io' + * mount option is specified (see read operation). + * + * Changed in version 2.2 + */ + int (*write) (const char *, const char *, size_t, off_t, + struct fuse_file_info *); + + /** Get file system statistics + * + * The 'f_frsize', 'f_favail', 'f_fsid' and 'f_flag' fields are ignored + * + * Replaced 'struct statfs' parameter with 'struct statvfs' in + * version 2.5 + */ + int (*statfs) (const char *, struct statvfs *); + + /** Possibly flush cached data + * + * BIG NOTE: This is not equivalent to fsync(). It's not a + * request to sync dirty data. + * + * Flush is called on each close() of a file descriptor. So if a + * filesystem wants to return write errors in close() and the file + * has cached dirty data, this is a good place to write back data + * and return any errors. Since many applications ignore close() + * errors this is not always useful. + * + * NOTE: The flush() method may be called more than once for each + * open(). This happens if more than one file descriptor refers + * to an opened file due to dup(), dup2() or fork() calls. It is + * not possible to determine if a flush is final, so each flush + * should be treated equally. Multiple write-flush sequences are + * relatively rare, so this shouldn't be a problem. + * + * Filesystems shouldn't assume that flush will always be called + * after some writes, or that if will be called at all. + * + * Changed in version 2.2 + */ + int (*flush) (const char *, struct fuse_file_info *); + + /** Release an open file + * + * Release is called when there are no more references to an open + * file: all file descriptors are closed and all memory mappings + * are unmapped. + * + * For every open() call there will be exactly one release() call + * with the same flags and file descriptor. It is possible to + * have a file opened more than once, in which case only the last + * release will mean, that no more reads/writes will happen on the + * file. The return value of release is ignored. + * + * Changed in version 2.2 + */ + int (*release) (const char *, struct fuse_file_info *); + + /** Synchronize file contents + * + * If the datasync parameter is non-zero, then only the user data + * should be flushed, not the meta data. + * + * Changed in version 2.2 + */ + int (*fsync) (const char *, int, struct fuse_file_info *); + + /** Set extended attributes */ + int (*setxattr) (const char *, const char *, const char *, size_t, int); + + /** Get extended attributes */ + int (*getxattr) (const char *, const char *, char *, size_t); + + /** List extended attributes */ + int (*listxattr) (const char *, char *, size_t); + + /** Remove extended attributes */ + int (*removexattr) (const char *, const char *); + + /** Open directory + * + * Unless the 'default_permissions' mount option is given, + * this method should check if opendir is permitted for this + * directory. Optionally opendir may also return an arbitrary + * filehandle in the fuse_file_info structure, which will be + * passed to readdir, closedir and fsyncdir. + * + * Introduced in version 2.3 + */ + int (*opendir) (const char *, struct fuse_file_info *); + + /** Read directory + * + * This supersedes the old getdir() interface. New applications + * should use this. + * + * The filesystem may choose between two modes of operation: + * + * 1) The readdir implementation ignores the offset parameter, and + * passes zero to the filler function's offset. The filler + * function will not return '1' (unless an error happens), so the + * whole directory is read in a single readdir operation. This + * works just like the old getdir() method. + * + * 2) The readdir implementation keeps track of the offsets of the + * directory entries. It uses the offset parameter and always + * passes non-zero offset to the filler function. When the buffer + * is full (or an error happens) the filler function will return + * '1'. + * + * Introduced in version 2.3 + */ + int (*readdir) (const char *, void *, fuse_fill_dir_t, off_t, + struct fuse_file_info *); + + /** Release directory + * + * Introduced in version 2.3 + */ + int (*releasedir) (const char *, struct fuse_file_info *); + + /** Synchronize directory contents + * + * If the datasync parameter is non-zero, then only the user data + * should be flushed, not the meta data + * + * Introduced in version 2.3 + */ + int (*fsyncdir) (const char *, int, struct fuse_file_info *); + + /** + * Initialize filesystem + * + * The return value will passed in the private_data field of + * fuse_context to all file operations and as a parameter to the + * destroy() method. + * + * Introduced in version 2.3 + * Changed in version 2.6 + */ + void *(*init) (struct fuse_conn_info *conn); + + /** + * Clean up filesystem + * + * Called on filesystem exit. + * + * Introduced in version 2.3 + */ + void (*destroy) (void *); + + /** + * Check file access permissions + * + * This will be called for the access() system call. If the + * 'default_permissions' mount option is given, this method is not + * called. + * + * This method is not called under Linux kernel versions 2.4.x + * + * Introduced in version 2.5 + */ + int (*access) (const char *, int); + + /** + * Create and open a file + * + * If the file does not exist, first create it with the specified + * mode, and then open it. + * + * If this method is not implemented or under Linux kernel + * versions earlier than 2.6.15, the mknod() and open() methods + * will be called instead. + * + * Introduced in version 2.5 + */ + int (*create) (const char *, mode_t, struct fuse_file_info *); + + /** + * Change the size of an open file + * + * This method is called instead of the truncate() method if the + * truncation was invoked from an ftruncate() system call. + * + * If this method is not implemented or under Linux kernel + * versions earlier than 2.6.15, the truncate() method will be + * called instead. + * + * Introduced in version 2.5 + */ + int (*ftruncate) (const char *, off_t, struct fuse_file_info *); + + /** + * Get attributes from an open file + * + * This method is called instead of the getattr() method if the + * file information is available. + * + * Currently this is only called after the create() method if that + * is implemented (see above). Later it may be called for + * invocations of fstat() too. + * + * Introduced in version 2.5 + */ + int (*fgetattr) (const char *, struct stat *, struct fuse_file_info *); + + /** + * Perform POSIX file locking operation + * + * The cmd argument will be either F_GETLK, F_SETLK or F_SETLKW. + * + * For the meaning of fields in 'struct flock' see the man page + * for fcntl(2). The l_whence field will always be set to + * SEEK_SET. + * + * For checking lock ownership, the 'fuse_file_info->owner' + * argument must be used. + * + * For F_GETLK operation, the library will first check currently + * held locks, and if a conflicting lock is found it will return + * information without calling this method. This ensures, that + * for local locks the l_pid field is correctly filled in. The + * results may not be accurate in case of race conditions and in + * the presence of hard links, but it's unlikely that an + * application would rely on accurate GETLK results in these + * cases. If a conflicting lock is not found, this method will be + * called, and the filesystem may fill out l_pid by a meaningful + * value, or it may leave this field zero. + * + * For F_SETLK and F_SETLKW the l_pid field will be set to the pid + * of the process performing the locking operation. + * + * Note: if this method is not implemented, the kernel will still + * allow file locking to work locally. Hence it is only + * interesting for network filesystems and similar. + * + * Introduced in version 2.6 + */ + int (*lock) (const char *, struct fuse_file_info *, int cmd, + struct flock *); + + /** + * Change the access and modification times of a file with + * nanosecond resolution + * + * This supersedes the old utime() interface. New applications + * should use this. + * + * See the utimensat(2) man page for details. + * + * Introduced in version 2.6 + */ + int (*utimens) (const char *, const struct timespec tv[2]); + + /** + * Map block index within file to block index within device + * + * Note: This makes sense only for block device backed filesystems + * mounted with the 'blkdev' option + * + * Introduced in version 2.6 + */ + int (*bmap) (const char *, size_t blocksize, uint64_t *idx); + + /** + * Flag indicating that the filesystem can accept a NULL path + * as the first argument for the following operations: + * + * read, write, flush, release, fsync, readdir, releasedir, + * fsyncdir, ftruncate, fgetattr, lock, ioctl and poll + * + * If this flag is set these operations continue to work on + * unlinked files even if "-ohard_remove" option was specified. + */ + unsigned int flag_nullpath_ok:1; + + /** + * Flag indicating that the path need not be calculated for + * the following operations: + * + * read, write, flush, release, fsync, readdir, releasedir, + * fsyncdir, ftruncate, fgetattr, lock, ioctl and poll + * + * Closely related to flag_nullpath_ok, but if this flag is + * set then the path will not be calculaged even if the file + * wasn't unlinked. However the path can still be non-NULL if + * it needs to be calculated for some other reason. + */ + unsigned int flag_nopath:1; + + /** + * Flag indicating that the filesystem accepts special + * UTIME_NOW and UTIME_OMIT values in its utimens operation. + */ + unsigned int flag_utime_omit_ok:1; + + /** + * Reserved flags, don't set + */ + unsigned int flag_reserved:29; + + /** + * Ioctl + * + * flags will have FUSE_IOCTL_COMPAT set for 32bit ioctls in + * 64bit environment. The size and direction of data is + * determined by _IOC_*() decoding of cmd. For _IOC_NONE, + * data will be NULL, for _IOC_WRITE data is out area, for + * _IOC_READ in area and if both are set in/out area. In all + * non-NULL cases, the area is of _IOC_SIZE(cmd) bytes. + * + * If flags has FUSE_IOCTL_DIR then the fuse_file_info refers to a + * directory file handle. + * + * Introduced in version 2.8 + */ + int (*ioctl) (const char *, int cmd, void *arg, + struct fuse_file_info *, unsigned int flags, void *data); + + /** + * Poll for IO readiness events + * + * Note: If ph is non-NULL, the client should notify + * when IO readiness events occur by calling + * fuse_notify_poll() with the specified ph. + * + * Regardless of the number of times poll with a non-NULL ph + * is received, single notification is enough to clear all. + * Notifying more times incurs overhead but doesn't harm + * correctness. + * + * The callee is responsible for destroying ph with + * fuse_pollhandle_destroy() when no longer in use. + * + * Introduced in version 2.8 + */ + int (*poll) (const char *, struct fuse_file_info *, + struct fuse_pollhandle *ph, unsigned *reventsp); + + /** Write contents of buffer to an open file + * + * Similar to the write() method, but data is supplied in a + * generic buffer. Use fuse_buf_copy() to transfer data to + * the destination. + * + * Introduced in version 2.9 + */ + int (*write_buf) (const char *, struct fuse_bufvec *buf, off_t off, + struct fuse_file_info *); + + /** Store data from an open file in a buffer + * + * Similar to the read() method, but data is stored and + * returned in a generic buffer. + * + * No actual copying of data has to take place, the source + * file descriptor may simply be stored in the buffer for + * later data transfer. + * + * The buffer must be allocated dynamically and stored at the + * location pointed to by bufp. If the buffer contains memory + * regions, they too must be allocated using malloc(). The + * allocated memory will be freed by the caller. + * + * Introduced in version 2.9 + */ + int (*read_buf) (const char *, struct fuse_bufvec **bufp, + size_t size, off_t off, struct fuse_file_info *); + /** + * Perform BSD file locking operation + * + * The op argument will be either LOCK_SH, LOCK_EX or LOCK_UN + * + * Nonblocking requests will be indicated by ORing LOCK_NB to + * the above operations + * + * For more information see the flock(2) manual page. + * + * Additionally fi->owner will be set to a value unique to + * this open file. This same value will be supplied to + * ->release() when the file is released. + * + * Note: if this method is not implemented, the kernel will still + * allow file locking to work locally. Hence it is only + * interesting for network filesystems and similar. + * + * Introduced in version 2.9 + */ + int (*flock) (const char *, struct fuse_file_info *, int op); + + /** + * Allocates space for an open file + * + * This function ensures that required space is allocated for specified + * file. If this function returns success then any subsequent write + * request to specified range is guaranteed not to fail because of lack + * of space on the file system media. + * + * Introduced in version 2.9.1 + */ + int (*fallocate) (const char *, int, off_t, off_t, + struct fuse_file_info *); +}; + +/** Extra context that may be needed by some filesystems + * + * The uid, gid and pid fields are not filled in case of a writepage + * operation. + */ +struct fuse_context { + /** Pointer to the fuse object */ + struct fuse *fuse; + + /** User ID of the calling process */ + uid_t uid; + + /** Group ID of the calling process */ + gid_t gid; + + /** Thread ID of the calling process */ + pid_t pid; + + /** Private filesystem data */ + void *private_data; + + /** Umask of the calling process (introduced in version 2.8) */ + mode_t umask; +}; + +/** + * Main function of FUSE. + * + * This is for the lazy. This is all that has to be called from the + * main() function. + * + * This function does the following: + * - parses command line options (-d -s and -h) + * - passes relevant mount options to the fuse_mount() + * - installs signal handlers for INT, HUP, TERM and PIPE + * - registers an exit handler to unmount the filesystem on program exit + * - creates a fuse handle + * - registers the operations + * - calls either the single-threaded or the multi-threaded event loop + * + * Note: this is currently implemented as a macro. + * + * @param argc the argument counter passed to the main() function + * @param argv the argument vector passed to the main() function + * @param op the file system operation + * @param user_data user data supplied in the context during the init() method + * @return 0 on success, nonzero on failure + */ +/* + int fuse_main(int argc, char *argv[], const struct fuse_operations *op, + void *user_data); +*/ +#define fuse_main(argc, argv, op, user_data) \ + fuse_main_real(argc, argv, op, sizeof(*(op)), user_data) + +/* ----------------------------------------------------------- * + * More detailed API * + * ----------------------------------------------------------- */ + +/** + * Create a new FUSE filesystem. + * + * @param ch the communication channel + * @param args argument vector + * @param op the filesystem operations + * @param op_size the size of the fuse_operations structure + * @param user_data user data supplied in the context during the init() method + * @return the created FUSE handle + */ +struct fuse *fuse_new(struct fuse_chan *ch, struct fuse_args *args, + const struct fuse_operations *op, size_t op_size, + void *user_data); + +/** + * Destroy the FUSE handle. + * + * The communication channel attached to the handle is also destroyed. + * + * NOTE: This function does not unmount the filesystem. If this is + * needed, call fuse_unmount() before calling this function. + * + * @param f the FUSE handle + */ +void fuse_destroy(struct fuse *f); + +/** + * FUSE event loop. + * + * Requests from the kernel are processed, and the appropriate + * operations are called. + * + * @param f the FUSE handle + * @return 0 if no error occurred, -1 otherwise + */ +int fuse_loop(struct fuse *f); + +/** + * Exit from event loop + * + * @param f the FUSE handle + */ +void fuse_exit(struct fuse *f); + +/** + * FUSE event loop with multiple threads + * + * Requests from the kernel are processed, and the appropriate + * operations are called. Request are processed in parallel by + * distributing them between multiple threads. + * + * Calling this function requires the pthreads library to be linked to + * the application. + * + * @param f the FUSE handle + * @return 0 if no error occurred, -1 otherwise + */ +int fuse_loop_mt(struct fuse *f); + +/** + * Get the current context + * + * The context is only valid for the duration of a filesystem + * operation, and thus must not be stored and used later. + * + * @return the context + */ +struct fuse_context *fuse_get_context(void); + +/** + * Get the current supplementary group IDs for the current request + * + * Similar to the getgroups(2) system call, except the return value is + * always the total number of group IDs, even if it is larger than the + * specified size. + * + * The current fuse kernel module in linux (as of 2.6.30) doesn't pass + * the group list to userspace, hence this function needs to parse + * "/proc/$TID/task/$TID/status" to get the group IDs. + * + * This feature may not be supported on all operating systems. In + * such a case this function will return -ENOSYS. + * + * @param size size of given array + * @param list array of group IDs to be filled in + * @return the total number of supplementary group IDs or -errno on failure + */ +int fuse_getgroups(int size, gid_t list[]); + +/** + * Check if the current request has already been interrupted + * + * @return 1 if the request has been interrupted, 0 otherwise + */ +int fuse_interrupted(void); + +/** + * Obsolete, doesn't do anything + * + * @return -EINVAL + */ +int fuse_invalidate(struct fuse *f, const char *path); + +/* Deprecated, don't use */ +int fuse_is_lib_option(const char *opt); + +/** + * The real main function + * + * Do not call this directly, use fuse_main() + */ +int fuse_main_real(int argc, char *argv[], const struct fuse_operations *op, + size_t op_size, void *user_data); + +/** + * Start the cleanup thread when using option "remember". + * + * This is done automatically by fuse_loop_mt() + * @param fuse struct fuse pointer for fuse instance + * @return 0 on success and -1 on error + */ +int fuse_start_cleanup_thread(struct fuse *fuse); + +/** + * Stop the cleanup thread when using option "remember". + * + * This is done automatically by fuse_loop_mt() + * @param fuse struct fuse pointer for fuse instance + */ +void fuse_stop_cleanup_thread(struct fuse *fuse); + +/** + * Iterate over cache removing stale entries + * use in conjunction with "-oremember" + * + * NOTE: This is already done for the standard sessions + * + * @param fuse struct fuse pointer for fuse instance + * @return the number of seconds until the next cleanup + */ +int fuse_clean_cache(struct fuse *fuse); + +/* + * Stacking API + */ + +/** + * Fuse filesystem object + * + * This is opaque object represents a filesystem layer + */ +struct fuse_fs; + +/* + * These functions call the relevant filesystem operation, and return + * the result. + * + * If the operation is not defined, they return -ENOSYS, with the + * exception of fuse_fs_open, fuse_fs_release, fuse_fs_opendir, + * fuse_fs_releasedir and fuse_fs_statfs, which return 0. + */ + +int fuse_fs_getattr(struct fuse_fs *fs, const char *path, struct stat *buf); +int fuse_fs_fgetattr(struct fuse_fs *fs, const char *path, struct stat *buf, + struct fuse_file_info *fi); +int fuse_fs_rename(struct fuse_fs *fs, const char *oldpath, + const char *newpath); +int fuse_fs_unlink(struct fuse_fs *fs, const char *path); +int fuse_fs_rmdir(struct fuse_fs *fs, const char *path); +int fuse_fs_symlink(struct fuse_fs *fs, const char *linkname, + const char *path); +int fuse_fs_link(struct fuse_fs *fs, const char *oldpath, const char *newpath); +int fuse_fs_release(struct fuse_fs *fs, const char *path, + struct fuse_file_info *fi); +int fuse_fs_open(struct fuse_fs *fs, const char *path, + struct fuse_file_info *fi); +int fuse_fs_read(struct fuse_fs *fs, const char *path, char *buf, size_t size, + off_t off, struct fuse_file_info *fi); +int fuse_fs_read_buf(struct fuse_fs *fs, const char *path, + struct fuse_bufvec **bufp, size_t size, off_t off, + struct fuse_file_info *fi); +int fuse_fs_write(struct fuse_fs *fs, const char *path, const char *buf, + size_t size, off_t off, struct fuse_file_info *fi); +int fuse_fs_write_buf(struct fuse_fs *fs, const char *path, + struct fuse_bufvec *buf, off_t off, + struct fuse_file_info *fi); +int fuse_fs_fsync(struct fuse_fs *fs, const char *path, int datasync, + struct fuse_file_info *fi); +int fuse_fs_flush(struct fuse_fs *fs, const char *path, + struct fuse_file_info *fi); +int fuse_fs_statfs(struct fuse_fs *fs, const char *path, struct statvfs *buf); +int fuse_fs_opendir(struct fuse_fs *fs, const char *path, + struct fuse_file_info *fi); +int fuse_fs_readdir(struct fuse_fs *fs, const char *path, void *buf, + fuse_fill_dir_t filler, off_t off, + struct fuse_file_info *fi); +int fuse_fs_fsyncdir(struct fuse_fs *fs, const char *path, int datasync, + struct fuse_file_info *fi); +int fuse_fs_releasedir(struct fuse_fs *fs, const char *path, + struct fuse_file_info *fi); +int fuse_fs_create(struct fuse_fs *fs, const char *path, mode_t mode, + struct fuse_file_info *fi); +int fuse_fs_lock(struct fuse_fs *fs, const char *path, + struct fuse_file_info *fi, int cmd, struct flock *lock); +int fuse_fs_flock(struct fuse_fs *fs, const char *path, + struct fuse_file_info *fi, int op); +int fuse_fs_chmod(struct fuse_fs *fs, const char *path, mode_t mode); +int fuse_fs_chown(struct fuse_fs *fs, const char *path, uid_t uid, gid_t gid); +int fuse_fs_truncate(struct fuse_fs *fs, const char *path, off_t size); +int fuse_fs_ftruncate(struct fuse_fs *fs, const char *path, off_t size, + struct fuse_file_info *fi); +int fuse_fs_utimens(struct fuse_fs *fs, const char *path, + const struct timespec tv[2]); +int fuse_fs_access(struct fuse_fs *fs, const char *path, int mask); +int fuse_fs_readlink(struct fuse_fs *fs, const char *path, char *buf, + size_t len); +int fuse_fs_mknod(struct fuse_fs *fs, const char *path, mode_t mode, + dev_t rdev); +int fuse_fs_mkdir(struct fuse_fs *fs, const char *path, mode_t mode); +int fuse_fs_setxattr(struct fuse_fs *fs, const char *path, const char *name, + const char *value, size_t size, int flags); +int fuse_fs_getxattr(struct fuse_fs *fs, const char *path, const char *name, + char *value, size_t size); +int fuse_fs_listxattr(struct fuse_fs *fs, const char *path, char *list, + size_t size); +int fuse_fs_removexattr(struct fuse_fs *fs, const char *path, + const char *name); +int fuse_fs_bmap(struct fuse_fs *fs, const char *path, size_t blocksize, + uint64_t *idx); +int fuse_fs_ioctl(struct fuse_fs *fs, const char *path, int cmd, void *arg, + struct fuse_file_info *fi, unsigned int flags, void *data); +int fuse_fs_poll(struct fuse_fs *fs, const char *path, + struct fuse_file_info *fi, struct fuse_pollhandle *ph, + unsigned *reventsp); +int fuse_fs_fallocate(struct fuse_fs *fs, const char *path, int mode, + off_t offset, off_t length, struct fuse_file_info *fi); +void fuse_fs_init(struct fuse_fs *fs, struct fuse_conn_info *conn); +void fuse_fs_destroy(struct fuse_fs *fs); + +int fuse_notify_poll(struct fuse_pollhandle *ph); + +/** + * Create a new fuse filesystem object + * + * This is usually called from the factory of a fuse module to create + * a new instance of a filesystem. + * + * @param op the filesystem operations + * @param op_size the size of the fuse_operations structure + * @param user_data user data supplied in the context during the init() method + * @return a new filesystem object + */ +struct fuse_fs *fuse_fs_new(const struct fuse_operations *op, size_t op_size, + void *user_data); + +/** + * Filesystem module + * + * Filesystem modules are registered with the FUSE_REGISTER_MODULE() + * macro. + * + * If the "-omodules=modname:..." option is present, filesystem + * objects are created and pushed onto the stack with the 'factory' + * function. + */ +struct fuse_module { + /** + * Name of filesystem + */ + const char *name; + + /** + * Factory for creating filesystem objects + * + * The function may use and remove options from 'args' that belong + * to this module. + * + * For now the 'fs' vector always contains exactly one filesystem. + * This is the filesystem which will be below the newly created + * filesystem in the stack. + * + * @param args the command line arguments + * @param fs NULL terminated filesystem object vector + * @return the new filesystem object + */ + struct fuse_fs *(*factory)(struct fuse_args *args, + struct fuse_fs *fs[]); + + struct fuse_module *next; + struct fusemod_so *so; + int ctr; +}; + +/** + * Register a filesystem module + * + * This function is used by FUSE_REGISTER_MODULE and there's usually + * no need to call it directly + */ +void fuse_register_module(struct fuse_module *mod); + +/** + * Register filesystem module + * + * For the parameters, see description of the fields in 'struct + * fuse_module' + */ +#define FUSE_REGISTER_MODULE(name_, factory_) \ + static __attribute__((constructor)) void name_ ## _register(void) \ + { \ + static struct fuse_module mod = \ + { #name_, factory_, NULL, NULL, 0 }; \ + fuse_register_module(&mod); \ + } + + +/* ----------------------------------------------------------- * + * Advanced API for event handling, don't worry about this... * + * ----------------------------------------------------------- */ + +/* NOTE: the following functions are deprecated, and will be removed + from the 3.0 API. Use the lowlevel session functions instead */ + +/** Function type used to process commands */ +typedef void (*fuse_processor_t)(struct fuse *, struct fuse_cmd *, void *); + +/** This is the part of fuse_main() before the event loop */ +struct fuse *fuse_setup(int argc, char *argv[], + const struct fuse_operations *op, size_t op_size, + char **mountpoint, int *multithreaded, + void *user_data); + +/** This is the part of fuse_main() after the event loop */ +void fuse_teardown(struct fuse *fuse, char *mountpoint); + +/** Read a single command. If none are read, return NULL */ +struct fuse_cmd *fuse_read_cmd(struct fuse *f); + +/** Process a single command */ +void fuse_process_cmd(struct fuse *f, struct fuse_cmd *cmd); + +/** Multi threaded event loop, which calls the custom command + processor function */ +int fuse_loop_mt_proc(struct fuse *f, fuse_processor_t proc, void *data); + +/** Return the exited flag, which indicates if fuse_exit() has been + called */ +int fuse_exited(struct fuse *f); + +/** This function is obsolete and implemented as a no-op */ +void fuse_set_getcontext_func(struct fuse_context *(*func)(void)); + +/** Get session from fuse object */ +struct fuse_session *fuse_get_session(struct fuse *f); + +/* ----------------------------------------------------------- * + * Compatibility stuff * + * ----------------------------------------------------------- */ + +#if FUSE_USE_VERSION < 26 +# include "fuse_compat.h" +# undef fuse_main +# if FUSE_USE_VERSION == 25 +# define fuse_main(argc, argv, op) \ + fuse_main_real_compat25(argc, argv, op, sizeof(*(op))) +# define fuse_new fuse_new_compat25 +# define fuse_setup fuse_setup_compat25 +# define fuse_teardown fuse_teardown_compat22 +# define fuse_operations fuse_operations_compat25 +# elif FUSE_USE_VERSION == 22 +# define fuse_main(argc, argv, op) \ + fuse_main_real_compat22(argc, argv, op, sizeof(*(op))) +# define fuse_new fuse_new_compat22 +# define fuse_setup fuse_setup_compat22 +# define fuse_teardown fuse_teardown_compat22 +# define fuse_operations fuse_operations_compat22 +# define fuse_file_info fuse_file_info_compat +# elif FUSE_USE_VERSION == 24 +# error Compatibility with high-level API version 24 not supported +# else +# define fuse_dirfil_t fuse_dirfil_t_compat +# define __fuse_read_cmd fuse_read_cmd +# define __fuse_process_cmd fuse_process_cmd +# define __fuse_loop_mt fuse_loop_mt_proc +# if FUSE_USE_VERSION == 21 +# define fuse_operations fuse_operations_compat2 +# define fuse_main fuse_main_compat2 +# define fuse_new fuse_new_compat2 +# define __fuse_setup fuse_setup_compat2 +# define __fuse_teardown fuse_teardown_compat22 +# define __fuse_exited fuse_exited +# define __fuse_set_getcontext_func fuse_set_getcontext_func +# else +# define fuse_statfs fuse_statfs_compat1 +# define fuse_operations fuse_operations_compat1 +# define fuse_main fuse_main_compat1 +# define fuse_new fuse_new_compat1 +# define FUSE_DEBUG FUSE_DEBUG_COMPAT1 +# endif +# endif +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _FUSE_H_ */ diff --git a/src/include/fuse_common.h b/src/include/fuse_common.h new file mode 100644 index 00000000..bdeed7c7 --- /dev/null +++ b/src/include/fuse_common.h @@ -0,0 +1,527 @@ +// IBM_PROLOG_BEGIN_TAG +// This is an automatically generated prolog. +// +// $Source: src/include/fuse_common.h $ +// +// IBM CONFIDENTIAL +// +// COPYRIGHT International Business Machines Corp. 2017 +// +// p1 +// +// Object Code Only (OCO) source materials +// Licensed Internal Code Source Materials +// IBM Surelock Licensed Internal Code +// +// The source code for this program is not published or other- +// wise divested of its trade secrets, irrespective of what has +// been deposited with the U.S. Copyright Office. +// +// Origin: 30 +// +// IBM_PROLOG_END +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2007 Miklos Szeredi + + This program can be distributed under the terms of the GNU LGPLv2. + See the file COPYING.LIB. +*/ + +/** @file */ + +#if !defined(_FUSE_H_) && !defined(_FUSE_LOWLEVEL_H_) +#error "Never include directly; use or instead." +#endif + +#ifndef _FUSE_COMMON_H_ +#define _FUSE_COMMON_H_ + +#include "fuse_opt.h" +#include +#include + +/** Major version of FUSE library interface */ +#define FUSE_MAJOR_VERSION 2 + +/** Minor version of FUSE library interface */ +#define FUSE_MINOR_VERSION 9 + +#define FUSE_MAKE_VERSION(maj, min) ((maj) * 10 + (min)) +#define FUSE_VERSION FUSE_MAKE_VERSION(FUSE_MAJOR_VERSION, FUSE_MINOR_VERSION) + +/* This interface uses 64 bit off_t */ +#if _FILE_OFFSET_BITS != 64 +#error Please add -D_FILE_OFFSET_BITS=64 to your compile flags! +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Information about open files + * + * Changed in version 2.5 + */ +struct fuse_file_info { + /** Open flags. Available in open() and release() */ + int flags; + + /** Old file handle, don't use */ + unsigned long fh_old; + + /** In case of a write operation indicates if this was caused by a + writepage */ + int writepage; + + /** Can be filled in by open, to use direct I/O on this file. + Introduced in version 2.4 */ + unsigned int direct_io : 1; + + /** Can be filled in by open, to indicate, that cached file data + need not be invalidated. Introduced in version 2.4 */ + unsigned int keep_cache : 1; + + /** Indicates a flush operation. Set in flush operation, also + maybe set in highlevel lock operation and lowlevel release + operation. Introduced in version 2.6 */ + unsigned int flush : 1; + + /** Can be filled in by open, to indicate that the file is not + seekable. Introduced in version 2.8 */ + unsigned int nonseekable : 1; + + /* Indicates that flock locks for this file should be + released. If set, lock_owner shall contain a valid value. + May only be set in ->release(). Introduced in version + 2.9 */ + unsigned int flock_release : 1; + + /** Padding. Do not use*/ + unsigned int padding : 27; + + /** File handle. May be filled in by filesystem in open(). + Available in all other file operations */ + uint64_t fh; + + /** Lock owner id. Available in locking operations and flush */ + uint64_t lock_owner; +}; + +/** + * Capability bits for 'fuse_conn_info.capable' and 'fuse_conn_info.want' + * + * FUSE_CAP_ASYNC_READ: filesystem supports asynchronous read requests + * FUSE_CAP_POSIX_LOCKS: filesystem supports "remote" locking + * FUSE_CAP_ATOMIC_O_TRUNC: filesystem handles the O_TRUNC open flag + * FUSE_CAP_EXPORT_SUPPORT: filesystem handles lookups of "." and ".." + * FUSE_CAP_BIG_WRITES: filesystem can handle write size larger than 4kB + * FUSE_CAP_DONT_MASK: don't apply umask to file mode on create operations + * FUSE_CAP_SPLICE_WRITE: ability to use splice() to write to the fuse device + * FUSE_CAP_SPLICE_MOVE: ability to move data to the fuse device with splice() + * FUSE_CAP_SPLICE_READ: ability to use splice() to read from the fuse device + * FUSE_CAP_IOCTL_DIR: ioctl support on directories + */ +#define FUSE_CAP_ASYNC_READ (1 << 0) +#define FUSE_CAP_POSIX_LOCKS (1 << 1) +#define FUSE_CAP_ATOMIC_O_TRUNC (1 << 3) +#define FUSE_CAP_EXPORT_SUPPORT (1 << 4) +#define FUSE_CAP_BIG_WRITES (1 << 5) +#define FUSE_CAP_DONT_MASK (1 << 6) +#define FUSE_CAP_SPLICE_WRITE (1 << 7) +#define FUSE_CAP_SPLICE_MOVE (1 << 8) +#define FUSE_CAP_SPLICE_READ (1 << 9) +#define FUSE_CAP_FLOCK_LOCKS (1 << 10) +#define FUSE_CAP_IOCTL_DIR (1 << 11) + +/** + * Ioctl flags + * + * FUSE_IOCTL_COMPAT: 32bit compat ioctl on 64bit machine + * FUSE_IOCTL_UNRESTRICTED: not restricted to well-formed ioctls, retry allowed + * FUSE_IOCTL_RETRY: retry with new iovecs + * FUSE_IOCTL_DIR: is a directory + * + * FUSE_IOCTL_MAX_IOV: maximum of in_iovecs + out_iovecs + */ +#define FUSE_IOCTL_COMPAT (1 << 0) +#define FUSE_IOCTL_UNRESTRICTED (1 << 1) +#define FUSE_IOCTL_RETRY (1 << 2) +#define FUSE_IOCTL_DIR (1 << 4) + +#define FUSE_IOCTL_MAX_IOV 256 + +/** + * Connection information, passed to the ->init() method + * + * Some of the elements are read-write, these can be changed to + * indicate the value requested by the filesystem. The requested + * value must usually be smaller than the indicated value. + */ +struct fuse_conn_info { + /** + * Major version of the protocol (read-only) + */ + unsigned proto_major; + + /** + * Minor version of the protocol (read-only) + */ + unsigned proto_minor; + + /** + * Is asynchronous read supported (read-write) + */ + unsigned async_read; + + /** + * Maximum size of the write buffer + */ + unsigned max_write; + + /** + * Maximum readahead + */ + unsigned max_readahead; + + /** + * Capability flags, that the kernel supports + */ + unsigned capable; + + /** + * Capability flags, that the filesystem wants to enable + */ + unsigned want; + + /** + * Maximum number of backgrounded requests + */ + unsigned max_background; + + /** + * Kernel congestion threshold parameter + */ + unsigned congestion_threshold; + + /** + * For future use. + */ + unsigned reserved[23]; +}; + +struct fuse_session; +struct fuse_chan; +struct fuse_pollhandle; + +/** + * Create a FUSE mountpoint + * + * Returns a control file descriptor suitable for passing to + * fuse_new() + * + * @param mountpoint the mount point path + * @param args argument vector + * @return the communication channel on success, NULL on failure + */ +struct fuse_chan *fuse_mount(const char *mountpoint, struct fuse_args *args); + +/** + * Umount a FUSE mountpoint + * + * @param mountpoint the mount point path + * @param ch the communication channel + */ +void fuse_unmount(const char *mountpoint, struct fuse_chan *ch); + +/** + * Parse common options + * + * The following options are parsed: + * + * '-f' foreground + * '-d' '-odebug' foreground, but keep the debug option + * '-s' single threaded + * '-h' '--help' help + * '-ho' help without header + * '-ofsname=..' file system name, if not present, then set to the program + * name + * + * All parameters may be NULL + * + * @param args argument vector + * @param mountpoint the returned mountpoint, should be freed after use + * @param multithreaded set to 1 unless the '-s' option is present + * @param foreground set to 1 if one of the relevant options is present + * @return 0 on success, -1 on failure + */ +int fuse_parse_cmdline(struct fuse_args *args, char **mountpoint, + int *multithreaded, int *foreground); + +/** + * Go into the background + * + * @param foreground if true, stay in the foreground + * @return 0 on success, -1 on failure + */ +int fuse_daemonize(int foreground); + +/** + * Get the version of the library + * + * @return the version + */ +int fuse_version(void); + +/** + * Destroy poll handle + * + * @param ph the poll handle + */ +void fuse_pollhandle_destroy(struct fuse_pollhandle *ph); + +/* ----------------------------------------------------------- * + * Data buffer * + * ----------------------------------------------------------- */ + +/** + * Buffer flags + */ +enum fuse_buf_flags { + /** + * Buffer contains a file descriptor + * + * If this flag is set, the .fd field is valid, otherwise the + * .mem fields is valid. + */ + FUSE_BUF_IS_FD = (1 << 1), + + /** + * Seek on the file descriptor + * + * If this flag is set then the .pos field is valid and is + * used to seek to the given offset before performing + * operation on file descriptor. + */ + FUSE_BUF_FD_SEEK = (1 << 2), + + /** + * Retry operation on file descriptor + * + * If this flag is set then retry operation on file descriptor + * until .size bytes have been copied or an error or EOF is + * detected. + */ + FUSE_BUF_FD_RETRY = (1 << 3), +}; + +/** + * Buffer copy flags + */ +enum fuse_buf_copy_flags { + /** + * Don't use splice(2) + * + * Always fall back to using read and write instead of + * splice(2) to copy data from one file descriptor to another. + * + * If this flag is not set, then only fall back if splice is + * unavailable. + */ + FUSE_BUF_NO_SPLICE = (1 << 1), + + /** + * Force splice + * + * Always use splice(2) to copy data from one file descriptor + * to another. If splice is not available, return -EINVAL. + */ + FUSE_BUF_FORCE_SPLICE = (1 << 2), + + /** + * Try to move data with splice. + * + * If splice is used, try to move pages from the source to the + * destination instead of copying. See documentation of + * SPLICE_F_MOVE in splice(2) man page. + */ + FUSE_BUF_SPLICE_MOVE = (1 << 3), + + /** + * Don't block on the pipe when copying data with splice + * + * Makes the operations on the pipe non-blocking (if the pipe + * is full or empty). See SPLICE_F_NONBLOCK in the splice(2) + * man page. + */ + FUSE_BUF_SPLICE_NONBLOCK= (1 << 4), +}; + +/** + * Single data buffer + * + * Generic data buffer for I/O, extended attributes, etc... Data may + * be supplied as a memory pointer or as a file descriptor + */ +struct fuse_buf { + /** + * Size of data in bytes + */ + size_t size; + + /** + * Buffer flags + */ + enum fuse_buf_flags flags; + + /** + * Memory pointer + * + * Used unless FUSE_BUF_IS_FD flag is set. + */ + void *mem; + + /** + * File descriptor + * + * Used if FUSE_BUF_IS_FD flag is set. + */ + int fd; + + /** + * File position + * + * Used if FUSE_BUF_FD_SEEK flag is set. + */ + off_t pos; +}; + +/** + * Data buffer vector + * + * An array of data buffers, each containing a memory pointer or a + * file descriptor. + * + * Allocate dynamically to add more than one buffer. + */ +struct fuse_bufvec { + /** + * Number of buffers in the array + */ + size_t count; + + /** + * Index of current buffer within the array + */ + size_t idx; + + /** + * Current offset within the current buffer + */ + size_t off; + + /** + * Array of buffers + */ + struct fuse_buf buf[1]; +}; + +/* Initialize bufvec with a single buffer of given size */ +#define FUSE_BUFVEC_INIT(size__) \ + ((struct fuse_bufvec) { \ + /* .count= */ 1, \ + /* .idx = */ 0, \ + /* .off = */ 0, \ + /* .buf = */ { /* [0] = */ { \ + /* .size = */ (size__), \ + /* .flags = */ (enum fuse_buf_flags) 0, \ + /* .mem = */ NULL, \ + /* .fd = */ -1, \ + /* .pos = */ 0, \ + } } \ + } ) + +/** + * Get total size of data in a fuse buffer vector + * + * @param bufv buffer vector + * @return size of data + */ +size_t fuse_buf_size(const struct fuse_bufvec *bufv); + +/** + * Copy data from one buffer vector to another + * + * @param dst destination buffer vector + * @param src source buffer vector + * @param flags flags controlling the copy + * @return actual number of bytes copied or -errno on error + */ +ssize_t fuse_buf_copy(struct fuse_bufvec *dst, struct fuse_bufvec *src, + enum fuse_buf_copy_flags flags); + +/* ----------------------------------------------------------- * + * Signal handling * + * ----------------------------------------------------------- */ + +/** + * Exit session on HUP, TERM and INT signals and ignore PIPE signal + * + * Stores session in a global variable. May only be called once per + * process until fuse_remove_signal_handlers() is called. + * + * @param se the session to exit + * @return 0 on success, -1 on failure + */ +int fuse_set_signal_handlers(struct fuse_session *se); + +/** + * Restore default signal handlers + * + * Resets global session. After this fuse_set_signal_handlers() may + * be called again. + * + * @param se the same session as given in fuse_set_signal_handlers() + */ +void fuse_remove_signal_handlers(struct fuse_session *se); + +/* ----------------------------------------------------------- * + * Compatibility stuff * + * ----------------------------------------------------------- */ + +#if FUSE_USE_VERSION < 26 +# ifdef __FreeBSD__ +# if FUSE_USE_VERSION < 25 +# error On FreeBSD API version 25 or greater must be used +# endif +# endif +# include "fuse_common_compat.h" +# undef FUSE_MINOR_VERSION +# undef fuse_main +# define fuse_unmount fuse_unmount_compat22 +# if FUSE_USE_VERSION == 25 +# define FUSE_MINOR_VERSION 5 +# define fuse_mount fuse_mount_compat25 +# elif FUSE_USE_VERSION == 24 || FUSE_USE_VERSION == 22 +# define FUSE_MINOR_VERSION 4 +# define fuse_mount fuse_mount_compat22 +# elif FUSE_USE_VERSION == 21 +# define FUSE_MINOR_VERSION 1 +# define fuse_mount fuse_mount_compat22 +# elif FUSE_USE_VERSION == 11 +# warning Compatibility with API version 11 is deprecated +# undef FUSE_MAJOR_VERSION +# define FUSE_MAJOR_VERSION 1 +# define FUSE_MINOR_VERSION 1 +# define fuse_mount fuse_mount_compat1 +# else +# error Compatibility with API version other than 21, 22, 24, 25 and 11 not supported +# endif +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _FUSE_COMMON_H_ */ diff --git a/src/include/fuse_common_compat.h b/src/include/fuse_common_compat.h new file mode 100644 index 00000000..d7544cff --- /dev/null +++ b/src/include/fuse_common_compat.h @@ -0,0 +1,48 @@ +// IBM_PROLOG_BEGIN_TAG +// This is an automatically generated prolog. +// +// $Source: src/include/fuse_common_compat.h $ +// +// IBM CONFIDENTIAL +// +// COPYRIGHT International Business Machines Corp. 2017 +// +// p1 +// +// Object Code Only (OCO) source materials +// Licensed Internal Code Source Materials +// IBM Surelock Licensed Internal Code +// +// The source code for this program is not published or other- +// wise divested of its trade secrets, irrespective of what has +// been deposited with the U.S. Copyright Office. +// +// Origin: 30 +// +// IBM_PROLOG_END +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2007 Miklos Szeredi + + This program can be distributed under the terms of the GNU LGPLv2. + See the file COPYING.LIB. +*/ + +/* these definitions provide source compatibility to prior versions. + Do not include this file directly! */ + +struct fuse_file_info_compat { + int flags; + unsigned long fh; + int writepage; + unsigned int direct_io : 1; + unsigned int keep_cache : 1; +}; + +int fuse_mount_compat25(const char *mountpoint, struct fuse_args *args); + +int fuse_mount_compat22(const char *mountpoint, const char *opts); + +int fuse_mount_compat1(const char *mountpoint, const char *args[]); + +void fuse_unmount_compat22(const char *mountpoint); diff --git a/src/include/fuse_opt.h b/src/include/fuse_opt.h new file mode 100644 index 00000000..4e2e9080 --- /dev/null +++ b/src/include/fuse_opt.h @@ -0,0 +1,292 @@ +// IBM_PROLOG_BEGIN_TAG +// This is an automatically generated prolog. +// +// $Source: src/include/fuse_opt.h $ +// +// IBM CONFIDENTIAL +// +// COPYRIGHT International Business Machines Corp. 2017 +// +// p1 +// +// Object Code Only (OCO) source materials +// Licensed Internal Code Source Materials +// IBM Surelock Licensed Internal Code +// +// The source code for this program is not published or other- +// wise divested of its trade secrets, irrespective of what has +// been deposited with the U.S. Copyright Office. +// +// Origin: 30 +// +// IBM_PROLOG_END +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2007 Miklos Szeredi + + This program can be distributed under the terms of the GNU LGPLv2. + See the file COPYING.LIB. +*/ + +#ifndef _FUSE_OPT_H_ +#define _FUSE_OPT_H_ + +/** @file + * + * This file defines the option parsing interface of FUSE + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Option description + * + * This structure describes a single option, and action associated + * with it, in case it matches. + * + * More than one such match may occur, in which case the action for + * each match is executed. + * + * There are three possible actions in case of a match: + * + * i) An integer (int or unsigned) variable determined by 'offset' is + * set to 'value' + * + * ii) The processing function is called, with 'value' as the key + * + * iii) An integer (any) or string (char *) variable determined by + * 'offset' is set to the value of an option parameter + * + * 'offset' should normally be either set to + * + * - 'offsetof(struct foo, member)' actions i) and iii) + * + * - -1 action ii) + * + * The 'offsetof()' macro is defined in the header. + * + * The template determines which options match, and also have an + * effect on the action. Normally the action is either i) or ii), but + * if a format is present in the template, then action iii) is + * performed. + * + * The types of templates are: + * + * 1) "-x", "-foo", "--foo", "--foo-bar", etc. These match only + * themselves. Invalid values are "--" and anything beginning + * with "-o" + * + * 2) "foo", "foo-bar", etc. These match "-ofoo", "-ofoo-bar" or + * the relevant option in a comma separated option list + * + * 3) "bar=", "--foo=", etc. These are variations of 1) and 2) + * which have a parameter + * + * 4) "bar=%s", "--foo=%lu", etc. Same matching as above but perform + * action iii). + * + * 5) "-x ", etc. Matches either "-xparam" or "-x param" as + * two separate arguments + * + * 6) "-x %s", etc. Combination of 4) and 5) + * + * If the format is "%s", memory is allocated for the string unlike + * with scanf(). + */ +struct fuse_opt { + /** Matching template and optional parameter formatting */ + const char *templ; + + /** + * Offset of variable within 'data' parameter of fuse_opt_parse() + * or -1 + */ + unsigned long offset; + + /** + * Value to set the variable to, or to be passed as 'key' to the + * processing function. Ignored if template has a format + */ + int value; +}; + +/** + * Key option. In case of a match, the processing function will be + * called with the specified key. + */ +#define FUSE_OPT_KEY(templ, key) { templ, -1U, key } + +/** + * Last option. An array of 'struct fuse_opt' must end with a NULL + * template value + */ +#define FUSE_OPT_END { NULL, 0, 0 } + +/** + * Argument list + */ +struct fuse_args { + /** Argument count */ + int argc; + + /** Argument vector. NULL terminated */ + char **argv; + + /** Is 'argv' allocated? */ + int allocated; +}; + +/** + * Initializer for 'struct fuse_args' + */ +#define FUSE_ARGS_INIT(argc, argv) { argc, argv, 0 } + +/** + * Key value passed to the processing function if an option did not + * match any template + */ +#define FUSE_OPT_KEY_OPT -1 + +/** + * Key value passed to the processing function for all non-options + * + * Non-options are the arguments beginning with a character other than + * '-' or all arguments after the special '--' option + */ +#define FUSE_OPT_KEY_NONOPT -2 + +/** + * Special key value for options to keep + * + * Argument is not passed to processing function, but behave as if the + * processing function returned 1 + */ +#define FUSE_OPT_KEY_KEEP -3 + +/** + * Special key value for options to discard + * + * Argument is not passed to processing function, but behave as if the + * processing function returned zero + */ +#define FUSE_OPT_KEY_DISCARD -4 + +/** + * Processing function + * + * This function is called if + * - option did not match any 'struct fuse_opt' + * - argument is a non-option + * - option did match and offset was set to -1 + * + * The 'arg' parameter will always contain the whole argument or + * option including the parameter if exists. A two-argument option + * ("-x foo") is always converted to single argument option of the + * form "-xfoo" before this function is called. + * + * Options of the form '-ofoo' are passed to this function without the + * '-o' prefix. + * + * The return value of this function determines whether this argument + * is to be inserted into the output argument vector, or discarded. + * + * @param data is the user data passed to the fuse_opt_parse() function + * @param arg is the whole argument or option + * @param key determines why the processing function was called + * @param outargs the current output argument list + * @return -1 on error, 0 if arg is to be discarded, 1 if arg should be kept + */ +typedef int (*fuse_opt_proc_t)(void *data, const char *arg, int key, + struct fuse_args *outargs); + +/** + * Option parsing function + * + * If 'args' was returned from a previous call to fuse_opt_parse() or + * it was constructed from + * + * A NULL 'args' is equivalent to an empty argument vector + * + * A NULL 'opts' is equivalent to an 'opts' array containing a single + * end marker + * + * A NULL 'proc' is equivalent to a processing function always + * returning '1' + * + * @param args is the input and output argument list + * @param data is the user data + * @param opts is the option description array + * @param proc is the processing function + * @return -1 on error, 0 on success + */ +int fuse_opt_parse(struct fuse_args *args, void *data, + const struct fuse_opt opts[], fuse_opt_proc_t proc); + +/** + * Add an option to a comma separated option list + * + * @param opts is a pointer to an option list, may point to a NULL value + * @param opt is the option to add + * @return -1 on allocation error, 0 on success + */ +int fuse_opt_add_opt(char **opts, const char *opt); + +/** + * Add an option, escaping commas, to a comma separated option list + * + * @param opts is a pointer to an option list, may point to a NULL value + * @param opt is the option to add + * @return -1 on allocation error, 0 on success + */ +int fuse_opt_add_opt_escaped(char **opts, const char *opt); + +/** + * Add an argument to a NULL terminated argument vector + * + * @param args is the structure containing the current argument list + * @param arg is the new argument to add + * @return -1 on allocation error, 0 on success + */ +int fuse_opt_add_arg(struct fuse_args *args, const char *arg); + +/** + * Add an argument at the specified position in a NULL terminated + * argument vector + * + * Adds the argument to the N-th position. This is useful for adding + * options at the beginning of the array which must not come after the + * special '--' option. + * + * @param args is the structure containing the current argument list + * @param pos is the position at which to add the argument + * @param arg is the new argument to add + * @return -1 on allocation error, 0 on success + */ +int fuse_opt_insert_arg(struct fuse_args *args, int pos, const char *arg); + +/** + * Free the contents of argument list + * + * The structure itself is not freed + * + * @param args is the structure containing the argument list + */ +void fuse_opt_free_args(struct fuse_args *args); + + +/** + * Check if an option matches + * + * @param opts is the option description array + * @param opt is the option to match + * @return 1 if a match is found, 0 if not + */ +int fuse_opt_match(const struct fuse_opt opts[], const char *opt); + +#ifdef __cplusplus +} +#endif + +#endif /* _FUSE_OPT_H_ */ diff --git a/src/include/libcxl.h b/src/include/libcxl.h index de6daa9e..9e0c9c57 100644 --- a/src/include/libcxl.h +++ b/src/include/libcxl.h @@ -25,18 +25,11 @@ #ifndef _LIBCXL_H #define _LIBCXL_H -#include -#include "cxl.h" +#include +#include +#include -/* - * This is a very early library to simplify userspace code accessing a CXL - * device. - * - * Currently there are only a couple of functions here - more is on the way. - * - * Suggestions to improve the library, simplify it's usage, add additional - * functionality, etc. are welcome - */ +#define CXL_KERNEL_API_VERSION 1 #define CXL_SYSFS_CLASS "/sys/class/cxl" #define CXL_DEV_DIR "/dev/cxl" @@ -46,6 +39,7 @@ */ struct cxl_adapter_h; struct cxl_afu_h; +struct cxl_ioctl_start_work; /* * Adapter Enumeration @@ -57,7 +51,7 @@ struct cxl_afu_h; * last adapter, or cxl_adapter_free() can be called explicitly. */ struct cxl_adapter_h * cxl_adapter_next(struct cxl_adapter_h *adapter); -char * cxl_adapter_devname(struct cxl_adapter_h *adapter); +char * cxl_adapter_dev_name(struct cxl_adapter_h *adapter); void cxl_adapter_free(struct cxl_adapter_h *adapter); #define cxl_for_each_adapter(adapter) \ for (adapter = cxl_adapter_next(NULL); adapter; adapter = cxl_adapter_next(adapter)) @@ -79,12 +73,17 @@ void cxl_adapter_free(struct cxl_adapter_h *adapter); */ struct cxl_afu_h * cxl_adapter_afu_next(struct cxl_adapter_h *adapter, struct cxl_afu_h *afu); struct cxl_afu_h * cxl_afu_next(struct cxl_afu_h *afu); -char * cxl_afu_devname(struct cxl_afu_h *afu); +char * cxl_afu_dev_name(struct cxl_afu_h *afu); #define cxl_for_each_adapter_afu(adapter, afu) \ - for (afu = cxl_adapter_afu_next(adapter, NULL); afu; afu = cxl_adapter_afu_next(NULL, afu)) + for (afu = cxl_adapter_afu_next(adapter, NULL); afu; afu = cxl_adapter_afu_next(adapter, afu)) #define cxl_for_each_afu(afu) \ for (afu = cxl_afu_next(NULL); afu; afu = cxl_afu_next(afu)) +enum cxl_views { + CXL_VIEW_DEDICATED = 0, + CXL_VIEW_MASTER, + CXL_VIEW_SLAVE +}; /* * Open AFU - either by path, by AFU being enumerated, or tie into an AFU file @@ -92,16 +91,30 @@ char * cxl_afu_devname(struct cxl_afu_h *afu); * closed by cxl_afu_free() regardless of how it was opened. */ struct cxl_afu_h * cxl_afu_open_dev(char *path); -int cxl_afu_open_h(struct cxl_afu_h *afu, unsigned long master); +struct cxl_afu_h * cxl_afu_open_h(struct cxl_afu_h *afu, enum cxl_views view); struct cxl_afu_h * cxl_afu_fd_to_h(int fd); void cxl_afu_free(struct cxl_afu_h *afu); +int cxl_afu_opened(struct cxl_afu_h *afu); /* * Attach AFU context to this process */ +struct cxl_ioctl_start_work *cxl_work_alloc(void); +int cxl_work_free(struct cxl_ioctl_start_work *work); +int cxl_work_get_amr(struct cxl_ioctl_start_work *work, __u64 *valp); +int cxl_work_get_num_irqs(struct cxl_ioctl_start_work *work, __s16 *valp); +int cxl_work_get_wed(struct cxl_ioctl_start_work *work, __u64 *valp); +int cxl_work_set_amr(struct cxl_ioctl_start_work *work, __u64 amr); +int cxl_work_set_num_irqs(struct cxl_ioctl_start_work *work, __s16 num_irqs); +int cxl_work_set_wed(struct cxl_ioctl_start_work *work, __u64 wed); + +int cxl_afu_attach(struct cxl_afu_h *afu, __u64 wed); +int cxl_afu_attach_work(struct cxl_afu_h *afu, + struct cxl_ioctl_start_work *work); + +/* Deprecated interface */ int cxl_afu_attach_full(struct cxl_afu_h *afu, __u64 wed, __u16 num_interrupts, __u64 amr); -int cxl_afu_attach(struct cxl_afu_h *afu, __u64 wed); /* * Get AFU process element @@ -115,34 +128,126 @@ int cxl_afu_get_process_element(struct cxl_afu_h *afu); int cxl_afu_fd(struct cxl_afu_h *afu); /* - * TODO: All in one function - opens an AFU, verifies the operating mode and - * attaches the context. - * int cxl_afu_open_and_attach(struct cxl_afu_h *afu, mode) + * sysfs helpers */ /* - * sysfs helpers - * - * NOTE: On success, these functions automatically allocate the returned - * buffers, which must be freed by the caller (much like asprintf). + * NOTE: On success, this function automatically allocates the returned + * buffer, which must be freed by the caller (much like asprintf). */ -int cxl_afu_sysfs_pci(char **pathp, struct cxl_afu_h *afu); +int cxl_afu_sysfs_pci(struct cxl_afu_h *afu, char **pathp); + +/* Flags for cxl_get/set_mode and cxl_get_modes_supported */ +#define CXL_MODE_DEDICATED 0x1 +#define CXL_MODE_DIRECTED 0x2 +#define CXL_MODE_TIME_SLICED 0x4 + +/* Values for cxl_get/set_prefault_mode */ +enum cxl_prefault_mode { + CXL_PREFAULT_MODE_NONE = 0, + CXL_PREFAULT_MODE_WED, + CXL_PREFAULT_MODE_ALL, +}; + +/* Values for cxl_get_image_loaded */ +enum cxl_image { + CXL_IMAGE_FACTORY = 0, + CXL_IMAGE_USER, +}; + +/* + * Get/set attribute values. + * Return 0 on success, -1 on error. + */ +int cxl_get_api_version(struct cxl_afu_h *afu, long *valp); +int cxl_get_api_version_compatible(struct cxl_afu_h *afu, long *valp); +int cxl_get_cr_class(struct cxl_afu_h *afu, long cr_num, long *valp); +int cxl_get_cr_device(struct cxl_afu_h *afu, long cr_num, long *valp); +int cxl_get_cr_vendor(struct cxl_afu_h *afu, long cr_num, long *valp); +int cxl_get_irqs_max(struct cxl_afu_h *afu, long *valp); +int cxl_set_irqs_max(struct cxl_afu_h *afu, long value); +int cxl_get_irqs_min(struct cxl_afu_h *afu, long *valp); +int cxl_get_mmio_size(struct cxl_afu_h *afu, long *valp); +int cxl_get_mode(struct cxl_afu_h *afu, long *valp); +int cxl_set_mode(struct cxl_afu_h *afu, long value); +int cxl_get_modes_supported(struct cxl_afu_h *afu, long *valp); +int cxl_get_prefault_mode(struct cxl_afu_h *afu, enum cxl_prefault_mode *valp); +int cxl_set_prefault_mode(struct cxl_afu_h *afu, enum cxl_prefault_mode value); +int cxl_get_pp_mmio_len(struct cxl_afu_h *afu, long *valp); +int cxl_get_pp_mmio_off(struct cxl_afu_h *afu, long *valp); +int cxl_get_base_image(struct cxl_adapter_h *adapter, long *valp); +int cxl_get_caia_version(struct cxl_adapter_h *adapter, long *majorp, + long *minorp); +int cxl_get_image_loaded(struct cxl_adapter_h *adapter, enum cxl_image *valp); +int cxl_get_psl_revision(struct cxl_adapter_h *adapter, long *valp); /* * Events */ -bool cxl_pending_event(struct cxl_afu_h *afu); +int cxl_event_pending(struct cxl_afu_h *afu); int cxl_read_event(struct cxl_afu_h *afu, struct cxl_event *event); int cxl_read_expected_event(struct cxl_afu_h *afu, struct cxl_event *event, __u32 type, __u16 irq); /* * fprint wrappers to print out CXL events - useful for debugging. - * fprint_cxl_event will select the appropriate implementation based on the - * event type and fprint_cxl_unknown_event will print out a hex dump of the + * cxl_fprint_event will select the appropriate implementation based on the + * event type and cxl_fprint_unknown_event will print out a hex dump of the * raw event. */ -int fprint_cxl_event(FILE *stream, struct cxl_event *event); -int fprint_cxl_unknown_event(FILE *stream, struct cxl_event *event); +int cxl_fprint_event(FILE *stream, struct cxl_event *event); +int cxl_fprint_unknown_event(FILE *stream, struct cxl_event *event); + +/* + * AFU MMIO functions + * + * The below assessors will byte swap based on what is passed to map. Also a + * full memory barrier 'sync' will proceed a write and follow a read. More + * relaxed assessors can be created using a pointer derived from cxl_mmio_ptr(). + */ +#define CXL_MMIO_BIG_ENDIAN 0x1 +#define CXL_MMIO_LITTLE_ENDIAN 0x2 +#define CXL_MMIO_HOST_ENDIAN 0x3 +#define CXL_MMIO_ENDIAN_MASK 0x3 +#define CXL_MMIO_FLAGS 0x3 +int cxl_mmio_map(struct cxl_afu_h *afu, __u32 flags); +int cxl_mmio_unmap(struct cxl_afu_h *afu); +int cxl_mmio_ptr(struct cxl_afu_h *afu, void **mmio_ptrp); +int cxl_mmio_write64(struct cxl_afu_h *afu, uint64_t offset, uint64_t data); +int cxl_mmio_read64(struct cxl_afu_h *afu, uint64_t offset, uint64_t *data); +int cxl_mmio_write32(struct cxl_afu_h *afu, uint64_t offset, uint32_t data); +int cxl_mmio_read32(struct cxl_afu_h *afu, uint64_t offset, uint32_t *data); +/* + * Calling this function will install the libcxl SIGBUS handler. This will + * catch bad MMIO accesses (e.g. due to hardware failures) that would otherwise + * terminate the program and make the above mmio functions return errors + * instead. + * + * Call this once per process prior to any MMIO accesses. + */ +int cxl_mmio_install_sigbus_handler(); + +/** + * Returns the size of afu_err_buff in bytes. + * @param afu Handle to the afu. + * @param valp Pointer to the location where size is copied to. + * @return In case of success '0' is returned. In case of an error or + * the afu_err_buff doesn't exist, -1 is returned and errno is set + * appropriately. + */ +int cxl_errinfo_size(struct cxl_afu_h *afu, size_t *valp); + +/** + * Read and copy the contents of afu_err_info buffer into the provided buffer. + * @param afu Handle to the afu + * @param dst Pointer to the buffer where data would be copied. + * @param off Start offset within the afu_err_info handle. + * @param len Number of bytes to be copied after the start offset. + * @return The number of bytes copied from the afu_err_buff to dst. In case of + * an error or the afu_err_buff doesn't exist, -1 is returned and errno is set + * appropriately. + */ +ssize_t cxl_errinfo_read(struct cxl_afu_h *afu, void *dst, off_t off, + size_t len); #endif diff --git a/src/include/sislite.h b/src/include/sislite.h index 1d29db09..f4da1ddf 100755 --- a/src/include/sislite.h +++ b/src/include/sislite.h @@ -39,16 +39,22 @@ /************************************************************************/ /* Byte offsets to various host interface registers */ -#define CAPI_ENDIAN_CTRL_OFFSET 0x00 /* Endian control */ -#define CAPI_INTR_STATUS_OFFSET 0x08 /* Interrupt Staus register */ -#define CAPI_INTR_CLEAR_OFFSET 0x10 /* Interrupt Clear register */ -#define CAPI_INTR_MASK_OFFSET 0x18 /* Interrupt Mask register */ -#define CAPI_IOARRIN_OFFSET 0x20 /* IOARRIN register */ -#define CAPI_RRQ0_START_EA_OFFSET 0x28 /* RRQ#0 start EA register */ -#define CAPI_RRQ0_END_EA_OFFSET 0x30 /* RRQ#0 end EA register */ -#define CAPI_CMD_ROOM_OFFSET 0x38 /* CMD_ROOM register */ -#define CAPI_CTX_CTRL_OFFSET 0x40 /* Context Control register */ -#define CAPI_MBOX_W_OFFSET 0x48 /* Mailbox write register */ +#define CAPI_ENDIAN_CTRL_OFFSET 0x00 /* Endian control */ +#define CAPI_INTR_STATUS_OFFSET 0x08 /* Interrupt Staus register */ +#define CAPI_INTR_CLEAR_OFFSET 0x10 /* Interrupt Clear register */ +#define CAPI_INTR_MASK_OFFSET 0x18 /* Interrupt Mask register */ +#define CAPI_IOARRIN_OFFSET 0x20 /* IOARRIN register */ +#define CAPI_RRQ0_START_EA_OFFSET 0x28 /* RRQ#0 start EA register */ +#define CAPI_RRQ0_END_EA_OFFSET 0x30 /* RRQ#0 end EA register */ +#define CAPI_CMD_ROOM_OFFSET 0x38 /* CMD_ROOM register */ +#define CAPI_CTX_CTRL_OFFSET 0x40 /* Context Control register */ +#define CAPI_MBOX_W_OFFSET 0x48 /* Mailbox write register */ +#define CAPI_SQ_START_EA_OFFSET 0x50 /* SQ start EA register */ +#define CAPI_SQ_END_EA_OFFSET 0x58 /* SQ end EA register */ +#define CAPI_SQ_HEAD_EA_OFFSET 0x60 /* SQ head EA register */ +#define CAPI_SQ_TAIL_EA_OFFSET 0x68 /* SQ tail EA register */ +#define CAPI_SQ_CTXTRST_EA_OFFSET 0x70 /* SQ context reset EA */ + /* register. */ #define CAPI_AFU_GLOBAL_OFFSET 0x10000 /* Offset of AFU Global area */ @@ -106,7 +112,12 @@ typedef struct sisl_ioarcb_s { __u16 timeout; /* in units specified by req_flags */ __u32 rsvd1; __u8 cdb[16]; /* must be in big endian */ - __u64 rsvd2; + union { + __u64 sq_ioasa_ea; /* When using Submission queues (SQ),*/ + /* this is the effective address of */ + /* ths associated IOASA. */ + __u64 rsvd2; + }; } sisl_ioarcb_t; struct sisl_rc { @@ -142,7 +153,13 @@ struct sisl_rc { */ #define SISL_AFU_RC_NO_CHANNELS 0x20u /* see afu_extra, may retry */ #define SISL_AFU_RC_CAP_VIOLATION 0x21u /* either user error or - afu reset/master restart + afu reset/master restart. + see afu_extra for more + information. It contains + the values of the ctx_cap + register. With the ctx_cap + upper and lower bytes + contiguous in the afu_extra. */ #define SISL_AFU_RC_OUT_OF_DATA_BUFS 0x30u /* always retry */ #define SISL_AFU_RC_DATA_DMA_ERR 0x31u /* see afu_extra @@ -192,6 +209,8 @@ struct sisl_rc { reported len, possbly due to dropped frames */ #define SISL_FC_RC_TGTABORT 0x5C /* command aborted by target */ +#define SISL_FC_RC_SHUTDOWN 0x5E /* command aborted by AFU due to */ + /* shutdown. */ }; @@ -278,6 +297,13 @@ struct sisl_host_map { __u64 cmd_room; __u64 ctx_ctrl; /* least signiifcant byte or b56:63 is LISN# */ __u64 mbox_w; /* restricted use */ + __u64 sq_start; /* Submission queue (SQ) start register */ + __u64 sq_end; /* Submission queue (SQ) end register */ + __u64 sq_head; /* Submission queue (SQ) head register */ + __u64 sq_tail; /* Submission queue (SQ) tail register */ + __u64 sq_context_reset; /* Submission queue (SQ) context reset + register. */ + }; /* per context provisioning & control MMIO */ diff --git a/src/include/ticks.h b/src/include/ticks.h new file mode 100644 index 00000000..b19e8bf0 --- /dev/null +++ b/src/include/ticks.h @@ -0,0 +1,169 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/include/ticks.h $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#ifndef _TICKS_H__ +#define _TICKS_H__ +#include +#include +#include + +typedef uint64_t ticks; + +static __inline__ ticks getticks(void); + +/** + ******************************************************************************* + * \brief + * seconds delta of current ticks and sticks + ******************************************************************************/ +static __inline__ uint32_t SDELTA(ticks sticks, double ns) +{ + return (uint32_t)((((getticks())-sticks) * ns) / 1000000000); +} + +/** + ******************************************************************************* + * \brief + * milliseconds delta of current ticks and sticks + ******************************************************************************/ +static __inline__ uint64_t MDELTA(ticks sticks, double ns) +{ + return (((getticks())-sticks) * ns) / 1000000; +} + +/** + ******************************************************************************* + * \brief + * microseconds delta of current ticks and sticks + ******************************************************************************/ +static __inline__ uint64_t UDELTA(ticks sticks, double ns) +{ + return (((getticks())-sticks) * ns) / 1000; +} + +/** + ******************************************************************************* + * \brief + * nanoseconds delta of current ticks and sticks + ******************************************************************************/ +static __inline__ uint64_t NDELTA(ticks sticks, double ns) +{ + return (((getticks())-sticks) * ns); +} + +/** + ******************************************************************************* + * \brief + ******************************************************************************/ +#if defined(__powerpc__) || defined(__ppc__) +static __inline__ ticks getticks(void) +{ + unsigned int x, x0, x1; + do { + __asm__ __volatile__ ("mftbu %0" : "=r"(x0)); + __asm__ __volatile__ ("mftb %0" : "=r"(x)); + __asm__ __volatile__ ("mftbu %0" : "=r"(x1)); + } while (x0 != x1); + + return (((uint64_t)x0) << 32) | x; +} +#endif + +/** + ******************************************************************************* + * \brief + ******************************************************************************/ +#if defined(__x86_64__) +static __inline__ ticks getticks(void) +{ + uint32_t x, y; + asm volatile("rdtsc" : "=a" (x), "=d" (y)); + return ((ticks)x) | (((ticks)y) << 32); +} +#endif + +/** + ******************************************************************************* + * \brief + ******************************************************************************/ +static __inline__ double elapsed(ticks t1, ticks t0) +{ + return (double)t1 - (double)t0; +} + +/** + ******************************************************************************* + * \brief + ******************************************************************************/ +static __inline__ double time_diff(struct timeval x , struct timeval y) +{ + double x_ms , y_ms , diff; + + x_ms = (double)x.tv_sec*1000000 + (double)x.tv_usec; + y_ms = (double)y.tv_sec*1000000 + (double)y.tv_usec; + + diff = (double)y_ms - (double)x_ms; + + return diff; +} + +/** + ******************************************************************************* + * \brief + * return nanoseconds per tick + ******************************************************************************/ +static __inline__ double time_per_tick(int n, int del) +{ + double *td = malloc(n * sizeof(double)); + double *tv = malloc(n * sizeof(double)); + struct timeval tvs; + struct timeval tve; + ticks ts; + ticks te; + int i; + + for (i=0; iark; +ARK * ark; Description ----------- @@ -117,19 +117,19 @@ int ark\_set(ark, klen, key, vlen, val, res) int ark\_set\_async\_cb(ark, klen, key, vlen, val, callback, dt) -**ARK \*** ark; +ARK * ark; -**uint64\_t** klen; +uint64\_t klen; -**void \*** key; +void * key; -**uint64\_t** vlen; +uint64\_t vlen; -**void \*** val; +void * val; -**void \***(\*callback)(int errcode, uint64\_t dt, uint64\_t res); +void *(*callback)(int errcode, uint64\_t dt, uint64\_t res); -**uint64\_t** dt; +uint64\_t dt; Description ----------- @@ -182,21 +182,21 @@ int ark\_get(ark, klen, key, vbuflen, vbuf, voff, res) int ark\_get\_async\_cb(ark, klen, key, vbuflen, vbuf, voff, callback, dt) -**ARK \*** ark; +ARK * ark; -**uint64\_t** klen; +uint64\_t klen; -**void \*** key; +void * key; -**uint64\_t** vbuflen; +uint64\_t vbuflen; -**void \*** vbuf; +void * vbuf; -**uint64\_t** voff; +uint64\_t voff; -**void \***(\*callback)(int errcode, uint64\_t dt, uint64\_t res); +void *(\*callback)(int errcode, uint64\_t dt, uint64\_t res); -**uint64\_t** dt; +uint64\_t dt; Description ----------- @@ -248,15 +248,15 @@ int ark\_del(ark, klen, key, res) int ark\_del\_async\_cb(ark, klen, key, callback, dt) -**ARK \*** ark +ARK * ark; -**uint64\_t** klen; +uint64\_t klen; -**void \*** key; +void * key; -**void \***(\*callback)(int errcode, uint64\_t dt, uint64\_t res); +void *(\*callback)(int errcode, uint64\_t dt, uint64\_t res); -**uint64\_t** dt; +uint64\_t dt; Description ----------- @@ -305,15 +305,16 @@ int ark\_exists(ark, klen, key, res) int ark\_exists\_async\_cb(ark, klen, key, callback, dt) -**ARK \*** ark; +ARK * ark; -**uint64\_t** klen; +uint64\_t klen; -**void \*** key; +void * key; -**void \***(\*callback)(int errcode, uint64\_t dt, uint64\_t res); +void *(\*callback)(int errcode, uint64\_t dt, uint64\_t res); + +uint64\_t dt; -**uint64\_t** dt; Description ----------- @@ -358,13 +359,14 @@ Syntax ARI \* ark\_first(ark, kbuflen, klen, kbuf) -**ARK \* ark**; +ARK * ark; + +uint64\_t kbuflen -**uint64\_t** kbuflen +int64\_t klen; -**int64\_t** \*klen; +void * kbuf; -**void \*** kbuf; Description ----------- @@ -409,13 +411,14 @@ Syntax ARI \* ark\_next(iter, kbuflen, klen, kbuf) -**ARI \*** iter; +ARI * iter; -**uint64\_t** kbuflen +uint64\_t kbuflen -**int64\_t** \*klen; +int64\_t klen; + +void * kbuf; -**void \*** kbuf; Description ----------- @@ -462,9 +465,9 @@ Syntax int ark\_allocated(ark, size) -**ARK \*** ark; +ARK * ark; -**uint64\_t \***size; +uint64\_t *size; Description ----------- @@ -503,9 +506,9 @@ Syntax int ark\_inuse(ark, size) -**ARK \*** ark; +ARK *ark; -**uint64\_t \***size; +uint64\_t *size; Description ----------- @@ -544,9 +547,10 @@ Syntax int ark\_actual(ark, size) -**ARK \*** ark; +ARK *ark; + +uint64\_t *size; -**uint64 \*** size; Description ----------- @@ -585,11 +589,11 @@ Syntax int ark\_error(ark) -**ARK \* ark**; +ARK * ark; -char \* ark\_errorstring(ark) +char * ark\_errorstring(ark) -**ARK \* ark**; +ARK * ark; Description ----------- @@ -631,7 +635,7 @@ int ark\_fork(ark) int ark\_fork\_done(ark) -**ARK \*** handle; +ARK * handle; Description ----------- @@ -675,13 +679,13 @@ Syntax int ark\_random(ark, kbuflen, klen, kbuf) -**ARK \*** ark; +ARK * ark; -**uint64\_t** kbuflen +uint64\_t kbuflen; -**int64\_t** \*klen; +int64\_t * klen; -**void \*** kbuf; +void * kbuf; Description ----------- @@ -722,9 +726,9 @@ Syntax int ark\_count(ark, int \*count) -**ARK \*** ark; +ARK * ark; -**int \*** count; +int * count; Description ----------- diff --git a/src/kv/am.c b/src/kv/am.c index 9fefc4de..e6d9512a 100644 --- a/src/kv/am.c +++ b/src/kv/am.c @@ -37,7 +37,6 @@ #include int x = 1; -#define H 64 #define ARK_KV_ALIGN 16 void *am_malloc(size_t size) @@ -49,22 +48,26 @@ void *am_malloc(size_t size) if ((int64_t)size > 0) { #ifdef _AIX - p = malloc(size + (ARK_KV_ALIGN - 1)); + p = malloc(size + ARK_KV_ALIGN); #else - p = zmalloc(size + (ARK_KV_ALIGN - 1)); + p = zmalloc(size + ARK_KV_ALIGN); #endif - if (p) {memset(p,0x00, size + (ARK_KV_ALIGN - 1));} + if (p) {memset(p,0x00, size + ARK_KV_ALIGN);} else { KV_TRC_FFDC(pAT, "ALLOC_FAIL errno=%d", errno); if (!errno) {errno = ENOMEM;} } } + KV_TRC_DBG(pAT, "ALLOC %p size:%ld up:%ld", p, size,size+ARK_KV_ALIGN); return p; } void am_free(void *ptr) { + if ((uint64_t)ptr < 0xFFFF) {KV_TRC_FFDC(pAT, "FREE BAD:%p", ptr); return;} + else {KV_TRC_DBG (pAT, "FREE %p", ptr);} + #ifdef _AIX if (ptr) {free(ptr); ptr=NULL;} #else @@ -81,9 +84,9 @@ void *am_realloc(void *ptr, size_t size) if (ptr && (int64_t)size > 0) { #ifdef _AIX - p = realloc(ptr, size + (ARK_KV_ALIGN - 1)); + p = realloc(ptr, size + ARK_KV_ALIGN); #else - p = zrealloc(ptr, size + (ARK_KV_ALIGN - 1)); + p = zrealloc(ptr, size + ARK_KV_ALIGN); #endif } else @@ -97,6 +100,7 @@ void *am_realloc(void *ptr, size_t size) KV_TRC_FFDC(pAT, "REALLOC_FAIL errno=%d", errno); if (!errno) {errno = ENOMEM;} } + KV_TRC_DBG(pAT, "REALLOC old:%p new:%p size:%ld up:%ld", ptr, p, size,size+ARK_KV_ALIGN); return p; } diff --git a/src/kv/ark.h b/src/kv/ark.h index 5e87a917..4f367b60 100644 --- a/src/kv/ark.h +++ b/src/kv/ark.h @@ -34,17 +34,47 @@ #include "tag.h" #include "queue.h" #include "sq.h" +#include "ut.h" #define PT_OFF 0 #define PT_IDLE 1 #define PT_RUN 2 #define PT_EXIT 3 +#define ARK_VERB_FN "/root/kv_verbosity" + #define DIVCEIL(_x, _y) (((_x) / (_y)) + (((_x) % (_y)) ? 0 : 1)) +#define UPDATE_LAT(park, prcb, _lat) \ +do \ +{ \ + if (prcb->cmd == K_SET) \ + { \ + park->set_opsT += 1; \ + park->set_latT += _lat; \ + } \ + else if (prcb->cmd == K_GET) \ + { \ + park->get_opsT += 1; \ + park->get_latT += _lat; \ + } \ + else if (prcb->cmd == K_EXI) \ + { \ + park->exi_opsT += 1; \ + park->exi_latT += _lat; \ + } \ + else if (prcb->cmd == K_DEL) \ + { \ + park->del_opsT += 1; \ + park->del_latT += _lat; \ + } \ +} \ +while (0); + // Stats to collect on number of K/V ops // and IOs -typedef struct ark_stats { +typedef struct ark_stats +{ volatile uint64_t ops_cnt; volatile uint64_t io_cnt; volatile uint64_t kv_cnt; @@ -102,16 +132,21 @@ typedef struct p_ark #define ARK_VERBOSE_SIZE_DEF 1048576 #define ARK_VERBOSE_BSIZE_DEF 4096 -#define ARK_VERBOSE_HASH_DEF 1048576 +#define ARK_VERBOSE_HASH_DEF 200000 #define ARK_VERBOSE_VLIMIT_DEF 256 #define ARK_VERBOSE_BLKBITS_DEF 34 #define ARK_VERBOSE_GROW_DEF 1024 #define ARK_VERBOSE_NTHRDS_DEF 1 -#define ARK_MAX_ASYNC_OPS 128 -#define ARK_MAX_TASK_OPS 48 +#define ARK_MIN_NASYNCS 128 +#define ARK_MAX_NASYNCS 512 +#define ARK_MIN_BASYNCS 1024 +#define ARK_MAX_TASK_OPS 64 +#define ARK_MAX_QUEUE 8196 +#define ARK_MIN_VB 256 -typedef struct _ark { +typedef struct _ark +{ uint64_t flags; uint32_t persload; uint64_t pers_cs_bytes; // amt required for only the Control Structures @@ -127,9 +162,6 @@ typedef struct _ark { uint64_t blkused; uint64_t nasyncs; uint64_t ntasks; - - ark_stats_t pers_stats; - uint32_t holds; int nthrds; @@ -143,8 +175,8 @@ typedef struct _ark { pthread_mutex_t mainmutex; - hash_t *ht; // hashtable - BL *bl; // block lists + hash_t *ht; // hashtable + BL *bl; // block lists struct _ea *ea; // in memory store space struct _rcb *rcbs; struct _tcb *tcbs; @@ -154,15 +186,31 @@ typedef struct _ark { struct _tags *rtags; struct _tags *ttags; + uint32_t issT; + uint64_t QDT; + uint64_t QDN; + uint64_t set_latT; + uint64_t get_latT; + uint64_t exi_latT; + uint64_t del_latT; + uint64_t set_opsT; + uint64_t get_opsT; + uint64_t exi_opsT; + uint64_t del_opsT; + double ns_per_tick; + ark_stats_t pers_stats; + } _ARK; #define _ARC _ARK -typedef struct _scb { +typedef struct _scb +{ pthread_t pooltid; queue_t *rqueue; queue_t *tqueue; - queue_t *ioqueue; + queue_t *scheduleQ; + queue_t *harvestQ; int32_t poolstate; int32_t rlast; int32_t dogc; @@ -173,7 +221,8 @@ typedef struct _scb { } scb_t; // pool thread gets its id and the database struct -typedef struct _pt { +typedef struct _pt +{ int id; _ARK *ark; } PT; @@ -217,13 +266,15 @@ typedef struct _rcb { int32_t sthrd; int32_t cmd; int32_t stat; + uint64_t stime; void (*cb)(int errcode, uint64_t dt,int64_t res); pthread_cond_t acond; pthread_mutex_t alock; } rcb_t; -typedef struct _ari { +typedef struct _ari +{ _ARK *ark; int64_t hpos; int64_t key; @@ -256,7 +307,8 @@ typedef struct _ari { #define ARK_NEXT_START 20 // operation -typedef struct _tcb { +typedef struct _tcb +{ int32_t rtag; int32_t ttag; int32_t state; @@ -264,14 +316,14 @@ typedef struct _tcb { uint64_t inblen; uint64_t oublen; - BTP inb; // input bucket space - aligned + BTP inb; // input bucket space - aligned BTP inb_orig; // input bucket space - BTP oub; // output bucket space - aligned + BTP oub; // output bucket space - aligned BTP oub_orig; // output bucket space uint64_t vbsize; - uint8_t *vb; // value buffer space - aligned - uint8_t *vb_orig; // value buffer space - uint8_t *vval; // value buffer space + uint8_t *vb; // value buffer space - aligned + uint8_t *vb_orig; // value buffer space + uint8_t *vval; // value buffer space uint64_t vblkcnt; int64_t vblk; uint64_t hpos; @@ -296,7 +348,11 @@ typedef struct _iocb int32_t tag; uint32_t issT; uint32_t cmpT; + uint32_t rd; + uint32_t lat; + uint64_t stime; uint32_t hmissN; + uint32_t aflags; uint32_t io_error; uint32_t io_done; } iocb_t; diff --git a/src/kv/arkdb.c b/src/kv/arkdb.c index 60767cd4..1007e4bf 100644 --- a/src/kv/arkdb.c +++ b/src/kv/arkdb.c @@ -33,16 +33,17 @@ REVISION_TAGS(arkdb); #include #include #include +#include #include "am.h" #include "ut.h" #include "tag.h" #include "vi.h" #include "ea.h" #include "bl.h" - #include "arkdb.h" #include "ark.h" #include "arp.h" +#include #ifdef _OS_INTERNAL #include @@ -50,7 +51,7 @@ REVISION_TAGS(arkdb); #include "capiblock.h" #endif -#include +#include #include #include @@ -418,6 +419,7 @@ int ark_create_verbose(char *path, ARK **arkret, uint64_t size, uint64_t bsize, uint64_t hcount, int nthrds, int nqueue, int basyncs, uint64_t flags) { + _ARK *ark = NULL; int rc = 0; int p_rc = 0; uint64_t bcount = 0; @@ -452,24 +454,29 @@ int ark_create_verbose(char *path, ARK **arkret, goto ark_create_ark_err; } - _ARK *ark = am_malloc(sizeof(_ARK)); - if (ark == NULL) { + ark = am_malloc(sizeof(_ARK)); + if (ark == NULL) + { rc = ENOMEM; KV_TRC_FFDC(pAT, "Out of memory allocating ARK control structure for %ld", sizeof(_ARK)); goto ark_create_ark_err; } + ark->ns_per_tick = time_per_tick(1000, 100); + KV_TRC(pAT, "%p path(%s) size %ld bsize %ld hcount %ld " "nthrds %d nqueue %d basyncs %d flags:%08lx", ark, path, size, bsize, hcount, nthrds, nqueue, basyncs, flags); + nqueue = nqueuebsize = bsize; ark->rthread = 0; ark->persload = 0; - ark->nasyncs = ((nqueue <= 0) ? ARK_MAX_ASYNC_OPS : nqueue); - ark->basyncs = basyncs; + ark->nasyncs = nqueuebasyncs = basyncsntasks = ARK_MAX_TASK_OPS; ark->nthrds = ARK_VERBOSE_NTHRDS_DEF; // hardcode, perf requirement @@ -623,7 +630,7 @@ int ark_create_verbose(char *path, ARK **arkret, for ( tnum = 0; tnum < ark->ntasks; tnum++ ) { - ark->tcbs[tnum].inb = bt_new(0, ark->vlimit, sizeof(uint64_t), + ark->tcbs[tnum].inb = bt_new(bsize, ark->vlimit, sizeof(uint64_t), &(ark->tcbs[tnum].inblen), &(ark->tcbs[tnum].inb_orig)); if (ark->tcbs[tnum].inb == NULL) @@ -634,7 +641,7 @@ int ark_create_verbose(char *path, ARK **arkret, goto ark_create_taskloop_err; } - ark->tcbs[tnum].oub = bt_new(0, ark->vlimit, sizeof(uint64_t), + ark->tcbs[tnum].oub = bt_new(bsize, ark->vlimit, sizeof(uint64_t), &(ark->tcbs[tnum].oublen), &(ark->tcbs[tnum].oub_orig)); if (ark->tcbs[tnum].oub == NULL) @@ -645,8 +652,7 @@ int ark_create_verbose(char *path, ARK **arkret, goto ark_create_taskloop_err; } - //ark->tcbs[tnum].vbsize = bsize * 1024; - ark->tcbs[tnum].vbsize = bsize * 256; + ark->tcbs[tnum].vbsize = bsize * ARK_MIN_VB; ark->tcbs[tnum].vb_orig = am_malloc(ark->tcbs[tnum].vbsize); if (ark->tcbs[tnum].vb_orig == NULL) { @@ -689,9 +695,10 @@ int ark_create_verbose(char *path, ARK **arkret, pthread_mutex_init(&(scbp->poolmutex), NULL); pthread_cond_init(&(scbp->poolcond), NULL); - scbp->rqueue = queue_new(ark->nasyncs); - scbp->tqueue = queue_new(ark->ntasks); - scbp->ioqueue = queue_new(ark->ntasks); + scbp->rqueue = queue_new(ark->nasyncs); + scbp->tqueue = queue_new(ark->ntasks); + scbp->scheduleQ = queue_new(ARK_MAX_QUEUE); + scbp->harvestQ = queue_new(ARK_MAX_QUEUE); pt->id = i; pt->ark = ark; @@ -703,13 +710,6 @@ int ark_create_verbose(char *path, ARK **arkret, } } -#if 0 - while (ark->nactive < ark->nthrds) { - usleep(1); - //printf("Create waiting %d/%d\n", ark->nactive, ark->nthrds); - } -#endif - ark->pcmd = PT_RUN; goto ark_create_return; @@ -730,20 +730,10 @@ int ark_create_verbose(char *path, ARK **arkret, pthread_mutex_destroy(&(scbp->poolmutex)); pthread_cond_destroy(&(scbp->poolcond)); - if ( scbp->rqueue != NULL ) - { - queue_free(scbp->rqueue); - } - - if ( scbp->tqueue != NULL ) - { - queue_free(scbp->tqueue); - } - - if ( scbp->ioqueue != NULL ) - { - queue_free(scbp->ioqueue); - } + if (scbp->rqueue != NULL) {queue_free(scbp->rqueue);} + if (scbp->tqueue != NULL) {queue_free(scbp->tqueue);} + if (scbp->scheduleQ != NULL) {queue_free(scbp->scheduleQ);} + if (scbp->harvestQ != NULL) {queue_free(scbp->harvestQ);} } } @@ -753,79 +743,81 @@ int ark_create_verbose(char *path, ARK **arkret, } ark_create_taskloop_err: + KV_TRC_DBG(pAT, "free inb/oub/vb"); for ( tnum = 0; tnum < ark->ntasks; tnum++ ) { - if (ark->tcbs[tnum].inb) - { - bt_delete(ark->tcbs[tnum].inb); - } - - if (ark->tcbs[tnum].oub) - { - bt_delete(ark->tcbs[tnum].oub); - } - - if (ark->tcbs[tnum].vb_orig) - { - am_free(ark->tcbs[tnum].vb_orig); - } + bt_delete(ark->tcbs[tnum].inb_orig); + bt_delete(ark->tcbs[tnum].oub_orig); + am_free (ark->tcbs[tnum].vb_orig); } + KV_TRC_DBG(pAT, "destroy rcb cond/lock"); for (rnum = 0; rnum < ark->nasyncs; rnum++) { pthread_cond_destroy(&(ark->rcbs[rnum].acond)); pthread_mutex_destroy(&(ark->rcbs[rnum].alock)); } + KV_TRC_DBG(pAT, "free pool threads"); if ( ark->poolthreads != NULL ) { am_free(ark->poolthreads); } ark_create_poolthreads_err: + KV_TRC_DBG(pAT, "free iocbs"); if (ark->iocbs) { am_free(ark->iocbs); } ark_create_iocbs_err: + KV_TRC_DBG(pAT, "free tcbs"); if (ark->tcbs) { am_free(ark->tcbs); } ark_create_tcbs_err: + KV_TRC_DBG(pAT, "free rcbs"); if (ark->rcbs) { am_free(ark->rcbs); } ark_create_rcbs_err: + KV_TRC_DBG(pAT, "free ttags"); if (ark->ttags) { tag_free(ark->ttags); } ark_create_ttag_err: + KV_TRC_DBG(pAT, "free rtags"); if (ark->rtags) { tag_free(ark->rtags); } ark_create_rtag_err: + KV_TRC_DBG(pAT, "destroy mainmutex"); pthread_mutex_destroy(&ark->mainmutex); ark_create_pth_mutex_err: + KV_TRC_DBG(pAT, "bl_delete"); bl_delete(ark->bl); ark_create_bl_err: + KV_TRC_DBG(pAT, "free ht"); hash_free(ark->ht); ark_create_ht_err: ark_create_persist_err: + KV_TRC_DBG(pAT, "ea_delete"); ea_delete(ark->ea); ark_create_ea_err: + KV_TRC_DBG(pAT, "free ark"); am_free(ark); *arkret = NULL; @@ -833,6 +825,10 @@ int ark_create_verbose(char *path, ARK **arkret, KV_TRC_CLOSE(pAT); ark_create_return: + KV_TRC(pAT, "DONE: %p path(%s) size %ld bsize %ld hcount %ld ntasks:%ld " + "nthrds %d nqueue %ld basyncs %d flags:%08lx", + ark, path, size, bsize, hcount, ark->ntasks, + nthrds, ark->nasyncs, ark->basyncs, flags); return rc; } @@ -843,12 +839,13 @@ int ark_create(char *path, ARK **arkret, uint64_t flags) ARK_VERBOSE_BSIZE_DEF, ARK_VERBOSE_HASH_DEF, ARK_VERBOSE_NTHRDS_DEF, - ARK_MAX_ASYNC_OPS, + ARK_MAX_NASYNCS, ARK_EA_BLK_ASYNC_CMDS, flags); } -int ark_delete(ARK *ark) { +int ark_delete(ARK *ark) +{ int rc = 0; int i = 0; _ARK *_arkp = (_ARK *)ark; @@ -861,6 +858,8 @@ int ark_delete(ARK *ark) { goto ark_delete_ark_err; } + KV_TRC_DBG(pAT, "free thread resources"); + // Wait for all active threads to exit for (i = 0; i < _arkp->nthrds; i++) { @@ -875,55 +874,49 @@ int ark_delete(ARK *ark) { queue_free(scbp->rqueue); queue_free(scbp->tqueue); - queue_free(scbp->ioqueue); + queue_free(scbp->scheduleQ); + queue_free(scbp->harvestQ); pthread_mutex_destroy(&(scbp->poolmutex)); pthread_cond_destroy(&(scbp->poolcond)); KV_TRC(pAT, "thread %d joined", i); } - if (_arkp->poolthreads) am_free(_arkp->poolthreads); + KV_TRC_DBG(pAT, "free arkp->poolthreads"); + if (_arkp->poolthreads) {am_free(_arkp->poolthreads);} - if (_arkp->pts) am_free(_arkp->pts); + KV_TRC_DBG(pAT, "free arkp->pts"); + if (_arkp->pts) {am_free(_arkp->pts);} + KV_TRC_DBG(pAT, "destroy rcb cond/lock"); for ( i = 0; i < _arkp->nasyncs ; i++ ) { pthread_cond_destroy(&(_arkp->rcbs[i].acond)); pthread_mutex_destroy(&(_arkp->rcbs[i].alock)); } + KV_TRC_DBG(pAT, "free inb/oub/vb"); for ( i = 0; i < _arkp->ntasks; i++ ) { - - bt_delete(_arkp->tcbs[i].inb); - bt_delete(_arkp->tcbs[i].oub); - am_free(_arkp->tcbs[i].vb_orig); + bt_delete(_arkp->tcbs[i].inb_orig); + bt_delete(_arkp->tcbs[i].oub_orig); + am_free (_arkp->tcbs[i].vb_orig); } - if (_arkp->iocbs) - { - am_free(_arkp->iocbs); - } + KV_TRC_DBG(pAT, "free iocbs"); + am_free(_arkp->iocbs); - if (_arkp->tcbs) - { - am_free(_arkp->tcbs); - } + KV_TRC_DBG(pAT, "free tcbs"); + am_free(_arkp->tcbs); - if (_arkp->rcbs) - { - am_free(_arkp->rcbs); - } + KV_TRC_DBG(pAT, "free rcbs"); + am_free(_arkp->rcbs); - if (_arkp->ttags) - { - tag_free(_arkp->ttags); - } + KV_TRC_DBG(pAT, "free ttags"); + if (_arkp->ttags) {tag_free(_arkp->ttags);} - if (_arkp->rtags) - { - tag_free(_arkp->rtags); - } + KV_TRC_DBG(pAT, "free rtags"); + if (_arkp->rtags) {tag_free(_arkp->rtags);} if (!(_arkp->flags & ARK_KV_VIRTUAL_LUN)) { @@ -934,12 +927,19 @@ int ark_delete(ARK *ark) { } } + KV_TRC_DBG(pAT, "destroy mainmutex"); pthread_mutex_destroy(&_arkp->mainmutex); + KV_TRC_DBG(pAT, "ea_delete"); (void)ea_delete(_arkp->ea); + + KV_TRC_DBG(pAT, "free ht"); hash_free(_arkp->ht); + + KV_TRC_DBG(pAT, "bl_delete"); bl_delete(_arkp->bl); - KV_TRC(pAT, "ark_delete done %p", _arkp); + + KV_TRC(pAT, "done, free arkp %p", _arkp); am_free(_arkp); ark_delete_ark_err: @@ -993,8 +993,9 @@ int ark_set(ARK *ark, uint64_t klen, ((vlen > 0) && (val == NULL)) || (rval == NULL)) { - KV_TRC_FFDC(pAT, "rc = EINVAL: ark %p, klen %"PRIu64",\ - key %p, vlen %"PRIu64", val %p, rval %p", ark, klen, key, vlen, val, rval); + KV_TRC_FFDC(pAT, "FFDC: EINVAL: ark %p klen %"PRIu64" " + "key %p vlen %"PRIu64" val %p rval %p", + ark, klen, key, vlen, val, rval); rc = EINVAL; } else @@ -1004,7 +1005,7 @@ int ark_set(ARK *ark, uint64_t klen, if (tag < 0) { rc = EAGAIN; - KV_TRC_FFDC(pAT, "ark_set_async_tag failed rc = %d", rc); + KV_TRC_FFDC(pAT, "EAGAIN: ark_set_async_tag"); } else { @@ -1014,7 +1015,7 @@ int ark_set(ARK *ark, uint64_t klen, { if (errcode != 0) { - KV_TRC_FFDC(pAT, "ark_wait_tag failed rc = %d", errcode); + KV_TRC_FFDC(pAT, "FFDC: ark_wait_tag failed rc = %d", errcode); rc = errcode; } @@ -1039,8 +1040,8 @@ int ark_exists(ARK *ark, uint64_t klen, void *key, int64_t *rval) { ((klen > 0) && (key == NULL)) || (rval == NULL)) { - KV_TRC_FFDC(pAT, "rc = EINVAL: ark %p, klen %"PRIu64",\ - key %p, rval %p", ark, klen, key, rval); + KV_TRC_FFDC(pAT, "FFDC: rc:EINVAL ark %p, klen %"PRIu64" " + "key %p, rval %p", ark, klen, key, rval); rc = EINVAL; } else @@ -1049,8 +1050,8 @@ int ark_exists(ARK *ark, uint64_t klen, void *key, int64_t *rval) { tag = ark_exists_async_tag(_arkp, klen, key); if (tag < 0) { - KV_TRC_FFDC(pAT, "ark_set_async_tag failed rc = %d", rc); rc = EAGAIN; + KV_TRC_FFDC(pAT, "EAGAIN: ark_exists_async_tag"); } else { @@ -1060,7 +1061,7 @@ int ark_exists(ARK *ark, uint64_t klen, void *key, int64_t *rval) { { if (errcode != 0) { - KV_TRC_FFDC(pAT, "ark_wait_tag failed rc = %d", errcode); + KV_TRC_FFDC(pAT, "FFDC: ark_wait_tag failed rc = %d", errcode); rc = errcode; } @@ -1089,9 +1090,9 @@ int ark_get(ARK *ark, uint64_t klen, void *key, ((vbuflen > 0) && (vbuf == NULL)) || (rval == NULL)) { - KV_TRC_FFDC(pAT, "rc = EINVAL: ark %p, klen %"PRIu64", key %p, \ -vbuflen %"PRIu64", vbuf %p, rval %p", - _arkp, klen, key, vbuflen, vbuf, rval); + KV_TRC_FFDC(pAT, "FFDC: EINVAL: ark %p klen %"PRIu64" key %p " + "vbuflen %"PRIu64" vbuf %p rval %p", + _arkp, klen, key, vbuflen, vbuf, rval); rc = EINVAL; } else @@ -1100,7 +1101,7 @@ vbuflen %"PRIu64", vbuf %p, rval %p", tag = ark_get_async_tag(_arkp, klen, key, vbuflen, vbuf, voff); if (tag < 0) { - KV_TRC_FFDC(pAT, "ark_get_async_tag failed rc = %d", rc); + KV_TRC_FFDC(pAT, "EAGAIN: ark_get_async_tag failed"); rc = EAGAIN; } else @@ -1111,7 +1112,7 @@ vbuflen %"PRIu64", vbuf %p, rval %p", { if (errcode != 0) { - KV_TRC_FFDC(pAT, "ark_wait_tag failed rc = %d", errcode); + KV_TRC_FFDC(pAT, "FFDC: ark_wait_tag failed rc = %d", errcode); rc = errcode; } @@ -1135,8 +1136,8 @@ int ark_del(ARK *ark, uint64_t klen, void *key, int64_t *rval) { ((klen > 0) && (key == NULL)) || (rval == NULL)) { - KV_TRC_FFDC(pAT, "rc = EINVAL: ark %p, klen %"PRIu64", key %p, rval %p", - _arkp, klen, key, rval); + KV_TRC_FFDC(pAT, "FFDC: EINVAL: ark %p klen %"PRIu64" key %p rval %p", + _arkp, klen, key, rval); rc = EINVAL; } else @@ -1145,7 +1146,7 @@ int ark_del(ARK *ark, uint64_t klen, void *key, int64_t *rval) { tag = ark_del_async_tag(_arkp, klen, key); if (tag < 0) { - KV_TRC_FFDC(pAT, "ark_del_async_tag failed rc = %d", rc); + KV_TRC_FFDC(pAT, "EAGAIN: ark_del_async_tag"); rc = EAGAIN; } else @@ -1156,7 +1157,7 @@ int ark_del(ARK *ark, uint64_t klen, void *key, int64_t *rval) { { if (errcode != 0) { - KV_TRC_FFDC(pAT, "ark_wait_tag failed rc = %d", errcode); + KV_TRC_FFDC(pAT, "FFDC: ark_wait_tag failed rc = %d", errcode); rc = errcode; } @@ -1673,7 +1674,8 @@ pid_t ark_fork(ARK *ark) { int c_rc = 0; - c_rc = cblk_clone_after_fork(_arkp->ea->st_flash, O_RDONLY, 0); + c_rc = cblk_cg_clone_after_fork(_arkp->ea->st_flash, + O_RDONLY, CBLK_GROUP_RAID0); // If we encountered an error, force the child to // exit with a non-zero status code diff --git a/src/kv/arp.c b/src/kv/arp.c index 6a4b300f..9afdecf9 100644 --- a/src/kv/arp.c +++ b/src/kv/arp.c @@ -33,6 +33,7 @@ #include #include +#include #include #include "ct.h" @@ -42,6 +43,7 @@ #include "ark.h" #include "arp.h" #include "am.h" +#include #include #include @@ -65,17 +67,17 @@ int ark_wait_tag(_ARK *_arkp, int tag, int *errcode, int64_t *res) if (rcbp->stat == A_COMPLETE) { - *res = rcbp->res; - *errcode = rcbp->rc; - rcbp->stat = A_NULL; - tag_bury(_arkp->rtags, tag); + *res = rcbp->res; + *errcode = rcbp->rc; + rcbp->stat = A_NULL; + tag_bury(_arkp->rtags, tag); } else { rc = EINVAL; KV_TRC_FFDC(pAT, "tag = %d rc = %d", tag, rc); } - + pthread_mutex_unlock(&(rcbp->alock)); return rc; @@ -85,24 +87,24 @@ int ark_wait_tag(_ARK *_arkp, int tag, int *errcode, int64_t *res) ******************************************************************************* * \brief ******************************************************************************/ -int ark_anyreturn(_ARK *_arkp, int *tag, int64_t *res) { +int ark_anyreturn(_ARK *_arkp, int *tag, int64_t *res) +{ int i; int astart = _arkp->astart; int nasyncs = _arkp->nasyncs; int itag = -1; - for (i = 0; i < _arkp->nasyncs; i++) { - itag = (astart + i) % nasyncs; - if (_arkp->rcbs[itag].stat == A_COMPLETE) { - //KV_TRC_IO(pAT, pAT, "Found completion tag %d", itag); - *tag = itag; - *res = _arkp->rcbs[(astart+i) % nasyncs].res; - KV_TRC_IO(pAT, "A_COMPLETE: arp %p res %"PRIi64"", _arkp->rcbs+itag, - _arkp->rcbs[itag].res); - _arkp->rcbs[itag].stat = A_NULL; - tag_bury(_arkp->rtags, itag); - return 0; - } + for (i = 0; i < _arkp->nasyncs; i++) + { + itag = (astart + i) % nasyncs; + if (_arkp->rcbs[itag].stat == A_COMPLETE) + { + *tag = itag; + *res = _arkp->rcbs[(astart+i) % nasyncs].res; + _arkp->rcbs[itag].stat = A_NULL; + tag_bury(_arkp->rtags, itag); + return 0; + } } _arkp->astart++; _arkp->astart %= nasyncs; @@ -193,9 +195,9 @@ void ark_drop_pool(_ARK *_arkp, ark_stats_t *stats, uint64_t blk) ******************************************************************************* * \brief ******************************************************************************/ -int ark_enq_cmd(int cmd, _ARK *_arkp, uint64_t klen, void *key, +int ark_enq_cmd(int cmd, _ARK *_arkp, uint64_t klen, void *key, uint64_t vlen,void *val,uint64_t voff, - void (*cb)(int errcode, uint64_t dt,int64_t res), + void (*cb)(int errcode, uint64_t dt,int64_t res), uint64_t dt, int pthr, int *ptag) { int32_t rtag = -1; @@ -214,31 +216,32 @@ int ark_enq_cmd(int cmd, _ARK *_arkp, uint64_t klen, void *key, rc = tag_unbury(_arkp->rtags, &rtag); if (rc == 0) { - rcbp = &(_arkp->rcbs[rtag]); - rcbp->ark = _arkp; - rcbp->cmd = cmd; - rcbp->rtag = rtag; - rcbp->ttag = -1; - rcbp->stat = A_INIT; - rcbp->klen = klen; - rcbp->key = key; - rcbp->vlen = vlen; - rcbp->val = val; - rcbp->voff = voff; - rcbp->cb = cb; - rcbp->dt = dt; - rcbp->rc = 0; - rcbp->hold = -1; - rcbp->res = -1; + rcbp = &(_arkp->rcbs[rtag]); + rcbp->stime = getticks(); + rcbp->ark = _arkp; + rcbp->cmd = cmd; + rcbp->rtag = rtag; + rcbp->ttag = -1; + rcbp->stat = A_INIT; + rcbp->klen = klen; + rcbp->key = key; + rcbp->vlen = vlen; + rcbp->val = val; + rcbp->voff = voff; + rcbp->cb = cb; + rcbp->dt = dt; + rcbp->rc = 0; + rcbp->hold = -1; + rcbp->res = -1; // If the caller has asked to run on a specific thread, then // do so. Otherwise, use the key to find which thread // will handle command if (pthr == -1) { - rcbp->pos = hash_pos(_arkp->ht, key, klen); + rcbp->pos = hash_pos(_arkp->ht, key, klen); rcbp->sthrd = rcbp->pos / _arkp->npart; - pt = rcbp->pos / _arkp->npart; + pt = rcbp->pos / _arkp->npart; } else { @@ -258,7 +261,7 @@ int ark_enq_cmd(int cmd, _ARK *_arkp, uint64_t klen, void *key, } else { - KV_TRC_DBG(pAT, "NO_TAG: %"PRIu64"", dt); + KV_TRC_DBG(pAT, "NO_TAG: %"PRIx64"", dt); } ark_enq_cmd_err: @@ -276,13 +279,13 @@ int ark_enq_cmd(int cmd, _ARK *_arkp, uint64_t klen, void *key, ******************************************************************************* * \brief ******************************************************************************/ -int ark_rand_pool(_ARK *_arkp, int id, tcb_t *tcbp) { +void ark_rand_pool(_ARK *_arkp, int id, tcb_t *tcbp) +{ // find the hashtable position rcb_t *rcbp = &(_arkp->rcbs[tcbp->rtag]); int32_t found = 0; int32_t i = 0; int32_t ea_rc = 0; - int32_t state = ARK_CMD_DONE; uint8_t *buf = NULL; uint8_t *pos = NULL; uint64_t hblk; @@ -405,7 +408,7 @@ int ark_rand_pool(_ARK *_arkp, int id, tcb_t *tcbp) { } ark_rand_pool_err: - return state; + tcbp->state = ARK_CMD_DONE; } /** @@ -415,12 +418,12 @@ int ark_rand_pool(_ARK *_arkp, int id, tcb_t *tcbp) { * arp->res returns the length of the key\n * arp->val is for the ARI structure...both for passing in and returning. ******************************************************************************/ -int ark_first_pool(_ARK *_arkp, int id, tcb_t *tcbp) { +void ark_first_pool(_ARK *_arkp, int id, tcb_t *tcbp) +{ rcb_t *rcbp = &(_arkp->rcbs[tcbp->rtag]); _ARI *_arip = NULL; uint64_t i; int32_t rc = 0; - int32_t state = ARK_CMD_DONE; int32_t found = 0; uint8_t *buf = NULL; uint8_t *pos = NULL; @@ -548,7 +551,7 @@ int ark_first_pool(_ARK *_arkp, int id, tcb_t *tcbp) { _arip->bt = NULL; } - return state; + tcbp->state = ARK_CMD_DONE; } /** @@ -558,12 +561,12 @@ int ark_first_pool(_ARK *_arkp, int id, tcb_t *tcbp) { * arp->res returns the length of the key\n * arp->val is for the ARI structure...both for passing in and returning. ******************************************************************************/ -int ark_next_pool(_ARK *_arkp, int id, tcb_t *tcbp) { +void ark_next_pool(_ARK *_arkp, int id, tcb_t *tcbp) +{ rcb_t *rcbp = &(_arkp->rcbs[tcbp->rtag]); _ARI *_arip = (_ARI *)rcbp->val; uint64_t i; int32_t rc = 0; - int32_t state = ARK_CMD_DONE; int32_t found = 0; int32_t kcnt = 0; uint8_t *buf = NULL; @@ -753,7 +756,7 @@ int ark_next_pool(_ARK *_arkp, int id, tcb_t *tcbp) { _arip->bt = NULL; } - return state; + tcbp->state = ARK_CMD_DONE; } /** @@ -761,48 +764,45 @@ int ark_next_pool(_ARK *_arkp, int id, tcb_t *tcbp) { * \brief * reduce memory footprint if possible after an io is complete ******************************************************************************/ -void -cleanup_task_memory(_ARK *_arkp, tcb_t *tcbp, int tid) +void cleanup_task_memory(_ARK *_arkp, tcb_t *tcbp, int tid) { - if ((tcbp->inb != NULL) && (tcbp->inblen > (3*_arkp->bsize))) - { - KV_TRC_DBG(pAT,"REDUCE INB: tid:%d %"PRIu64" to %"PRIu64"", - tid, tcbp->inblen, _arkp->bsize); - bt_delete(tcbp->inb); - tcbp->inb = bt_new(0, _arkp->vlimit, sizeof(uint64_t), - &(tcbp->inblen), - &(tcbp->inb_orig)); - } - - if ((tcbp->oub != NULL) && (tcbp->oublen > (3*_arkp->bsize))) - { - KV_TRC_DBG(pAT,"REDUCE OTB: tid:%d %"PRIu64" to %"PRIu64"", - tid, tcbp->oublen, _arkp->bsize); - bt_delete(tcbp->oub); - tcbp->oub = bt_new(0, _arkp->vlimit, sizeof(uint64_t), - &(tcbp->oublen), - &(tcbp->oub_orig)); - } + if (tcbp->inb != NULL && tcbp->inblen > 2*_arkp->bsize) + { + KV_TRC_DBG(pAT,"REDUCE INB: tid:%d %"PRIu64" to %"PRIu64"", + tid, tcbp->inblen, _arkp->bsize); + bt_delete(tcbp->inb_orig); + tcbp->inb = bt_new(0, _arkp->vlimit, sizeof(uint64_t), + &(tcbp->inblen), &(tcbp->inb_orig)); + } - if ((tcbp->vb != NULL) && (tcbp->vbsize > (_arkp->bsize * 256))) - { - KV_TRC_DBG(pAT,"REDUCE VAB: tid:%d %"PRIu64" to %"PRIu64"", - tid, tcbp->vbsize, _arkp->bsize * 256); - am_free(tcbp->vb_orig); - tcbp->vbsize = _arkp->bsize * 256; - tcbp->vb_orig = am_malloc(tcbp->vbsize); - if (tcbp->vb_orig != NULL) + if (tcbp->oub != NULL && tcbp->oublen > 2*_arkp->bsize) { - tcbp->vb = ptr_align(tcbp->vb_orig); + KV_TRC_DBG(pAT,"REDUCE OTB: tid:%d %"PRIu64" to %"PRIu64"", + tid, tcbp->oublen, _arkp->bsize); + bt_delete(tcbp->oub_orig); + tcbp->oub = bt_new(0, _arkp->vlimit, sizeof(uint64_t), + &(tcbp->oublen), &(tcbp->oub_orig)); } - else + + if (tcbp->vb != NULL && tcbp->vbsize > _arkp->bsize*ARK_MIN_VB) { - // Clear out the vbsize. When this buffer is used - // for a command, the command will fail gracefully - tcbp->vbsize = 0; - tcbp->vb = NULL; + KV_TRC_DBG(pAT,"REDUCE VAB: tid:%d %"PRIu64" to %"PRIu64"", + tid, tcbp->vbsize, _arkp->bsize*ARK_MIN_VB); + am_free(tcbp->vb_orig); + tcbp->vbsize = _arkp->bsize*ARK_MIN_VB; + tcbp->vb_orig = am_malloc(tcbp->vbsize); + if (tcbp->vb_orig != NULL) + { + tcbp->vb = ptr_align(tcbp->vb_orig); + } + else + { + // Clear out the vbsize. When this buffer is used + // for a command, the command will fail gracefully + tcbp->vbsize = 0; + tcbp->vb = NULL; + } } - } } /** @@ -881,76 +881,112 @@ void *pool_function(void *arg) iocb_t *iocbp = NULL; queue_t *rq = scbp->rqueue; queue_t *tq = scbp->tqueue; - queue_t *ioq = scbp->ioqueue; + queue_t *sq = scbp->scheduleQ; + queue_t *hq = scbp->harvestQ; int32_t i = 0; int32_t reqrc = EAGAIN; int32_t tskrc = EAGAIN; int32_t tskact = 0; int32_t reqact = 0; - int32_t iotask = 0; + int32_t itag = 0; int32_t reqtag = -1; int32_t tsktag = 0; uint64_t hval = 0; uint64_t hlba = 0; + uint64_t hsleep = 0; + uint32_t MAX_POLL=0; + uint64_t perfT = 0; + uint64_t verbT = 0; + int max = 0; KV_TRC_DBG(pAT, "start tid:%d nactive:%d", id, _arkp->nactive); - // Run until the thread state is EXIT or the global - // state, ark_exit, is not set showing we are shutting - // down the ark db. + /*---------------------------------------------------------------------------- + * Run until the thread state is EXIT or the global + * state, ark_exit, is not set showing we are shutting + * down the ark db. + *--------------------------------------------------------------------------*/ while ((scbp->poolstate != PT_EXIT)) { - // loop through the iocbs in the ioq and process them - for (i=0; itcbs[iotask]); - iocbp = &(_arkp->iocbs[iotask]); + MAX_POLL = MAX_POLL < 10 ? 1 : MAX_POLL/10; + } + for (i=0; itcbs[itag]); + iocbp = &(_arkp->iocbs[itag]); iorcbp = &(_arkp->rcbs[iotcbp->rtag]); - KV_TRC_DBG(pAT, "IO: IQ_DEQ tid:%d state:%d ttag:%d", - id, iotcbp->state, iotask); + KV_TRC_EXT3(pAT, "HQ_DEQ tid:%d ttag:%3d state:%d", + id, itag, iotcbp->state); + + if (iotcbp->state == ARK_IO_HARVEST) + { + ea_async_io_harvest(_arkp, id, iotcbp, iocbp, iorcbp); + if (iocbp->hmissN) {++hsleep;} + } + + // place the iocb on a queue for the next step + if (iotcbp->state==ARK_IO_SCHEDULE) {queue_enq_unsafe(sq, itag);} + else if (iotcbp->state==ARK_IO_HARVEST) {queue_enq_unsafe(hq, itag);} + else {queue_enq_unsafe(tq, itag);} + } + + /*------------------------------------------------------------------------ + * schedule IOs + *----------------------------------------------------------------------*/ + for (i=0; itcbs[itag]); + iocbp = &(_arkp->iocbs[itag]); + iorcbp = &(_arkp->rcbs[iotcbp->rtag]); + + KV_TRC_EXT3(pAT, "SQ_DEQ tid:%d ttag:%3d state:%d", + id, itag, iotcbp->state); - // do schedule or harvest if (iotcbp->state == ARK_IO_SCHEDULE) {ea_async_io_schedule(_arkp, id, iotcbp, iocbp);} - else if (iotcbp->state == ARK_IO_HARVEST) - {ea_async_io_harvest(_arkp, id, iotcbp, iocbp, iorcbp);} // place the iocb on a queue for the next step - if (iotcbp->state == ARK_IO_SCHEDULE || - iotcbp->state == ARK_IO_HARVEST) - { - (void)queue_enq_unsafe(ioq, iotask); - KV_TRC_DBG(pAT, "IO: IQ_ENQ tid:%d state:%d ttag:%d", - id, iotcbp->state, iotask); - } - else - { - if (iotcbp->state == ARK_IO_DONE) - {iotcbp->state = iocbp->io_done;} - (void)queue_enq_unsafe(tq, iotask); - KV_TRC_DBG(pAT, "IO: TQ_ENQ tid:%d state:%d ttag:%d", - id, iotcbp->state,iotask); - } + if (iotcbp->state==ARK_IO_SCHEDULE) {queue_enq_unsafe(sq, itag);} + else if (iotcbp->state==ARK_IO_HARVEST) {queue_enq_unsafe(hq, itag);} + else {queue_enq_unsafe(tq, itag);} } - // Now we check the request queue and try to pull off - // as many requests as possible and queue them up - // in the task queue - queue_lock(rq); - if ( (queue_empty(rq)) && (reqtag == -1) ) + /*------------------------------------------------------------------------ + * usleep if appropriate + *----------------------------------------------------------------------*/ + if (hsleep>=1000 && queue_count(rq)==0 && queue_count(tq)==0) { - if ( queue_empty(ioq) && queue_empty(tq) ) - { - // We have reached a point where there is absolutely - // no work for this worker thread to do. So we - // go to sleep waiting for new requests to come in - KV_TRC_IO(pAT, "IO: IDLE_THREAD: tid:%d", id); - queue_wait(rq); - } + hsleep=0; + usleep(5); + KV_TRC_DBG(pAT,"IO: USLEEP"); } + /*------------------------------------------------------------------------ + * check the request queue and try to pull off + * as many requests as possible and queue them up + * in the task queue + *----------------------------------------------------------------------*/ + queue_lock(rq); + if (queue_empty(rq) && reqtag == -1) + { + if (queue_empty(sq) && queue_empty(hq) && queue_empty(tq)) + { + // We have reached a point where there is absolutely + // no work for this worker thread to do. So we + // go to sleep waiting for new requests to come in + KV_TRC_IO(pAT, "IO: tid:%d IDLE_THREAD", id); + queue_wait(rq); + } + } while (((reqrc == EAGAIN) && !(queue_empty(rq))) || ((reqrc == 0) && (!tag_empty(_arkp->ttags)))) { @@ -964,7 +1000,7 @@ void *pool_function(void *arg) rcbp->ttag = -1; rcbp->hold = -1; } - KV_TRC_DBG(pAT, "IO: RQ_DEQ tid:%d rtag:%d", id, reqtag); + KV_TRC_EXT3(pAT, "RQ_DEQ tid:%d ttag:%3d", id, reqtag); } if (reqrc == 0) @@ -984,20 +1020,14 @@ void *pool_function(void *arg) reqrc = EAGAIN; reqtag = -1; rcbp = NULL; - KV_TRC_DBG(pAT,"IO: RQ EAGAIN tid:%2d rq:%3d tq:%2d iq:%2d", - id, rq->c, tq->c, ioq->c); + KV_TRC_DBG(pAT,"IO: tid:%2d RQ_EAGAIN rq:%3d tq:%2d sq:%2d " + "hq:%2d", id, rq->c, tq->c, sq->c, hq->c); } else { tskrc = tag_unbury(_arkp->ttags, &tsktag); - if (tskrc == 0) - { - tcbp = &(_arkp->tcbs[tsktag]); - } - else - { - tcbp = NULL; - } + if (tskrc == 0) {tcbp = &(_arkp->tcbs[tsktag]);} + else {tcbp = NULL;} if (tcbp) { @@ -1011,8 +1041,8 @@ void *pool_function(void *arg) hlba = HASH_LBA(hval); HASH_SET(_arkp->ht, rcbp->pos, HASH_MAKE(1, tsktag, hlba)); (void)queue_enq_unsafe(tq, tsktag); - KV_TRC_DBG(pAT, "IO: TQ_ENQ START tid:%d rtag:%d ttag:%d", - id, reqtag, tsktag); + KV_TRC_DBG(pAT, "IO: tid:%d ttag:%3d rtag:%d TQ_ENQ START ", + id, tsktag, reqtag); reqtag = -1; reqrc = EAGAIN; tskrc = EAGAIN; @@ -1022,183 +1052,228 @@ void *pool_function(void *arg) } } } - queue_unlock(rq); - // trace queues - KV_TRC_DBG(pAT,"IO: QUEUES tid:%2d rq:%3d tq:%3d iq:%3d", - id, rq->c, tq->c, ioq->c); + /* do calcs for avg QD */ + _arkp->QDT += tq->c + rq->c + sq->c + hq->c; + _arkp->QDN += 1; - while(!queue_empty(tq)) + /*------------------------------------------------------------------------ + * initiate or continue tasks + *----------------------------------------------------------------------*/ + max = tq->c; + for (i=0; itcbs[tskact]); - reqact = tactp->rtag; - ractp = &(_arkp->rcbs[reqact]); - KV_TRC_DBG(pAT, "IO: TQ_DEQ tid:%d state:%d ttag:%d", - id, tactp->state, tskact); - - switch (tactp->state) - { - case ARK_SET_START : - { - scbp->poolstats.ops_cnt++; - ark_set_start(_arkp, id, tactp); - break; - } - case ARK_SET_PROCESS_INB : - { - ark_set_process_inb(_arkp, id, tactp); - break; - } - case ARK_SET_WRITE : - { - ark_set_write(_arkp, id, tactp); - break; - } - case ARK_SET_FINISH : - { - ark_set_finish(_arkp, id, tactp); - break; - } - case ARK_GET_START : - { - scbp->poolstats.ops_cnt++; - ark_get_start(_arkp, id, tactp); - break; - } - case ARK_GET_PROCESS : - { - ark_get_process(_arkp, id, tactp); - break; - } - case ARK_GET_FINISH : - { - ark_get_finish(_arkp, id, tactp); - break; - } - case ARK_DEL_START : - { - scbp->poolstats.ops_cnt++; - ark_del_start(_arkp, id, tactp); - break; - } - case ARK_DEL_PROCESS : + int s=0; + if (queue_deq_unsafe(tq, &tskact) != 0) {continue;} + + tactp = &(_arkp->tcbs[tskact]); + reqact = tactp->rtag; + ractp = &(_arkp->rcbs[reqact]); + s = tactp->state; + + KV_TRC_EXT3(pAT, "TQ_DEQ tid:%d ttag:%3d state:%d ", + id, tskact, s); + + if (s==ARK_SET_START) {ark_set_start (_arkp,id,tactp);} + else if (s==ARK_SET_PROCESS_INB){ark_set_process_inb(_arkp,id,tactp);} + else if (s==ARK_SET_WRITE) {ark_set_write (_arkp,id,tactp);} + else if (s==ARK_SET_FINISH) {ark_set_finish (_arkp,id,tactp);} + else if (s==ARK_GET_START) {ark_get_start (_arkp,id,tactp);} + else if (s==ARK_GET_PROCESS) {ark_get_process (_arkp,id,tactp);} + else if (s==ARK_GET_FINISH) {ark_get_finish (_arkp,id,tactp);} + else if (s==ARK_DEL_START) {ark_del_start (_arkp,id,tactp);} + else if (s==ARK_DEL_PROCESS) {ark_del_process (_arkp,id,tactp);} + else if (s==ARK_DEL_FINISH) {ark_del_finish (_arkp,id,tactp);} + else if (s==ARK_EXIST_START) {ark_exist_start (_arkp,id,tactp);} + else if (s==ARK_EXIST_FINISH) {ark_exist_finish (_arkp,id,tactp);} + else if (s==ARK_RAND_START) {ark_rand_pool (_arkp,id,tactp);} + else if (s==ARK_FIRST_START) {ark_first_pool (_arkp,id,tactp);} + else if (s==ARK_NEXT_START) {ark_next_pool (_arkp,id,tactp);} + + /*-------------------------------------------------------------------- + * schedule the first IO right way + *------------------------------------------------------------------*/ + if (tactp->state == ARK_IO_SCHEDULE) { - ark_del_process(_arkp, id, tactp); - break; + iocbp = &(_arkp->iocbs[tskact]); + ea_async_io_schedule(_arkp, id, tactp, iocbp); } - case ARK_DEL_FINISH : - { - ark_del_finish(_arkp, id, tactp); - break; - } - case ARK_EXIST_START : - { - scbp->poolstats.ops_cnt++; - ark_exist_start(_arkp, id, tactp); - break; - } - case ARK_EXIST_FINISH : + + /*-------------------------------------------------------------------- + * requeue task + *------------------------------------------------------------------*/ + if (tactp->state == ARK_IO_SCHEDULE) { - ark_exist_finish(_arkp, id, tactp); - break; + KV_TRC_EXT3(pAT, "SQ_ENQ tid:%d ttag:%3d state:%d ", + id, tskact, tactp->state); + queue_enq_unsafe(sq,tskact); } - case ARK_RAND_START : + else if (tactp->state == ARK_IO_HARVEST) { - tactp->state = ark_rand_pool(_arkp, id, tactp); - break; + KV_TRC_EXT3(pAT, "HQ_ENQ tid:%d ttag:%3d state:%d ", + id, tskact, tactp->state); + queue_enq_unsafe(hq,tskact); } - case ARK_FIRST_START : + else { - tactp->state = ark_first_pool(_arkp, id, tactp); - break; + KV_TRC_EXT3(pAT, "TQ_ENQ tid:%d ttag:%3d state:%d ", + id, tskact, tactp->state); + (void)queue_enq_unsafe(tq, tskact); } - case ARK_NEXT_START : + } + + /*------------------------------------------------------------------------ + * complete requests + *----------------------------------------------------------------------*/ + max = tq->c; + for (i=0; itcbs[tskact]); + reqact = tactp->rtag; + ractp = &(_arkp->rcbs[reqact]); + + KV_TRC_EXT3(pAT, "TQ_DEQ tid:%d ttag:%3d state:%d ", + id, tskact, tactp->state); + + if (tactp->state == ARK_CMD_DONE) { - tactp->state = ark_next_pool(_arkp, id, tactp); - break; + uint32_t lat = UDELTA(ractp->stime, _arkp->ns_per_tick); + UPDATE_LAT(_arkp, ractp, lat); + + cleanup_task_memory(_arkp, tactp, id); + + if (ractp->hold == -1) + { + hlba = HASH_LBA(HASH_GET(_arkp->ht, ractp->pos)); + HASH_SET(_arkp->ht, ractp->pos, HASH_MAKE(0, 0, hlba)); + if ( ractp->cb != NULL ) + { + KV_TRC(pAT, "CMD_CMP tid:%d ttag:%3d rtag:%3d lat:%6d", + id, tskact, ractp->rtag, lat); + (ractp->cb)(ractp->rc, ractp->dt, ractp->res); + ractp->stat = A_NULL; + (void)tag_bury(_arkp->rtags, reqact); + } + else + { + pthread_mutex_lock(&(ractp->alock)); + ractp->stat = A_COMPLETE; + pthread_cond_broadcast(&(ractp->acond)); + pthread_mutex_unlock(&(ractp->alock)); + KV_TRC(pAT, "CMD_CMP tid:%d ttag:%3d rtag:%3d lat:%6d", + id, tskact, ractp->rtag, lat); + } + (void)tag_bury(_arkp->ttags, tskact); + } + else + { + tactp->rtag = ractp->hold; + ractp->hold = -1; + if ( ractp->cb != NULL) + { + KV_TRC(pAT, "CMD_CMP tid:%d ttag:%3d rtag:%3d lat:%6d", + id, tskact, ractp->rtag, lat); + (ractp->cb)(ractp->rc, ractp->dt, ractp->res); + ractp->stat = A_NULL; + (void)tag_bury(_arkp->rtags, reqact); + } + else + { + pthread_mutex_lock(&(ractp->alock)); + ractp->stat = A_COMPLETE; + pthread_cond_broadcast(&(ractp->acond)); + pthread_mutex_unlock(&(ractp->alock)); + KV_TRC(pAT, "CMD_CMP tid:%d ttag:%3d rtag:%3d lat:%6d", + id, tskact, ractp->rtag, lat); + } + + scbp->holds--; + ractp = &(_arkp->rcbs[tactp->rtag]); + ractp->ttag = tactp->ttag; + tactp->rtag = ractp->rtag; + tactp->state = init_task_state(_arkp, tactp); + (void)queue_enq_unsafe(tq, tskact); + KV_TRC_EXT3(pAT, "TQ_ENQ tid:%d ttag:%3d state:%d", + id, tskact, tactp->state); + } + continue; } - default : + else { - // The only state left is ARK_CMD_DONE, so we - // just break out and end the task - break; + (void)queue_enq_unsafe(tq, tskact); + KV_TRC_EXT3(pAT, "TQ_ENQ tid:%d ttag:%3d state:%d ", + id, tskact, tactp->state); } - } + } - if (tactp->state == ARK_CMD_DONE) - { - if (ractp->hold == -1) - { - hlba = HASH_LBA(HASH_GET(_arkp->ht, ractp->pos)); - HASH_SET(_arkp->ht, ractp->pos, HASH_MAKE(0, 0, hlba)); - if ( ractp->cb != NULL ) - { - (ractp->cb)(ractp->rc, ractp->dt, ractp->res); - ractp->stat = A_NULL; - (void)tag_bury(_arkp->rtags, reqact); - } - else - { - pthread_mutex_lock(&(ractp->alock)); - ractp->stat = A_COMPLETE; - pthread_cond_broadcast(&(ractp->acond)); - pthread_mutex_unlock(&(ractp->alock)); - KV_TRC_DBG(pAT, "IO: A_COMPLETE tid:%d rtag:%d", - id, ractp->rtag); - } - cleanup_task_memory(_arkp, tactp, id); - (void)tag_bury(_arkp->ttags, tskact); - } - else + /*------------------------------------------------------------------------ + * performance and dynamic tracing + *----------------------------------------------------------------------*/ + if (SDELTA(perfT, _arkp->ns_per_tick) > 1) + { + uint64_t opsT = _arkp->set_opsT + _arkp->get_opsT + _arkp->exi_opsT +\ + _arkp->del_opsT; + uint64_t latT = _arkp->set_latT + _arkp->get_latT + _arkp->exi_latT +\ + _arkp->del_latT; + uint64_t lat=0,set_lat=0,get_lat=0,exi_lat=0,del_lat=0,QDA=0; + if (opsT) {lat = latT/opsT;} + if (_arkp->set_opsT) {set_lat = _arkp->set_latT/_arkp->set_opsT;} + if (_arkp->get_opsT) {get_lat = _arkp->get_latT/_arkp->get_opsT;} + if (_arkp->exi_opsT) {exi_lat = _arkp->exi_latT/_arkp->exi_opsT;} + if (_arkp->del_opsT) {del_lat = _arkp->del_latT/_arkp->del_opsT;} + if (_arkp->QDT) {QDA = _arkp->QDT/_arkp->QDN;} + KV_TRC_PERF1(pAT,"IO: tid:%d PERF opsT:%7ld lat:%7ld " + "opsT(%7ld %7ld %7ld %7ld) " + "lat(%7ld %6ld %6ld %6ld) QD:%4ld issT:%4d", + id, opsT, lat, + _arkp->set_opsT, _arkp->get_opsT, _arkp->exi_opsT, + _arkp->del_opsT, + set_lat, get_lat, exi_lat, del_lat, + QDA, _arkp->issT); + KV_TRC_PERF2(pAT,"IO: tid:%d QUEUES rq:%3d tq:%3d sq:%4d hq:%4d ", + id, rq->c, tq->c, sq->c, hq->c); + _arkp->set_latT = 0; + _arkp->get_latT = 0; + _arkp->exi_latT = 0; + _arkp->del_latT = 0; + _arkp->set_opsT = 0; + _arkp->get_opsT = 0; + _arkp->exi_opsT = 0; + _arkp->del_opsT = 0; + _arkp->QDT = 0; + _arkp->QDN = 0; + perfT = getticks(); + + /* check for dynamic verbosity every 5 seconds */ + if (SDELTA(verbT, _arkp->ns_per_tick) > 5) { - tactp->rtag = ractp->hold; - ractp->hold = -1; - if ( ractp->cb != NULL) - { - (ractp->cb)(ractp->rc, ractp->dt, ractp->res); - ractp->stat = A_NULL; - (void)tag_bury(_arkp->rtags, reqact); - } - else - { - pthread_mutex_lock(&(ractp->alock)); - ractp->stat = A_COMPLETE; - pthread_cond_broadcast(&(ractp->acond)); - pthread_mutex_unlock(&(ractp->alock)); - KV_TRC_DBG(pAT, "IO: A_COMPLETE tid:%d rtag:%d", - id, ractp->rtag); - } + struct stat sbuf; + FILE *fp = NULL; + char verbS[2] = {0}; + int verbI = 0; - scbp->holds--; - ractp = &(_arkp->rcbs[tactp->rtag]); - ractp->ttag = tactp->ttag; - tactp->rtag = ractp->rtag; - tactp->state = init_task_state(_arkp, tactp); - (void)queue_enq_unsafe(tq, tskact); - KV_TRC_DBG(pAT, "IO: TQ_ENQ tid:%d state:%d ttag:%d", - id, tactp->state, tskact); + verbT = getticks(); + + if (stat(ARK_VERB_FN,&sbuf) == 0) + { + if ((fp=fopen(ARK_VERB_FN,"r"))) + { + if (fgets(verbS, 2, fp)) + { + verbI = atoi(verbS); + if (verbI > 0 && verbI < 10) {pAT->verbosity=verbI;} + } + fclose(fp); + } + } } - } - else if (tactp->state == ARK_IO_DONE) - { - iocbp = &(_arkp->iocbs[tskact]); - tactp->state = iocbp->io_done; - (void)queue_enq_unsafe(tq, tskact); - KV_TRC_DBG(pAT, "IO: TQ_ENQ tid:%d state:%d ttag:%d", - id, tactp->state, tskact); - } - else - { - (void)queue_enq_unsafe(ioq, tskact); - KV_TRC_DBG(pAT, "IO: IQ_ENQ tid:%d state:%d ttag:%d", - id, tactp->state, tskact); - } } } - KV_TRC(pAT, "existing tid:%d nactive:%d", id, _arkp->nactive); + KV_TRC(pAT, "exiting tid:%d nactive:%d", id, _arkp->nactive); return NULL; } diff --git a/src/kv/arp_del.c b/src/kv/arp_del.c index aeda6a73..0dff166c 100644 --- a/src/kv/arp_del.c +++ b/src/kv/arp_del.c @@ -49,6 +49,8 @@ void ark_del_start(_ARK *_arkp, int tid, tcb_t *tcbp) ark_io_list_t *bl_array = NULL; int32_t rc = 0; + scbp->poolstats.ops_cnt+=1; + // Acquire the block that contains the hash entry // control information. tcbp->hblk = HASH_LBA(HASH_GET(_arkp->ht, rcbp->pos)); @@ -78,6 +80,7 @@ void ark_del_start(_ARK *_arkp, int tid, tcb_t *tcbp) tcbp->state = ARK_CMD_DONE; goto ark_del_start_err; } + rc = bt_growif(&(tcbp->inb), &(tcbp->inb_orig), &(tcbp->inblen), (tcbp->blen * _arkp->bsize)); if (rc != 0) @@ -103,14 +106,16 @@ void ark_del_start(_ARK *_arkp, int tid, tcb_t *tcbp) tcbp->old_btsize = tcbp->inb->len; scbp->poolstats.io_cnt += tcbp->blen; - KV_TRC_IO(pAT, "read hash entry ttag:%d", tcbp->ttag); + KV_TRC(pAT, "RD_HASH tid:%d ttag:%3d", tid, tcbp->ttag); // Schedule the IO to read the hash entry from storage ea_async_io_init(_arkp, ARK_EA_READ, (void *)tcbp->inb, bl_array, tcbp->blen, 0, tcbp->ttag, ARK_DEL_PROCESS); - if (ea_async_io_schedule(_arkp, tid, iotcbp, iocbp) && - ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp)) + + if (iocbp->ea->st_type == EA_STORE_TYPE_MEMORY) { - ark_del_process(_arkp, tid, tcbp); + ea_async_io_schedule(_arkp, tid, iotcbp, iocbp); + ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp); + if (iotcbp->state == ARK_DEL_PROCESS) {ark_del_process(_arkp, tid, tcbp);} } ark_del_start_err: @@ -145,14 +150,14 @@ void ark_del_process(_ARK *_arkp, int tid, tcb_t *tcbp) // and save off the length of the value rcbp->res = bt_del_def(tcbp->oub, tcbp->inb, rcbp->klen, rcbp->key, (uint8_t*)&dblk, &oldvlen); - + if (rcbp->res >= 0) { // Return the blocks of the hash entry back to // the free list ark_drop_pool(_arkp, &(scbp->poolstats), tcbp->hblk); if (dblk > 0) - { + { // Return the blocks used to store the value if it // wasn't stored in the hash entry ark_drop_pool(_arkp, &(scbp->poolstats), dblk); @@ -161,7 +166,6 @@ void ark_del_process(_ARK *_arkp, int tid, tcb_t *tcbp) // Are there entries in the hash bucket. if (tcbp->oub->cnt > 0) { - // Determine how many blocks will be needed for the // out buffer and then get them from the free // block list @@ -188,18 +192,20 @@ void ark_del_process(_ARK *_arkp, int tid, tcb_t *tcbp) goto ark_del_process_err; } - scbp->poolstats.io_cnt += blkcnt; + scbp->poolstats.io_cnt += blkcnt; scbp->poolstats.byte_cnt -= (tcbp->old_btsize + oldvlen); scbp->poolstats.byte_cnt += tcbp->oub->len; - KV_TRC_IO(pAT, "write updated hash entry ttag:%d", tcbp->ttag); + KV_TRC(pAT, "WR_HASH tid:%d ttag:%3d", tid, tcbp->ttag); // Schedule the WRITE IO of the updated hash entry. ea_async_io_init(_arkp, ARK_EA_WRITE, (void *)tcbp->oub, bl_array, blkcnt, 0, tcbp->ttag, ARK_DEL_FINISH); - if (ea_async_io_schedule(_arkp, tid, iotcbp, iocbp) && - ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp)) + + if (iocbp->ea->st_type == EA_STORE_TYPE_MEMORY) { - ark_del_finish(_arkp, tid, tcbp); + ea_async_io_schedule(_arkp, tid, iotcbp, iocbp); + ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp); + if (iotcbp->state == ARK_DEL_FINISH) {ark_del_finish(_arkp,tid,tcbp);} } } else @@ -208,7 +214,7 @@ void ark_del_process(_ARK *_arkp, int tid, tcb_t *tcbp) scbp->poolstats.byte_cnt += tcbp->oub->len; scbp->poolstats.kv_cnt--; - KV_TRC_IO(pAT, "no write required for hash entry ttag:%d", tcbp->ttag); + KV_TRC(pAT, "WR_HASH tid:%d ttag:%3d NO_WRITE_REQD", tid, tcbp->ttag); // Nothing left in this hash entry, so let's clear out // the hash entry control block to show there is no // data in the store for this hash entry diff --git a/src/kv/arp_exist.c b/src/kv/arp_exist.c index 1147bb64..412f4fc1 100644 --- a/src/kv/arp_exist.c +++ b/src/kv/arp_exist.c @@ -50,6 +50,8 @@ void ark_exist_start(_ARK *_arkp, int tid, tcb_t *tcbp) ark_io_list_t *bl_array = NULL; int32_t rc = 0; + scbp->poolstats.ops_cnt+=1; + // Now that we have the hash entry, get the block // that holds the control information for the entry. tcbp->hblk = HASH_LBA(HASH_GET(_arkp->ht, rcbp->pos)); @@ -94,13 +96,15 @@ void ark_exist_start(_ARK *_arkp, int tid, tcb_t *tcbp) scbp->poolstats.io_cnt += tcbp->blen; - KV_TRC_IO(pAT, "read hash entry ttag:%d", tcbp->ttag); + KV_TRC(pAT, "RD_HASH tid:%d ttag:%3d", tid, tcbp->ttag); ea_async_io_init(_arkp, ARK_EA_READ, (void *)tcbp->inb, bl_array, tcbp->blen, 0, tcbp->ttag, ARK_EXIST_FINISH); - if (ea_async_io_schedule(_arkp, tid, iotcbp, iocbp) && - ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp)) + + if (iocbp->ea->st_type == EA_STORE_TYPE_MEMORY) { - ark_exist_finish(_arkp, tid, tcbp); + ea_async_io_schedule(_arkp, tid, iotcbp, iocbp); + ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp); + if (iotcbp->state == ARK_EXIST_FINISH) {ark_exist_finish(_arkp,tid,tcbp);} } ark_exist_start_err: diff --git a/src/kv/arp_get.c b/src/kv/arp_get.c index e68de00f..5370224d 100644 --- a/src/kv/arp_get.c +++ b/src/kv/arp_get.c @@ -49,6 +49,8 @@ void ark_get_start(_ARK *_arkp, int tid, tcb_t *tcbp) ark_io_list_t *bl_array = NULL; int32_t rc = 0; + scbp->poolstats.ops_cnt+=1; + // Now that we have the hash entry, get the block // that holds the control information for the entry. tcbp->hblk = HASH_LBA(HASH_GET(_arkp->ht, rcbp->pos)); @@ -93,13 +95,15 @@ void ark_get_start(_ARK *_arkp, int tid, tcb_t *tcbp) scbp->poolstats.io_cnt += tcbp->blen; - KV_TRC_IO(pAT, "read hash entry ttag:%d", tcbp->ttag); + KV_TRC(pAT, "RD_HASH tid:%d ttag:%3d", tid, tcbp->ttag); ea_async_io_init(_arkp, ARK_EA_READ, (void *)tcbp->inb, bl_array, tcbp->blen, 0, tcbp->ttag, ARK_GET_PROCESS); - if (ea_async_io_schedule(_arkp, tid, iotcbp, iocbp) && - ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp)) + + if (iocbp->ea->st_type == EA_STORE_TYPE_MEMORY) { - ark_get_process(_arkp, tid, tcbp); + ea_async_io_schedule(_arkp, tid, iotcbp, iocbp); + ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp); + if (iotcbp->state == ARK_GET_PROCESS) {ark_get_process(_arkp, tid, tcbp);} } ark_get_start_err: @@ -147,7 +151,7 @@ void ark_get_process(_ARK *_arkp, int tid, tcb_t *tcbp) tcbp->state = ARK_CMD_DONE; goto ark_get_process_err; } - + // The realloc succeeded. Set the new size, original // variable buffer, and adjusted variable buffer tcbp->vbsize = new_vbsize; @@ -168,17 +172,20 @@ void ark_get_process(_ARK *_arkp, int tid, tcb_t *tcbp) scbp->poolstats.io_cnt += tcbp->blen; - KV_TRC_IO(pAT, "read key value ttag:%d", tcbp->ttag); + KV_TRC(pAT, "RD_VAL tid:%d ttag:%3d vlen:%5ld", + tid, tcbp->ttag, rcbp->vlen); // Schedule the READ of the key's value into the // variable buffer. ea_async_io_init(_arkp, ARK_EA_READ, (void *)tcbp->vb, bl_array, tcbp->blen, 0, tcbp->ttag, ARK_GET_FINISH); - if (ea_async_io_schedule(_arkp, tid, iotcbp, iocbp) && - ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp)) + + if (iocbp->ea->st_type == EA_STORE_TYPE_MEMORY) { - ark_get_finish(_arkp, tid, tcbp); + ea_async_io_schedule(_arkp, tid, iotcbp, iocbp); + ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp); + if (iotcbp->state == ARK_GET_FINISH) {ark_get_finish(_arkp,tid,tcbp);} } - } +} else { ark_get_finish(_arkp, tid, tcbp); diff --git a/src/kv/arp_set.c b/src/kv/arp_set.c index ff739074..6e128006 100644 --- a/src/kv/arp_set.c +++ b/src/kv/arp_set.c @@ -54,6 +54,8 @@ void ark_set_start(_ARK *_arkp, int tid, tcb_t *tcbp) int32_t rc = 0; uint8_t *new_vb = NULL; + scbp->poolstats.ops_cnt+=1; + // Let's check to see if the value is larger than // what can fit inline in the bucket. if ( vlen > _arkp->vlimit ) @@ -73,7 +75,8 @@ void ark_set_start(_ARK *_arkp, int tid, tcb_t *tcbp) // the size in a new variable and wait to make sure // the realloc succeeds before updating vbsize new_vbsize = tcbp->vblkcnt * _arkp->bsize; - KV_TRC_DBG(pAT, "REALLOC ttag:%d VB:%"PRIu64"", tcbp->ttag, new_vbsize); + KV_TRC_DBG(pAT, "VB_REALLOC tid:%d ttag:%3d vlen:%5ld new_vbsize:%ld", + tid, tcbp->ttag, vlen, new_vbsize); new_vb = am_realloc(tcbp->vb_orig, new_vbsize); if ( NULL == new_vb ) { @@ -83,7 +86,7 @@ void ark_set_start(_ARK *_arkp, int tid, tcb_t *tcbp) KV_TRC_FFDC(pAT, "realloc failed, ttag = %d rc = %d", tcbp->ttag, rc); goto ark_set_start_err; } - tcbp->vbsize = new_vbsize; + tcbp->vbsize = new_vbsize; tcbp->vb_orig = new_vb; tcbp->vb = ptr_align(tcbp->vb_orig); } @@ -108,7 +111,7 @@ void ark_set_start(_ARK *_arkp, int tid, tcb_t *tcbp) } else { - KV_TRC_IO(pAT, "INLINE ttag:%d VLEN:%"PRIu64"", tcbp->ttag, vlen); + KV_TRC(pAT, "INLINE tid:%d ttag:%3d vlen:%5ld", tid, tcbp->ttag, vlen); tcbp->vval = (uint8_t *)rcbp->val; } @@ -116,13 +119,13 @@ void ark_set_start(_ARK *_arkp, int tid, tcb_t *tcbp) if ( 0 == tcbp->hblk ) { - KV_TRC_IO(pAT, "FIRST_KEY ttag:%d", tcbp->ttag); + KV_TRC(pAT, "1ST_KEY tid:%d ttag:%3d", tid, tcbp->ttag); // This is going to be the first key in this bucket. // Initialize the bucket data/header rc = bt_init(tcbp->inb); if (rc) { - KV_TRC_FFDC(pAT, "bt_init failed ttag:%d", tcbp->ttag); + KV_TRC_FFDC(pAT, "bt_init failed ttag:%3d", tcbp->ttag); rcbp->res = -1; rcbp->rc = rc; tcbp->state = ARK_CMD_DONE; @@ -150,7 +153,7 @@ void ark_set_start(_ARK *_arkp, int tid, tcb_t *tcbp) rcbp->res = -1; rcbp->rc = rc; tcbp->state = ARK_CMD_DONE; - KV_TRC_FFDC(pAT, "bt_growif failed, ttag:%d rc:%d errno:%d", + KV_TRC_FFDC(pAT, "bt_growif failed, ttag:%3d rc:%d errno:%d", tcbp->ttag, rc, errno); goto ark_set_start_err; } @@ -167,17 +170,20 @@ void ark_set_start(_ARK *_arkp, int tid, tcb_t *tcbp) KV_TRC_FFDC(pAT, "bl_chain failed, ttag = %d rc = %d", tcbp->ttag, rc); goto ark_set_start_err; } + scbp->poolstats.io_cnt += tcbp->blen; - KV_TRC_IO(pAT, "read hash entry, ttag:%d", tcbp->ttag); + KV_TRC(pAT, "RD_HASH tid:%d ttag:%3d", tid, tcbp->ttag); ea_async_io_init(_arkp, ARK_EA_READ, (void *)tcbp->inb, bl_array, tcbp->blen, 0, tcbp->ttag, ARK_SET_PROCESS_INB); - if (ea_async_io_schedule(_arkp, tid, iotcbp, iocbp) && - ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp)) + + if (_arkp->ea->st_type == EA_STORE_TYPE_MEMORY) { - ark_set_process_inb(_arkp, tid, tcbp); + ea_async_io_schedule(_arkp, tid, iotcbp, iocbp); + ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp); + ark_set_process_inb (_arkp, tid, tcbp); } - } +} ark_set_start_err: @@ -208,12 +214,11 @@ void ark_set_process(_ARK *_arkp, int tid, tcb_t *tcbp) // Let's see if we need to grow the out buffer tcbp->old_btsize = tcbp->inb->len; rc = bt_growif(&(tcbp->oub), &(tcbp->oub_orig), &(tcbp->oublen), - divceil((tcbp->blen * _arkp->bsize) + - (rcbp->klen + _arkp->vlimit + 16), - _arkp->bsize) * _arkp->bsize); + (tcbp->blen * _arkp->bsize) + + (rcbp->klen + _arkp->vlimit)); if (rc != 0) { - KV_TRC_FFDC(pAT, "bt_growif rc != 0, ttag:%d", tcbp->ttag); + KV_TRC_FFDC(pAT, "bt_growif rc != 0, ttag:%3d", tcbp->ttag); rcbp->res = -1; rcbp->rc = rc; tcbp->state = ARK_CMD_DONE; @@ -221,17 +226,24 @@ void ark_set_process(_ARK *_arkp, int tid, tcb_t *tcbp) } // modify bucket - tcbp->new_key = bt_set(tcbp->oub, tcbp->inb, rcbp->klen, rcbp->key, + if (tcbp->inb->cnt) + { + KV_TRC(pAT, "MOD_BT tid:%d ttag:%3d klen:%5ld vlen:%5ld", + tid, tcbp->ttag, rcbp->klen, rcbp->vlen); + } + tcbp->new_key = bt_set(tcbp->oub, tcbp->inb, rcbp->klen, rcbp->key, rcbp->vlen, tcbp->vval, &oldvlen); // write the value to the value allocated blocks - if (rcbp->vlen > _arkp->vlimit) { - KV_TRC_IO(pAT, "write value, ttag:%d", tcbp->ttag); + if (rcbp->vlen > _arkp->vlimit) + { + KV_TRC(pAT, "WR_VAL tid:%d ttag:%3d vlen:%5ld", + tid, tcbp->ttag, rcbp->vlen); bl_array = bl_chain(_arkp->bl, tcbp->vblk, tcbp->vblkcnt); if (bl_array == NULL) { - KV_TRC_FFDC(pAT, "bl_chain failed ttag:%d", tcbp->ttag); + KV_TRC_FFDC(pAT, "bl_chain failed ttag:%3d", tcbp->ttag); rcbp->rc = ENOMEM; rcbp->res = -1; tcbp->state = ARK_CMD_DONE; @@ -240,14 +252,16 @@ void ark_set_process(_ARK *_arkp, int tid, tcb_t *tcbp) scbp->poolstats.byte_cnt -= oldvlen; scbp->poolstats.byte_cnt += rcbp->vlen; - scbp->poolstats.io_cnt += tcbp->vblkcnt; + scbp->poolstats.io_cnt += tcbp->vblkcnt; - ea_async_io_init(_arkp, ARK_EA_WRITE, (void *)tcbp->vb, + ea_async_io_init(_arkp, ARK_EA_WRITE, (void*)tcbp->vb, bl_array, tcbp->vblkcnt, 0, tcbp->ttag, ARK_SET_WRITE); - if (ea_async_io_schedule(_arkp, tid, iotcbp, iocbp) && - ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp)) + + if (_arkp->ea->st_type == EA_STORE_TYPE_MEMORY) { - ark_set_write(_arkp, tid, tcbp); + ea_async_io_schedule(_arkp, tid, iotcbp, iocbp); + ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp); + if (iotcbp->state == ARK_SET_WRITE) {ark_set_write(_arkp, tid, tcbp);} } } else @@ -269,14 +283,14 @@ void ark_set_write(_ARK *_arkp, int tid, tcb_t *tcbp) ark_io_list_t *bl_array = NULL; uint64_t blkcnt = 0; - KV_TRC_IO(pAT, "WRITE_KEY, ttag:%d on %d", tcbp->ttag, tid); + KV_TRC(pAT, "WR_KEY tid:%d ttag:%3d klen:%5ld", tid, tcbp->ttag, rcbp->klen); // write obuf to new blocks blkcnt = divceil(tcbp->oub->len, _arkp->bsize); tcbp->nblk = ark_take_pool(_arkp, &(scbp->poolstats), blkcnt); if (tcbp->nblk == -1) { - KV_TRC_FFDC(pAT, "ark_take_pool %ld failed ttag:%d", + KV_TRC_FFDC(pAT, "ark_take_pool %ld failed ttag:%3d", blkcnt, tcbp->ttag); rcbp->rc = ENOSPC; rcbp->res = -1; @@ -287,7 +301,7 @@ void ark_set_write(_ARK *_arkp, int tid, tcb_t *tcbp) bl_array = bl_chain(_arkp->bl, tcbp->nblk, blkcnt); if (bl_array == NULL) { - KV_TRC_FFDC(pAT, "bl_chain failed ttag:%d", tcbp->ttag); + KV_TRC_FFDC(pAT, "bl_chain failed ttag:%3d", tcbp->ttag); rcbp->rc = ENOMEM; rcbp->res = -1; tcbp->state = ARK_CMD_DONE; @@ -296,14 +310,16 @@ void ark_set_write(_ARK *_arkp, int tid, tcb_t *tcbp) scbp->poolstats.byte_cnt -= tcbp->old_btsize; scbp->poolstats.byte_cnt += tcbp->oub->len; - scbp->poolstats.io_cnt += blkcnt; + scbp->poolstats.io_cnt += blkcnt; ea_async_io_init(_arkp, ARK_EA_WRITE, (void *)tcbp->oub, bl_array, blkcnt, 0, tcbp->ttag, ARK_SET_FINISH); - if (ea_async_io_schedule(_arkp, tid, iotcbp, iocbp) && - ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp)) + + if (_arkp->ea->st_type == EA_STORE_TYPE_MEMORY) { - ark_set_finish(_arkp, tid, tcbp); + ea_async_io_schedule(_arkp, tid, iotcbp, iocbp); + ea_async_io_harvest (_arkp, tid, iotcbp, iocbp, rcbp); + if (iotcbp->state == ARK_SET_FINISH) {ark_set_finish(_arkp, tid, tcbp);} } ark_set_write_err: @@ -316,6 +332,8 @@ void ark_set_finish(_ARK *_arkp, int tid, tcb_t *tcbp) scb_t *scbp = &(_arkp->poolthreads[tid]); rcb_t *rcbp = &(_arkp->rcbs[tcbp->rtag]); + KV_TRC(pAT, "HASHSET tid:%d ttag:%3d nblk:%5ld", tid, tcbp->ttag, tcbp->nblk); + HASH_SET(_arkp->ht, rcbp->pos, HASH_MAKE(1, tcbp->ttag, tcbp->nblk)); rcbp->res = rcbp->vlen; diff --git a/src/kv/bl.c b/src/kv/bl.c index 0484bafc..530eb9aa 100644 --- a/src/kv/bl.c +++ b/src/kv/bl.c @@ -293,7 +293,7 @@ void bl_check_take(BL *bl, int64_t n) return; } - avail = bl->n - bl->top; + avail = bl->n - bl->top -1; // if we need blocks and we have uninitialized blocks, init them if (n > bl->count && avail) @@ -570,7 +570,7 @@ ark_io_list_t *bl_chain(BL *bl, int64_t b, int64_t len) { while (0 < b) { bl_array[i].blkno = b; - bl_array[i].a_tag = -1; + bl_array[i].a_tag.tag = -1; if ((b=iv_get(bl->list, b)) < 0) { KV_TRC_FFDC(pAT, "invalid chain index:%ld", b); @@ -612,7 +612,7 @@ ark_io_list_t *bl_chain_blocks(BL *bl, int64_t start, int64_t len) for (i = 0; i < len; i++) { bl_array[i].blkno = start + i; - bl_array[i].a_tag = -1; + bl_array[i].a_tag.tag = -1; } } } @@ -638,7 +638,7 @@ ark_io_list_t *bl_chain_no_bl(int64_t start, int64_t len) for (i = 0; i < len; i++) { bl_array[i].blkno = start + i; - bl_array[i].a_tag = -1; + bl_array[i].a_tag.tag = -1; } } diff --git a/src/kv/bl.h b/src/kv/bl.h index a42a3bb2..a797c99a 100644 --- a/src/kv/bl.h +++ b/src/kv/bl.h @@ -28,13 +28,14 @@ #include #include #include "iv.h" +#include #define BL_INITN 10000 typedef struct ark_io_list { - int64_t blkno; - int a_tag; + int64_t blkno; + cflsh_cg_tag_t a_tag; } ark_io_list_t; typedef struct _bl diff --git a/src/kv/bt.c b/src/kv/bt.c index ce6fbd0a..0255240f 100644 --- a/src/kv/bt.c +++ b/src/kv/bt.c @@ -36,11 +36,13 @@ #include -BT *bt_new(uint64_t dlen, uint64_t vmx, uint64_t vdf, uint64_t *btsize, BT **bt_orig) { +BT *bt_new(uint64_t dlen, uint64_t vmx, uint64_t vdf, uint64_t *btsize, BT **bt_orig) +{ BT *bt = NULL; *bt_orig = am_malloc(sizeof(BT) + dlen); - if (*bt_orig == NULL) { + if (*bt_orig == NULL) + { errno = ENOMEM; KV_TRC_FFDC(pAT, "Out of memory dlen %ld vmx %ld vdf %ld, errno = %d", dlen, vmx, vdf, errno); @@ -62,23 +64,25 @@ BT *bt_new(uint64_t dlen, uint64_t vmx, uint64_t vdf, uint64_t *btsize, BT **bt *btsize = sizeof(BT) + dlen; } } - KV_TRC(pAT, "bt %p dlen %ld vmx %ld vdf %ld", - bt, dlen, vmx, vdf); + KV_TRC(pAT, "bt:%p dlen %ld vmx %ld vdf %ld bt_orig:%p", + bt, dlen, vmx, vdf, *bt_orig); return bt; } -int bt_growif(BT **bt, BT **bt_orig, uint64_t *lim, uint64_t size) { +int bt_growif(BT **bt, BT **bt_orig, uint64_t *lim, uint64_t size) +{ BT *btret = NULL; int rc = 0; // We need to include the size of the BT header struct // This is a bit of a hack, but we know that a // struct BT will fit within a block size of 4K - uint64_t newsize = (size + 4096); + uint64_t newsize = (size + sizeof(BT)); // If the sie of bt is already sufficient, do nothing // and just return success. - if (newsize > *lim) { + if (newsize > *lim) + { btret = am_realloc(*bt_orig, newsize); if (btret == NULL) { @@ -94,6 +98,9 @@ int bt_growif(BT **bt, BT **bt_orig, uint64_t *lim, uint64_t size) { // space *bt_orig = btret; *bt = (BT *)ptr_align(btret); + KV_TRC(pAT, "BT_REALLOC new:%p p_orig:%p bt_orig:%p lim:%ld size:%ld " + "newsize:%ld", + *bt, btret, *bt_orig, *lim, size, newsize); (*bt)->dlen = newsize - sizeof(BT); *lim = newsize; } @@ -103,6 +110,7 @@ int bt_growif(BT **bt, BT **bt_orig, uint64_t *lim, uint64_t size) { } int bt_init(BT *bt) { + if (bt) { bt->len = sizeof(BT); bt->cnt = 0; @@ -117,8 +125,12 @@ int bt_init(BT *bt) { void bt_delete(BT *bt) { - if (bt) {am_free(bt); KV_TRC_DBG(pAT, "bt %p", bt);} - else {KV_TRC_FFDC(pAT, "bt NULL");} + if (bt) + { + KV_TRC_DBG(pAT, "free bt:%p", bt); + am_free(bt); + } + else {KV_TRC_FFDC(pAT, "bt NULL");} } int64_t bt_exists(BT *bt, uint64_t kl, uint8_t *k) { @@ -141,10 +153,12 @@ int64_t bt_exists(BT *bt, uint64_t kl, uint8_t *k) { return BT_FAIL; } -int bt_set(BT *bto,BT *bti, uint64_t kl, uint8_t *k, uint64_t vl, uint8_t *v, uint64_t *ovlen) { +int bt_set(BT *bto,BT *bti, uint64_t kl, uint8_t *k, uint64_t vl, uint8_t *v, uint64_t *ovlen) +{ uint8_t *src = bti->data; uint8_t *dst = bto->data; uint64_t bytes = 0; + dst += vi_enc64(kl, dst); dst += vi_enc64(vl, dst); memcpy(dst, k, kl); @@ -181,6 +195,7 @@ int bt_set(BT *bto,BT *bti, uint64_t kl, uint8_t *k, uint64_t vl, uint8_t *v, ui bto->cnt = bti->cnt + add; bto->def = bti->def; bto->max = bti->max; + return add; } @@ -228,6 +243,7 @@ int64_t bt_del(BT *bto, BT *bti, uint64_t kl, uint8_t *k) { } return ret; } + // same as bt_del except returns the val if vlen > vlimit as this ref may be used to // to delete other things ref will be filled with def bytes // this could replace bt_del at the cost of the extra parameter diff --git a/src/kv/ea.c b/src/kv/ea.c index ec882b62..ecca0190 100644 --- a/src/kv/ea.c +++ b/src/kv/ea.c @@ -57,7 +57,6 @@ EA *ea_new(const char *path, uint64_t bsize, int basyncs, uint8_t *store = NULL; EA *ea = NULL; chunk_id_t chkid = NULL_CHUNK_ID; - chunk_ext_arg_t ext = 0; if (!(fetch_and_or(&cflsh_blk_lib_init,1))) { @@ -111,7 +110,7 @@ EA *ea_new(const char *path, uint64_t bsize, int basyncs, // Using a file. We don't care if it's an actual // file or a CAPI device, we let block layer // decide and we just use the chunk ID that is - // passed back from the cblk_open call. + // passed back from the cblk_cg_open call. ea->st_type = EA_STORE_TYPE_FILE; // Check to see if we need to create the store on a @@ -121,26 +120,25 @@ EA *ea_new(const char *path, uint64_t bsize, int basyncs, // can specify with a flag. if ( vlun == 0 ) { - KV_TRC(pAT, "cblk_open PHYSICAL LUN: %s", path); - chkid = cblk_open(path, basyncs, O_RDWR, ext, - CBLK_OPN_NO_INTRP_THREADS); + KV_TRC(pAT, "cblk_cg_open PHYSICAL LUN: %s", path); + chkid = cblk_cg_open(path, basyncs, O_RDWR, 1, 0, + CBLK_GROUP_RAID0|CBLK_OPN_NO_INTRP_THREADS); if (NULL_CHUNK_ID == chkid) { - printf("cblk_open physical lun failed\n"); - KV_TRC_FFDC(pAT, "cblk_open phys lun failed path:%s bsize:%ld " + KV_TRC_FFDC(pAT, "cblk_cg_open phys lun failed path:%s bsize:%ld " "size:%ld bcount:%ld, errno:%d", path, bsize, *size, *bcount, errno); goto error_exit; } - rc = cblk_get_size(chkid, (size_t *)bcount, 0); + rc = cblk_cg_get_size(chkid, (size_t *)bcount, CBLK_GROUP_RAID0); if ( (rc != 0) || (*bcount == 0) ) { // An error was encountered, close the chunk - cblk_close(chkid, 0); + cblk_cg_close(chkid, CBLK_GROUP_RAID0); chkid = NULL_CHUNK_ID; - KV_TRC_FFDC(pAT, "cblk_get_size failed path %s bsize %"PRIu64" " + KV_TRC_FFDC(pAT, "cblk_cg_get_size failed path %s bsize %"PRIu64" " "size %"PRIu64" bcount %"PRIu64", errno = %d", path, bsize, *size, *bcount, errno); goto error_exit; @@ -151,14 +149,13 @@ EA *ea_new(const char *path, uint64_t bsize, int basyncs, } else { - KV_TRC(pAT, "cblk_open VIRTUAL LUN: %s", path); - chkid = cblk_open(path, basyncs, O_RDWR, ext, - CBLK_OPN_VIRT_LUN|CBLK_OPN_NO_INTRP_THREADS); + KV_TRC(pAT, "cblk_cg_open VIRTUAL LUN: %s", path); + chkid = cblk_cg_open(path, basyncs, O_RDWR, 1, 0, + CBLK_GROUP_RAID0|CBLK_OPN_VIRT_LUN|CBLK_OPN_NO_INTRP_THREADS); if (NULL_CHUNK_ID == chkid) { - printf("cblk_open virtual lun failed\n"); - KV_TRC_FFDC(pAT, "cblk_open virt lun failed path:%s bsize:%ld " + KV_TRC_FFDC(pAT, "cblk_cg_open virt lun failed path:%s bsize:%ld " "size:%ld bcount:%ld, errno:%d", path, bsize, *size, *bcount, errno); goto error_exit; @@ -167,14 +164,13 @@ EA *ea_new(const char *path, uint64_t bsize, int basyncs, // A specific size was passed in so we try to set the // size of the chunk. *bcount = *size / bsize; - rc = cblk_set_size(chkid, (size_t)*bcount, 0); + rc = cblk_cg_set_size(chkid, (size_t)*bcount, CBLK_GROUP_RAID0); if ( rc != 0 ) { - printf("cblk_set_size failed for %ld\n", *bcount); // An error was encountered, close the chunk - cblk_close(chkid, 0); + cblk_cg_close(chkid, CBLK_GROUP_RAID0); chkid = NULL_CHUNK_ID; - KV_TRC_FFDC(pAT, "cblk_set_size failed path %s bsize %"PRIu64" " + KV_TRC_FFDC(pAT, "cblk_cg_set_size failed path %s bsize %"PRIu64" " "size %"PRIu64" bcount %"PRIu64", errno = %d", path, bsize, *size, *bcount, errno); goto error_exit; @@ -187,7 +183,7 @@ EA *ea_new(const char *path, uint64_t bsize, int basyncs, ea->st_device = (char *)am_malloc(plen); if (!ea->st_device) { - cblk_close(chkid, 0); + cblk_cg_close(chkid, CBLK_GROUP_RAID0); KV_TRC_FFDC(pAT, "MALLOC st_device failed (%s) plen=%ld errno:%d", path, plen, errno); goto error_exit; @@ -244,7 +240,7 @@ int ea_resize(EA *ea, uint64_t bsize, uint64_t bcount) { // Call down to the block layer to set the // new size on the store. - rc = cblk_set_size(ea->st_flash, bcount, 0); + rc = cblk_cg_set_size(ea->st_flash, bcount, CBLK_GROUP_RAID0); if (rc == 0) { ea->bcount = bcount; @@ -253,7 +249,7 @@ int ea_resize(EA *ea, uint64_t bsize, uint64_t bcount) else { errno = ENOSPC; - KV_TRC_FFDC(pAT, "cblk_set_size failed ea %p bsize %lu bcount %lu, " + KV_TRC_FFDC(pAT, "cblk_cg_set_size failed ea %p bsize %lu bcount %lu, " "errno = %d", ea, bsize, bcount, errno); } @@ -277,7 +273,7 @@ int ea_read(EA *ea, uint64_t lba, void *dst) { else { // Call out to the block layer and retrive a block - rc = cblk_read(ea->st_flash, dst, lba, 1, 0); + rc = cblk_cg_read(ea->st_flash, dst, lba, 1, CBLK_GROUP_RAID0); } return rc; @@ -296,7 +292,7 @@ int ea_write(EA *ea, uint64_t lba, void *src) { else { // Send the value down to the block layer, 1 block - rc = cblk_write(ea->st_flash, src, lba, 1, 0); + rc = cblk_cg_write(ea->st_flash, src, lba, 1, CBLK_GROUP_RAID0); } return rc; @@ -371,13 +367,13 @@ int ea_async_io(EA *ea, int op, void *addr, ark_io_list_t *blist, int64_t len, i // of blocks read), or IO has been scheduled (rc == 0). if (op == ARK_EA_READ) { - rc = cblk_aread(ea->st_flash, p_addr, blist[i].blkno, 1, - &(blist[i].a_tag), NULL,CBLK_ARW_WAIT_CMD_FLAGS); + rc = cblk_cg_aread(ea->st_flash, p_addr, blist[i].blkno, 1, + &(blist[i].a_tag), NULL, CBLK_GROUP_RAID0|CBLK_ARW_WAIT_CMD_FLAGS); } else { - rc = cblk_awrite(ea->st_flash, p_addr, blist[i].blkno, 1, - &(blist[i].a_tag), NULL,CBLK_ARW_WAIT_CMD_FLAGS); + rc = cblk_cg_awrite(ea->st_flash, p_addr, blist[i].blkno, 1, + &(blist[i].a_tag), NULL,CBLK_GROUP_RAID0|CBLK_ARW_WAIT_CMD_FLAGS); } if (check_sched_error_injects(op)) {rc=-1;} @@ -389,9 +385,9 @@ int ea_async_io(EA *ea, int op, void *addr, ark_io_list_t *blist, int64_t len, i { // Error was encountered. Don't issue any more IO rc = errno; - KV_TRC_FFDC(pAT, "IO_ERR: cblk_aread/awrite failed, " + KV_TRC_FFDC(pAT, "IO_ERR: cblk_cg_aread/awrite failed, " "blkno:%"PRIi64" tag:%d, errno = %d", - blist[i].blkno, blist[i].a_tag, errno); + blist[i].blkno, blist[i].a_tag.tag, errno); break; } @@ -399,7 +395,7 @@ int ea_async_io(EA *ea, int op, void *addr, ark_io_list_t *blist, int64_t len, i // wait for the response below if ( rc > 0 ) { - blist[i].a_tag = -1; + blist[i].a_tag.tag = -1; rc = 0; } //_arkp->stats.io_cnt++; @@ -412,15 +408,15 @@ int ea_async_io(EA *ea, int op, void *addr, ark_io_list_t *blist, int64_t len, i { // Data has already been read - if (blist[j].a_tag == -1) + if (blist[j].a_tag.tag == -1) { continue; } do { - a_rc = cblk_aresult(ea->st_flash, &(blist[j].a_tag), - &status, CBLK_ARESULT_BLOCKING); + a_rc = cblk_cg_aresult(ea->st_flash, &(blist[j].a_tag), + &status, CBLK_GROUP_RAID0|CBLK_ARESULT_BLOCKING); if (check_harv_error_injects(op)) {a_rc=-1;} @@ -477,7 +473,7 @@ int ea_delete(EA *ea) { // Call to close out the chunk and free the space // for the device name - rc = cblk_close(ea->st_flash, 0); + rc = cblk_cg_close(ea->st_flash, CBLK_GROUP_RAID0); am_free(ea->st_device); } diff --git a/src/kv/ea_mod.c b/src/kv/ea_mod.c index 291cc36a..fd5d2598 100644 --- a/src/kv/ea_mod.c +++ b/src/kv/ea_mod.c @@ -33,10 +33,11 @@ #include #include -#include "ark.h" -#include "bl.h" -#include "ea.h" -#include "am.h" +#include +#include +#include +#include +#include #ifdef _OS_INTERNAL #include @@ -74,12 +75,15 @@ int ea_async_io_schedule(_ARK *_arkp, uint8_t *m_addr = NULL; char *ot = NULL; - KV_TRC_IO(pAT, "IO_BEG: SCHEDULE_START: tid:%d ttag:%d start:%"PRIu64" " - "nblks:%"PRIu64" issT:%d cmpT:%d", - tid, iocbp->tag, iocbp->start, iocbp->nblks, - iocbp->issT, iocbp->cmpT); + KV_TRC_IO(pAT, "IO_BEG: tid:%d ttag:%3d start:%4"PRIu64" " + "issT:%3d cmpT:%3d nblks:%3"PRIu64" addr:%p", + tid, iocbp->tag, iocbp->start, + iocbp->issT, iocbp->cmpT, iocbp->nblks, iocbp->addr); ARK_SYNC_EA_READ(iocbp->ea); + errno = 0; + iocbp->io_error = 0; + iocbp->stime = getticks(); if (iocbp->op == ARK_EA_READ) {ot="IO_RD";} else {ot="IO_WR";} @@ -88,8 +92,9 @@ int ea_async_io_schedule(_ARK *_arkp, { if (ea->st_type == EA_STORE_TYPE_MEMORY) { - p_addr = ((uint8_t *)(iocbp->addr)) + (i * ea->bsize); - m_addr = ea->st_memory + (iocbp->blist[i].blkno * ea->bsize); + p_addr = ((uint8_t *)(iocbp->addr)) + (i * ea->bsize); + m_addr = ea->st_memory + (iocbp->blist[i].blkno * ea->bsize); + iocbp->issT += 1; ++_arkp->issT; if (ARK_EA_READ == iocbp->op) {prc = memcpy(p_addr,m_addr,ea->bsize);} else {prc = memcpy(m_addr,p_addr,ea->bsize);} @@ -100,77 +105,87 @@ int ea_async_io_schedule(_ARK *_arkp, if (prc == NULL) { rc=FALSE; - KV_TRC_FFDC(pAT,"IO_ERR: tid:%d ttag:%d blkno:%"PRIi64"" - " errno:%d", tid, iocbp->tag, - iocbp->blist[i].blkno, errno); + KV_TRC_FFDC(pAT,"IO_ERR: tid:%d ttag:%3d issT:%3d cmpT:%3d " + "nblks:%3ld blkno:%"PRIi64" errno:%d", + tid, iocbp->tag, iocbp->issT, iocbp->cmpT, + iocbp->nblks, iocbp->blist[i].blkno, errno); if (!errno) {KV_TRC_FFDC(pAT, "IO: UNSET_ERRNO"); errno=EIO;} iocbp->io_error = errno; break; } - ++iocbp->issT; - iocbp->blist[i].a_tag = i; + iocbp->blist[i].a_tag.tag = i; } else // r/w to hw { - p_addr = ((uint8_t *)iocbp->addr) + (i * ea->bsize); + p_addr = ((uint8_t *)iocbp->addr) + (i * ea->bsize); + iocbp->issT += 1; ++_arkp->issT; if (check_sched_error_injects(iocbp->op)) { - arc=-1; + arc = -1; } else if ( iocbp->op == ARK_EA_READ ) { - arc = cblk_aread(ea->st_flash, p_addr, iocbp->blist[i].blkno, 1, - &(iocbp->blist[i].a_tag), NULL, 0); + iocbp->rd = 1; + arc = cblk_cg_aread(ea->st_flash, p_addr, iocbp->blist[i].blkno,1, + &(iocbp->blist[i].a_tag), NULL, iocbp->aflags); } else { - arc = cblk_awrite(ea->st_flash, p_addr, iocbp->blist[i].blkno, 1, - &(iocbp->blist[i].a_tag), NULL, 0); + arc = cblk_cg_awrite(ea->st_flash, p_addr,iocbp->blist[i].blkno,1, + &(iocbp->blist[i].a_tag), NULL, iocbp->aflags); } - if (arc == 0) // good status - { - ++iocbp->issT; rc=FALSE; - } - else if (arc < 0) + if (arc == 0) {rc=FALSE;} // good status + else if (arc < 0) { rc=FALSE; if (errno == EAGAIN) { + iocbp->issT -= 1; --_arkp->issT; // return, and an ark thread will re-schedule this iocb - KV_TRC_DBG(pAT,"IO: RW_EAGAIN: tid:%d ttag:%d " - "blkno:%"PRIi64"", - tid, iocbp->tag, iocbp->blist[i].blkno); + KV_TRC(pAT,"IO: tid:%d ttag:%3d EAGAIN " + "issT:%3d cmpT:%3d nblks:%3ld blkno:%"PRIi64"", + tid, iocbp->tag, iocbp->issT, iocbp->cmpT, + iocbp->nblks, iocbp->blist[i].blkno); break; } // Something bad went wrong, fail the iocb - KV_TRC_FFDC(pAT,"IO_ERR: tid:%d ttag:%d blkno:%"PRIi64"" - " errno:%d", tid, iocbp->tag, - iocbp->blist[i].blkno, errno); + KV_TRC_FFDC(pAT,"IO_ERR: tid:%d ttag:%3d issT:%3d cmpT:%3d " + "nblks:%3ld blkno:%"PRIi64" errno:%d", + tid, iocbp->tag, iocbp->issT, iocbp->cmpT, + iocbp->nblks, iocbp->blist[i].blkno, errno); if (!errno) {KV_TRC_FFDC(pAT, "IO: UNSET_ERRNO"); errno=EIO;} + iocbp->cmpT += 1; iocbp->io_error = errno; break; } else if (arc > 0) { - KV_TRC_IO(pAT,"IO_CMP: IMMEDIATE: tid:%d ttag:%d a_tag:%d " - "blkno:%"PRIi64"", - tid, iocbp->tag, - iocbp->blist[i].a_tag, iocbp->blist[i].blkno); + KV_TRC(pAT,"IO_CMP: tid:%d ttag:%3d a_tag:%4d IMMEDIATE " + "issT:%3d cmpT:%3d nblks:%3ld blkno:%"PRIi64"", + tid, iocbp->tag, iocbp->blist[i].a_tag.tag, + iocbp->issT, iocbp->cmpT, iocbp->nblks, + iocbp->blist[i].blkno); ++iocbp->issT; ++iocbp->cmpT; - iocbp->blist[i].a_tag = -1; // mark as harvested + iocbp->blist[i].a_tag.tag = -1; // mark as harvested } } - KV_TRC_IO(pAT, "%s: tid:%2d ttag:%4d a_tag:%4d blkno:%5"PRIi64"", ot,tid, - iocbp->tag, iocbp->blist[i].a_tag, iocbp->blist[i].blkno); - + KV_TRC_IO(pAT, "%s: tid:%d ttag:%3d a_tag:%4d issT:%3d cmpT:%3d " + "nblks:%3ld blkno:%5"PRIi64"", ot,tid, iocbp->tag, + iocbp->blist[i].a_tag.tag, iocbp->issT, iocbp->cmpT, + iocbp->nblks, iocbp->blist[i].blkno); } - iotcbp->state = ARK_IO_HARVEST; - iocbp->start = i; + /* if a cmd schedule failed OR cmds are waiting for harvest, call harvest */ + if ((iocbp->io_error) || + (iocbp->issT && (iocbp->issT != iocbp->cmpT))) + { + iotcbp->state = ARK_IO_HARVEST; + iocbp->start = i; + } ARK_SYNC_EA_UNLOCK(iocbp->ea); return rc; @@ -190,14 +205,11 @@ int ea_async_io_harvest(_ARK *_arkp, EA *ea = iocbp->ea; int32_t i = 0; int32_t arc = 0; + int32_t hrvst = (iocbp->aflags & CBLK_ARESULT_NO_HARVEST)==0; int32_t rc = FALSE; uint64_t status = 0; - scb_t *scbp = &(_arkp->poolthreads[tid]); - queue_t *rq = scbp->rqueue; - queue_t *tq = scbp->tqueue; - queue_t *ioq = scbp->ioqueue; - for (i=0; iissT; i++) + for (i=0; inblks; i++) { if (EA_STORE_TYPE_MEMORY == ea->st_type) { @@ -208,48 +220,61 @@ int ea_async_io_harvest(_ARK *_arkp, else { // skip previously harvested cmd - if (iocbp->blist[i].a_tag == -1) {continue;} + if (iocbp->blist[i].a_tag.tag == -1) {continue;} - arc = cblk_aresult(ea->st_flash, &(iocbp->blist[i].a_tag), &status,0); + arc = cblk_cg_aresult(ea->st_flash, + &iocbp->blist[i].a_tag, &status, iocbp->aflags); } if (check_harv_error_injects(iocbp->op)) {arc=-1;} if (arc == 0) { - KV_TRC_DBG(pAT,"IO: WAIT_NOT_CMP: tid:%d ttag:%d a_tag:%d " - "blkno:%"PRIi64"", - tid, iocbp->tag, iocbp->blist[i].a_tag, - iocbp->blist[i].blkno); + KV_TRC_DBG(pAT,"IO: tid:%d ttag:%3d a_tag:%4d issT:%3d cmpT:%3d " + "nblks:%3ld blkno:%5ld hrvst:%d WAIT_NOT_CMP", + tid, iocbp->tag, iocbp->blist[i].a_tag.tag, + iocbp->issT, iocbp->cmpT, iocbp->nblks, + iocbp->blist[i].blkno, hrvst); ++iocbp->hmissN; - - // if nothing to do and the first harvest missed, usleep - if (queue_empty(rq) && queue_empty(tq) && queue_count(ioq)<=8 && - iocbp->hmissN==1 && - _arkp->ea->st_type != EA_STORE_TYPE_MEMORY) - { - usleep(50); - KV_TRC_DBG(pAT,"IO: USLEEP"); - } - break; + continue; } + ++iocbp->cmpT; --_arkp->issT; + if (arc < 0) { - KV_TRC_FFDC(pAT, "IO_ERR: tid:%d ttag:%d errno=%d", - tid, iocbp->tag, errno); + KV_TRC_FFDC(pAT, "IO_ERR: tid:%d ttag:%3d a_tag:%4d issT:%3d " + "cmpT:%3d nblks:%3ld lat:%d errno=%d", + tid, iocbp->tag, iocbp->blist[i].a_tag.tag, + iocbp->issT, iocbp->cmpT, iocbp->nblks, + iocbp->lat, errno); if (!errno) {KV_TRC_FFDC(pAT, "UNSET_ERRNO"); errno=EIO;} iocbp->io_error = errno; } else { - KV_TRC_IO(pAT,"IO_CMP: tid:%2d ttag:%4d a_tag:%4d blkno:%5"PRIi64"", - tid, iocbp->tag, - iocbp->blist[i].a_tag, iocbp->blist[i].blkno); + iocbp->hmissN=0; + if (iocbp->nblks==1) + { + /* log per cmd latency if 1 blk io */ + iocbp->lat=UDELTA(iocbp->stime, _arkp->ns_per_tick); + KV_TRC_PERF2(pAT,"IO_CMP: tid:%d ttag:%3d a_tag:%4d issT:%3d " + "cmpT:%3d nblks:%3ld blkno:%5ld rd:%d lat:%d", + tid, iocbp->tag, iocbp->blist[i].a_tag.tag, + iocbp->issT, iocbp->cmpT, iocbp->nblks, + iocbp->blist[i].blkno, iocbp->rd, iocbp->lat); + } + else + { + KV_TRC_IO(pAT,"IO_CMP: tid:%d ttag:%3d a_tag:%4d issT:%3d " + "cmpT:%3d nblks:%3ld blkno:%5ld rd:%d", + tid, iocbp->tag, iocbp->blist[i].a_tag.tag, + iocbp->issT, iocbp->cmpT, iocbp->nblks, + iocbp->blist[i].blkno, iocbp->rd); + } } - ++iocbp->cmpT; - iocbp->blist[i].a_tag = -1; // mark as harvested + iocbp->blist[i].a_tag.tag = -1; // mark as harvested } if (iocbp->io_error) @@ -262,15 +287,16 @@ int ea_async_io_harvest(_ARK *_arkp, iorcbp->rc = iocbp->io_error; iotcbp->state = ARK_CMD_DONE; am_free(iocbp->blist); - KV_TRC_FFDC(pAT, "IO: ERROR_DONE: tid:%d ttag:%d rc:%d", + KV_TRC_FFDC(pAT, "IO: tid:%d ttag:%3d ERROR_DONE rc:%d", tid, iocbp->tag, iorcbp->rc); } else { // IOs outstanding, harvest the remaining IOs for this iocb - KV_TRC_FFDC(pAT,"IO: ERROR_RE_HARVEST: tid:%d ttag:%d " - "iocbp->issT:%d iocbp->cmpT:%d", - tid, iocbp->tag, iocbp->issT, iocbp->cmpT); + KV_TRC_FFDC(pAT,"IO: tid:%d ttag:%3d ERROR_RE_HARVEST " + "issT:%3d cmpT:%3d nblks:%3ld", + tid, iocbp->tag, iocbp->issT, iocbp->cmpT, + iocbp->nblks); } } // if all IO has completed successfully for this iocb, done @@ -278,24 +304,32 @@ int ea_async_io_harvest(_ARK *_arkp, { rc=TRUE; am_free(iocbp->blist); - iotcbp->state = ARK_IO_DONE; - KV_TRC_IO(pAT, "IO_END: SUCCESS tid:%d ttag:%d cmpT:%d", + iotcbp->state = iocbp->io_done; + KV_TRC_IO(pAT, "IO_END: tid:%d ttag:%3d SUCCESS cmpT:%d", tid, iocbp->tag, iocbp->cmpT); } + // if more blks need a harvest + else if (iocbp->cmpT < iocbp->issT) + { + iotcbp->state = ARK_IO_HARVEST; + KV_TRC_IO(pAT,"IO: tid:%d ttag:%3d hrvst:%4d " + "issT:%3d cmpT:%3d nblks:%3ld RE_HARVEST_PART", + tid, iocbp->tag, hrvst, iocbp->issT, iocbp->cmpT, iocbp->nblks); + } // if more blks need an IO, schedule else if (iocbp->issT < iocbp->nblks) { iotcbp->state = ARK_IO_SCHEDULE; - KV_TRC_IO(pAT,"IO: RE_SCHEDULE: tid:%d ttag:%d " - "iocbp->issT:%d iocbp->nblks:%"PRIi64" ", - tid, iocbp->tag, iocbp->issT, iocbp->nblks); + KV_TRC_IO(pAT,"IO: tid:%d ttag:%3d hrvst:%4d " + "issT:%3d cmpT:%3d, nblks:%3"PRIi64" RE_SCHEDULE", + tid, iocbp->tag, hrvst, iocbp->issT, iocbp->cmpT, iocbp->nblks); } else { // all IOs have been issued but not all are completed, do harvest - KV_TRC_IO(pAT,"IO: RE_HARVEST: tid:%d ttag:%d " - "iocbp->cmpT:%d iocbp->issT:%d", - tid, iocbp->tag, iocbp->cmpT, iocbp->issT); + KV_TRC_IO(pAT,"IO: tid:%d ttag:%3d hrvst:%4d " + "issT:%3d cmpT:%3d nblks:%3ld RE_HARVEST", + tid, iocbp->tag, hrvst, iocbp->issT, iocbp->cmpT, iocbp->nblks); } return rc; } @@ -309,7 +343,9 @@ void ea_async_io_init(_ARK *_arkp, int op, void *addr, ark_io_list_t *blist, int64_t nblks, int start, int32_t tag, int32_t io_done) { iocb_t *iocbp = &(_arkp->iocbs[tag]); + tcb_t *iotcbp = &(_arkp->tcbs[tag]); + iotcbp->state = ARK_IO_SCHEDULE; iocbp->ea = _arkp->ea; iocbp->op = op; iocbp->addr = addr; @@ -318,7 +354,10 @@ void ea_async_io_init(_ARK *_arkp, int op, void *addr, ark_io_list_t *blist, iocbp->start = start; iocbp->issT = 0; iocbp->cmpT = 0; + iocbp->rd = 0; + iocbp->lat = 0; iocbp->hmissN = 0; + iocbp->aflags = CBLK_GROUP_RAID0; iocbp->io_error = 0; iocbp->io_done = io_done; iocbp->tag = tag; diff --git a/src/kv/kv_trace.h b/src/kv/kv_trace.h index 9818031c..2c4f6ef6 100644 --- a/src/kv/kv_trace.h +++ b/src/kv/kv_trace.h @@ -34,8 +34,11 @@ * OFF KV_TRC_VERBOSITY == 0 * KV_TRC_FFDC KV_TRC_VERBOSITY <= 1 * KV_TRC KV_TRC_VERBOSITY <= 2 - * KV_TRC_IO KV_TRC_VERBOSITY <= 3 - * KV_TRC_DBG KV_TRC_VERBOSITY <= 4 + * KV_TRC_PERF KV_TRC_VERBOSITY <= 3 + * KV_TRC_IO KV_TRC_VERBOSITY <= 4 + * KV_TRC_DBG KV_TRC_VERBOSITY <= 5 + * KV_TRC_DBG6 KV_TRC_VERBOSITY <= 6 + * KV_TRC_DBG7 KV_TRC_VERBOSITY <= 7 * \ingroup ******************************************************************************/ #ifndef _H_KV_TRACE_LOG @@ -54,7 +57,7 @@ #define B_EYE 0xABCDBEEF #define E_EYE 0xDCBAFEED -#define STRLEN 256 +#define STRLEN 1024 #define EYEC_INVALID(_p_KT) ((_p_KT->b_eye != B_EYE) || (_p_KT->e_eye != E_EYE)) @@ -79,11 +82,13 @@ typedef struct * \brief * setup to run kv log trace macro ******************************************************************************/ -#define KV_TRC_OPEN( _pKT, _filename) \ +#define KV_TRC_OPEN( _pKT, _fn_in) \ do \ { \ char *env_verbosity = getenv("KV_TRC_VERBOSITY"); \ + char *env_dir = getenv("KV_TRC_DIR"); \ char *env_user = getenv("USER"); \ + char dir[STRLEN+1] = {0}; \ char fn[STRLEN+1] = {0}; \ int pid = getpid(); \ struct timeb _cur_time; \ @@ -106,8 +111,11 @@ do \ \ if (_pKT->init_done >= 1) {++_pKT->init_done; goto open_unlock;} \ \ - if (_filename) snprintf(fn, STRLEN,"/tmp/%s.%s.kv.log",env_user,_filename);\ - else snprintf(fn, STRLEN,"/tmp/%s.kv.log", env_user); \ + if (env_dir && strlen(env_dir)<256) {sprintf(dir, "%s", env_dir);} \ + else {sprintf(dir, "/tmp");} \ + \ + if (_fn_in) snprintf(fn, STRLEN, "%s/%s.%s.kv.log", dir, env_user, _fn_in);\ + else snprintf(fn, STRLEN, "%s/%s.kv.log", dir, env_user); \ \ if ((_pKT->logfp = fopen(fn, "a")) == NULL) \ { \ @@ -225,7 +233,7 @@ do \ } \ } while (0) -#define KV_TRC(_pKT, msg, ...) \ +#define KV_TRC_PERF1(_pKT, msg, ...) \ do \ { \ if (_pKT && _pKT->verbosity >= 2) \ @@ -234,7 +242,7 @@ do \ } \ } while (0) -#define KV_TRC_IO(_pKT, msg, ...) \ +#define KV_TRC_PERF2(_pKT, msg, ...) \ do \ { \ if (_pKT && _pKT->verbosity >= 3) \ @@ -243,7 +251,7 @@ do \ } \ } while (0) -#define KV_TRC_DBG(_pKT, msg, ...) \ +#define KV_TRC(_pKT, msg, ...) \ do \ { \ if (_pKT && _pKT->verbosity >= 4) \ @@ -251,8 +259,53 @@ do \ KV_TRACE_LOG_DATA(_pKT, msg, ## __VA_ARGS__); \ } \ } while (0) + +#define KV_TRC_IO(_pKT, msg, ...) \ + do \ + { \ + if (_pKT && _pKT->verbosity >= 5) \ + { \ + KV_TRACE_LOG_DATA(_pKT, msg, ## __VA_ARGS__); \ + } \ + } while (0) + +#define KV_TRC_DBG(_pKT, msg, ...) \ + do \ + { \ + if (_pKT && _pKT->verbosity >= 6) \ + { \ + KV_TRACE_LOG_DATA(_pKT, msg, ## __VA_ARGS__); \ + } \ + } while (0) + +#define KV_TRC_EXT1(_pKT, msg, ...) \ + do \ + { \ + if (_pKT && _pKT->verbosity >= 7) \ + { \ + KV_TRACE_LOG_DATA(_pKT, msg, ## __VA_ARGS__); \ + } \ + } while (0) + +#define KV_TRC_EXT2(_pKT, msg, ...) \ + do \ + { \ + if (_pKT && _pKT->verbosity >= 8) \ + { \ + KV_TRACE_LOG_DATA(_pKT, msg, ## __VA_ARGS__); \ + } \ + } while (0) + +#define KV_TRC_EXT3(_pKT, msg, ...) \ + do \ + { \ + if (_pKT && _pKT->verbosity >= 9) \ + { \ + KV_TRACE_LOG_DATA(_pKT, msg, ## __VA_ARGS__); \ + } \ + } while (0) #else -#define KV_TRC_OPEN(_pKT, _filename) +#define KV_TRC_OPEN(_pKT, _fn_in) #define KV_TRC_CLOSE(_pKT) #define KV_TRC_FFDC(_pKT, _fmt, ...) #define KV_TRC(_pKT, _fmt, ...) diff --git a/src/kv/test/fvt_ark_perf_tool.C b/src/kv/test/fvt_ark_perf_tool.C index 58b6dbac..3c35168b 100644 --- a/src/kv/test/fvt_ark_perf_tool.C +++ b/src/kv/test/fvt_ark_perf_tool.C @@ -123,7 +123,8 @@ int main(int argc, char **argv) if (mem) penv=NULL; - if (vlen < KV_4K) LEN=200; + if (vlen < 64) LEN=LEN; + else if (vlen < KV_4K) LEN=200; else if (vlen < KV_64K) LEN=50; else if (vlen < KV_250K) LEN=10; else LEN=1; diff --git a/src/kv/test/fvt_kv.h b/src/kv/test/fvt_kv.h index 4eac4823..51e38f7a 100644 --- a/src/kv/test/fvt_kv.h +++ b/src/kv/test/fvt_kv.h @@ -76,13 +76,39 @@ #define TESTCASE_SKIP(_reason) \ printf("[ SKIPPED ] %s\n", _reason); -#define TESTCASE_SKIP_IF_FILE \ - char *dev = getenv("FVT_DEV"); \ - if (dev != NULL && strncmp("/dev/", dev, 5) != 0) \ - { \ - TESTCASE_SKIP("FVT_DEV looks like a file"); \ - return; \ - } +#define TESTCASE_SKIP_IF_FILE \ +do \ +{ \ + char *dev = getenv("FVT_DEV"); \ + if (dev != NULL && (!(strncmp("/dev/", dev, 5)==0 || \ + strncmp("RAID", dev, 4)==0))) \ + { \ + TESTCASE_SKIP("FVT_DEV looks like a file"); \ + return; \ + } \ +} while (0); + +#define TESTCASE_SKIP_IF_MEM \ +do \ +{ \ + char *dev = getenv("FVT_DEV"); \ + if (dev == NULL) \ + { \ + TESTCASE_SKIP("FVT_DEV looks like MEM"); \ + return; \ + } \ +} while (0); + +#define TESTCASE_SKIP_IF_RAID0 \ +do \ +{ \ + char *dev = getenv("FVT_DEV"); \ + if (dev != NULL && (strncmp("RAID", dev, 4)==0)) \ + { \ + TESTCASE_SKIP("FVT_DEV is RAID0"); \ + return; \ + } \ +} while (0); #define KV_4K 4 *1024 #define KV_8K 8 *1024 diff --git a/src/kv/test/fvt_kv_tst_async_cb.C b/src/kv/test/fvt_kv_tst_async_cb.C index 936fc782..bd067b6c 100644 --- a/src/kv/test/fvt_kv_tst_async_cb.C +++ b/src/kv/test/fvt_kv_tst_async_cb.C @@ -98,6 +98,19 @@ TEST(FVT_KV_GOOD_PATH, ASYNC_CB_BIG_BLOCKS) kv_async_run_jobs(); } +/** + ******************************************************************************* + * \brief + ******************************************************************************/ +TEST(FVT_KV_GOOD_PATH, ASYNC_CB_BIG_BLOCKS_STARVE) +{ + uint32_t secs = 5; + + kv_async_init_ctxt_starve (ASYNC_SINGLE_CONTEXT, 20, 256, secs); + kv_async_init_job_BIG_BLOCKS(ASYNC_SINGLE_CONTEXT); + kv_async_run_jobs(); +} + /** ******************************************************************************* * \brief diff --git a/src/kv/test/fvt_kv_tst_errors.C b/src/kv/test/fvt_kv_tst_errors.C index 58fc47d2..9af8a59b 100644 --- a/src/kv/test/fvt_kv_tst_errors.C +++ b/src/kv/test/fvt_kv_tst_errors.C @@ -42,7 +42,6 @@ extern "C" #include #include -ARK *async_ark = NULL; uint32_t async_io = 0; int32_t async_err = 0; @@ -53,7 +52,7 @@ int32_t async_err = 0; void kv_tst_io_errors_cb(int errcode, uint64_t dt, int64_t res) { --async_io; - if (async_err != errcode) printf("tag=%"PRIx64"\n", dt); + if (async_err != errcode) printf("tag=%lx\n", dt); ASSERT_EQ(async_err, errcode); } @@ -305,6 +304,7 @@ TEST(FVT_KV_ERROR_PATH, BAD_PARMS_ark_fork) ******************************************************************************/ TEST(FVT_KV_ERROR_PATH, ALLOC_ERRORS) { + ARK *ark = NULL; char s[] = {"12345678"}; uint32_t klen = 8; int64_t res = 0; @@ -313,58 +313,58 @@ TEST(FVT_KV_ERROR_PATH, ALLOC_ERRORS) errno=0; KV_INJECT_ALLOC_ERROR; ASSERT_EQ(ENOMEM, ark_create(getenv("FVT_DEV"), - &async_ark, ARK_KV_VIRTUAL_LUN)); + &ark, ARK_KV_VIRTUAL_LUN)); errno=0; - ASSERT_EQ(0, ark_create(getenv("FVT_DEV"), &async_ark, ARK_KV_VIRTUAL_LUN)); - EXPECT_TRUE(async_ark != NULL); + ASSERT_EQ(0, ark_create(getenv("FVT_DEV"), &ark, ARK_KV_VIRTUAL_LUN)); + EXPECT_TRUE(ark != NULL); /* sync */ errno=0; KV_INJECT_ALLOC_ERROR; - EXPECT_EQ(ENOMEM, ark_set(async_ark, klen, s, klen, s, &res)); - EXPECT_EQ(ENOENT, ark_get(async_ark, klen, s, klen, s, 0, &res)); + EXPECT_EQ(ENOMEM, ark_set(ark, klen, s, klen, s, &res)); + EXPECT_EQ(ENOENT, ark_get(ark, klen, s, klen, s, 0, &res)); - EXPECT_EQ(0, ark_set(async_ark, klen, s, klen, s, &res)); - EXPECT_EQ(0, ark_get(async_ark, klen, s, klen, s, 0, &res)); + EXPECT_EQ(0, ark_set(ark, klen, s, klen, s, &res)); + EXPECT_EQ(0, ark_get(ark, klen, s, klen, s, 0, &res)); errno=0; KV_INJECT_ALLOC_ERROR; - EXPECT_EQ(ENOMEM, ark_del(async_ark, klen, s, &res)); - EXPECT_EQ(0, ark_del(async_ark, klen, s, &res)); + EXPECT_EQ(ENOMEM, ark_del(ark, klen, s, &res)); + EXPECT_EQ(0, ark_del(ark, klen, s, &res)); errno=0; ++async_io; async_err = ENOMEM; KV_INJECT_ALLOC_ERROR; - EXPECT_EQ(0, ark_set_async_cb(async_ark, klen, s, klen, s, + EXPECT_EQ(0, ark_set_async_cb(ark, klen, s, klen, s, kv_tst_io_errors_cb, 0xfee1)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = ENOENT; - EXPECT_EQ(0, ark_get_async_cb(async_ark, klen, s, klen, s, 0, + EXPECT_EQ(0, ark_get_async_cb(ark, klen, s, klen, s, 0, kv_tst_io_errors_cb, 0xfee2)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = 0; - EXPECT_EQ(0, ark_set_async_cb(async_ark, klen, s, klen, s, + EXPECT_EQ(0, ark_set_async_cb(ark, klen, s, klen, s, kv_tst_io_errors_cb, 0xfee3)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = 0; - EXPECT_EQ(0, ark_get_async_cb(async_ark, klen, s, klen, s, 0, + EXPECT_EQ(0, ark_get_async_cb(ark, klen, s, klen, s, 0, kv_tst_io_errors_cb, 0xfee4)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = ENOMEM; KV_INJECT_ALLOC_ERROR; - EXPECT_EQ(0, ark_del_async_cb(async_ark, klen, s, + EXPECT_EQ(0, ark_del_async_cb(ark, klen, s, kv_tst_io_errors_cb, 0xfee5)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = 0; - EXPECT_EQ(0, ark_del_async_cb(async_ark, klen, s, + EXPECT_EQ(0, ark_del_async_cb(ark, klen, s, kv_tst_io_errors_cb, 0xfee6)); while (async_io) usleep(50000); errno=0; KV_SET_INJECT_INACTIVE; - ASSERT_EQ(0, ark_delete(async_ark)); + ASSERT_EQ(0, ark_delete(ark)); } /** @@ -373,20 +373,21 @@ TEST(FVT_KV_ERROR_PATH, ALLOC_ERRORS) ******************************************************************************/ TEST(FVT_KV_ERROR_PATH, IO_ERRORS) { - char s[] = {"12345678"}; - uint32_t klen = 8; - int64_t res = 0; - char *path = getenv("FVT_DEV"); + ARK *ark = NULL; + char s[] = {"12345678"}; + uint32_t klen = 8; + int64_t res = 0; + char *path = getenv("FVT_DEV"); + ark_io_list_t *bl_array = NULL; + _ARK *arkp = NULL; char data[KV_4K]; - ark_io_list_t *bl_array = NULL; - _ARK *arkp = NULL; KV_SET_INJECT_ACTIVE; - ASSERT_EQ(0, ark_create(path, &async_ark, ARK_KV_VIRTUAL_LUN)); - EXPECT_TRUE(async_ark != NULL); + ASSERT_EQ(0, ark_create(path, &ark, ARK_KV_VIRTUAL_LUN)); + EXPECT_TRUE(ark != NULL); - arkp = (_ARK*)async_ark; + arkp = (_ARK*)ark; bl_array = bl_chain_blocks(arkp->bl, 0, 1); ASSERT_TRUE(NULL != bl_array); @@ -408,98 +409,98 @@ TEST(FVT_KV_ERROR_PATH, IO_ERRORS) /* sync */ errno=0; KV_INJECT_SCHD_WRITE_ERROR; - EXPECT_EQ(EIO, ark_set(async_ark, klen, s, klen, s, &res)); + EXPECT_EQ(EIO, ark_set(ark, klen, s, klen, s, &res)); errno=0; KV_INJECT_HARV_WRITE_ERROR; - EXPECT_EQ(EIO, ark_set(async_ark, klen, s, klen, s, &res)); - EXPECT_EQ(ENOENT, ark_get(async_ark, klen, s, klen, s, 0, &res)); + EXPECT_EQ(EIO, ark_set(ark, klen, s, klen, s, &res)); + EXPECT_EQ(ENOENT, ark_get(ark, klen, s, klen, s, 0, &res)); - EXPECT_EQ(0, ark_set(async_ark, klen, s, klen, s, &res)); + EXPECT_EQ(0, ark_set(ark, klen, s, klen, s, &res)); errno=0; KV_INJECT_SCHD_READ_ERROR; - EXPECT_EQ(EIO, ark_get(async_ark, klen, s, klen, s, 0, &res)); + EXPECT_EQ(EIO, ark_get(ark, klen, s, klen, s, 0, &res)); errno=0; KV_INJECT_SCHD_READ_ERROR; - EXPECT_EQ(EIO, ark_exists(async_ark, klen, s, &res)); + EXPECT_EQ(EIO, ark_exists(ark, klen, s, &res)); errno=0; KV_INJECT_HARV_READ_ERROR; - EXPECT_EQ(EIO, ark_get(async_ark, klen, s, klen, s, 0, &res)); + EXPECT_EQ(EIO, ark_get(ark, klen, s, klen, s, 0, &res)); errno=0; KV_INJECT_HARV_READ_ERROR; - EXPECT_EQ(EIO, ark_exists(async_ark, klen, s, &res)); + EXPECT_EQ(EIO, ark_exists(ark, klen, s, &res)); #if 0 errno=0; KV_INJECT_WRITE_ERROR; - EXPECT_EQ(EIO, ark_del(async_ark, klen, s, &res)); + EXPECT_EQ(EIO, ark_del(ark, klen, s, &res)); #endif - EXPECT_EQ(0, ark_del(async_ark, klen, s, &res)); + EXPECT_EQ(0, ark_del(ark, klen, s, &res)); /* async */ errno=0; ++async_io; async_err = EIO; KV_INJECT_SCHD_WRITE_ERROR; - EXPECT_EQ(0, ark_set_async_cb(async_ark, klen, s, klen, s, + EXPECT_EQ(0, ark_set_async_cb(ark, klen, s, klen, s, kv_tst_io_errors_cb, 0xfee1)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = EIO; KV_INJECT_HARV_WRITE_ERROR; - EXPECT_EQ(0, ark_set_async_cb(async_ark, klen, s, klen, s, + EXPECT_EQ(0, ark_set_async_cb(ark, klen, s, klen, s, kv_tst_io_errors_cb, 0xfee1)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = ENOENT; - EXPECT_EQ(0, ark_get_async_cb(async_ark, klen, s, klen, s, 0, + EXPECT_EQ(0, ark_get_async_cb(ark, klen, s, klen, s, 0, kv_tst_io_errors_cb, 0xfee2)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = 0; - EXPECT_EQ(0, ark_set_async_cb(async_ark, klen, s, klen, s, + EXPECT_EQ(0, ark_set_async_cb(ark, klen, s, klen, s, kv_tst_io_errors_cb, 0xfee3)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = 0; - EXPECT_EQ(0, ark_get_async_cb(async_ark, klen, s, klen, s, 0, + EXPECT_EQ(0, ark_get_async_cb(ark, klen, s, klen, s, 0, kv_tst_io_errors_cb, 0xfee4)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = EIO; KV_INJECT_SCHD_READ_ERROR; - EXPECT_EQ(0, ark_get_async_cb(async_ark, klen, s, klen, s, 0, + EXPECT_EQ(0, ark_get_async_cb(ark, klen, s, klen, s, 0, kv_tst_io_errors_cb, 0xfee5)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = EIO; KV_INJECT_SCHD_READ_ERROR; - EXPECT_EQ(0, ark_exists_async_cb(async_ark, klen, s, + EXPECT_EQ(0, ark_exists_async_cb(ark, klen, s, kv_tst_io_errors_cb, 0xfee6)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = EIO; KV_INJECT_HARV_READ_ERROR; - EXPECT_EQ(0, ark_get_async_cb(async_ark, klen, s, klen, s, 0, + EXPECT_EQ(0, ark_get_async_cb(ark, klen, s, klen, s, 0, kv_tst_io_errors_cb, 0xfee5)); while (async_io) usleep(50000); errno=0; ++async_io; async_err = EIO; KV_INJECT_HARV_READ_ERROR; - EXPECT_EQ(0, ark_exists_async_cb(async_ark, klen, s, + EXPECT_EQ(0, ark_exists_async_cb(ark, klen, s, kv_tst_io_errors_cb, 0xfee6)); while (async_io) usleep(50000); #if 0 errno=0; ++async_io; async_err = EIO; KV_INJECT_WRITE_ERROR; - EXPECT_EQ(0, ark_del_async_cb(async_ark, klen, s, + EXPECT_EQ(0, ark_del_async_cb(ark, klen, s, kv_tst_io_errors_cb, 0xfee7)); while (async_io) usleep(50000); #endif errno=0; ++async_io; async_err = 0; - EXPECT_EQ(0, ark_del_async_cb(async_ark, klen, s, + EXPECT_EQ(0, ark_del_async_cb(ark, klen, s, kv_tst_io_errors_cb, 0xfee8)); while (async_io) usleep(50000); errno=0; KV_SET_INJECT_INACTIVE; - ASSERT_EQ(0, ark_delete(async_ark)); + ASSERT_EQ(0, ark_delete(ark)); } /** diff --git a/src/kv/test/fvt_kv_tst_persist.C b/src/kv/test/fvt_kv_tst_persist.C index 9f0dbfa9..d6229a09 100644 --- a/src/kv/test/fvt_kv_tst_persist.C +++ b/src/kv/test/fvt_kv_tst_persist.C @@ -50,7 +50,11 @@ TEST(FVT_KV_GOOD_PATH, PERSIST_FIXED_512x128x50000) uint8_t gvalue[vlen]; struct stat sbuf; - if (NULL == dev || stat(dev,&sbuf) != 0) + if (NULL == dev || + (!(strncmp(dev,"/dev",4)==0 || + strncmp(dev,"RAID",4)==0 || + stat(dev,&sbuf) == 0)) + ) { TESTCASE_SKIP("FVT_DEV_PERSIST==NULL or file not found"); return; @@ -149,7 +153,11 @@ TEST(FVT_KV_GOOD_PATH, PERSIST) uint8_t gvalue[vlen]; struct stat sbuf; - if (NULL == dev || stat(dev,&sbuf) != 0) + if (NULL == dev || + (!(strncmp(dev,"/dev",4)==0 || + strncmp(dev,"RAID",4)==0 || + stat(dev,&sbuf) == 0)) + ) { TESTCASE_SKIP("FVT_DEV_PERSIST==NULL or file not found"); return; diff --git a/src/kv/test/fvt_kv_tst_scenario.C b/src/kv/test/fvt_kv_tst_scenario.C index 03c6972a..bf9c9de0 100644 --- a/src/kv/test/fvt_kv_tst_scenario.C +++ b/src/kv/test/fvt_kv_tst_scenario.C @@ -49,7 +49,7 @@ TEST(FVT_KV_GOOD_PATH, SCENARIO_CREATE_DELETE_LOOP) for (i=0; i #include #include +#include } /** @@ -864,25 +865,31 @@ TEST(FVT_KV_GOOD_PATH, SIMPLE_long_hash_list_smallkv) { ARK *ark = NULL; kv_t *db = NULL; - int32_t LEN = 2000; + int32_t LEN = 4000; + int vlen = 32; char *dev = getenv("FVT_DEV"); ASSERT_EQ(0, ark_create_verbose(dev, &ark, - 10000*KV_4K, - KV_4K, + ARK_VERBOSE_SIZE_DEF, + ARK_VERBOSE_BSIZE_DEF, 14, - 1, - 128, - 1024, + ARK_VERBOSE_NTHRDS_DEF, + ARK_MAX_NASYNCS, + ARK_EA_BLK_ASYNC_CMDS, ARK_KV_VIRTUAL_LUN)); - db = (kv_t*)kv_db_create_fixed(LEN, 16, 256); + db = (kv_t*)kv_db_create_fixed(LEN, 16, vlen); ASSERT_TRUE(db != NULL); fvt_kv_utils_load (ark, db, LEN); - fvt_kv_utils_query(ark, db, KV_4K, LEN); + fvt_kv_utils_query(ark, db, vlen, LEN); + fvt_kv_utils_del (ark, db, LEN); + + fvt_kv_utils_load (ark, db, LEN); + fvt_kv_utils_query(ark, db, vlen, LEN); + fvt_kv_utils_query(ark, db, vlen, LEN); + fvt_kv_utils_del (ark, db, LEN); - fvt_kv_utils_del(ark, db, LEN); kv_db_destroy(db, LEN); ARK_DELETE; } @@ -895,25 +902,26 @@ TEST(FVT_KV_GOOD_PATH, SIMPLE_long_hash_list) { ARK *ark = NULL; kv_t *db = NULL; - int32_t LEN = 1000; + int32_t LEN = 500; + int vlen = 5000; char *dev = getenv("FVT_DEV"); ASSERT_EQ(0, ark_create_verbose(dev, &ark, - 10000*KV_4K, - KV_4K, - 10, - 1, - 128, - 1024, + ARK_VERBOSE_SIZE_DEF, + ARK_VERBOSE_BSIZE_DEF, + 4, + ARK_VERBOSE_NTHRDS_DEF, + ARK_MAX_NASYNCS, + ARK_EA_BLK_ASYNC_CMDS, ARK_KV_VIRTUAL_LUN)); - db = (kv_t*)kv_db_create_fixed(LEN, 4096, 4096); + db = (kv_t*)kv_db_create_fixed(LEN, 4096, 5000); ASSERT_TRUE(db != NULL); fvt_kv_utils_load (ark, db, LEN); - fvt_kv_utils_query(ark, db, KV_4K, LEN); + fvt_kv_utils_query(ark, db, vlen, LEN); + fvt_kv_utils_del (ark, db, LEN); - fvt_kv_utils_del(ark, db, LEN); kv_db_destroy(db, LEN); ARK_DELETE; } diff --git a/src/kv/test/fvt_kv_tst_sync_async.C b/src/kv/test/fvt_kv_tst_sync_async.C index aa91e692..7406de05 100644 --- a/src/kv/test/fvt_kv_tst_sync_async.C +++ b/src/kv/test/fvt_kv_tst_sync_async.C @@ -104,6 +104,56 @@ TEST(FVT_KV_GOOD_PATH, SYNC_ASYNC_EASY) kv_async_wait_jobs(); } +/** + ******************************************************************************* + * \brief + ******************************************************************************/ +TEST(FVT_KV_GOOD_PATH, SYNC_ASYNC_BIG_BLOCKS) +{ + uint32_t klen = 256; + uint32_t vlen = KV_64K; + uint32_t LEN = 20; + uint32_t secs = 5; + + kv_async_init_ctxt(ASYNC_SINGLE_CONTEXT, secs); + kv_async_init_job_BIG_BLOCKS(ASYNC_SINGLE_CONTEXT); + + printf("start async jobs\n"); + kv_async_start_jobs(); + + printf("start sync job\n"); + fvt_kv_utils_SGD_LOOP(kv_async_get_ark(ASYNC_SINGLE_CONTEXT), + kv_db_create_fixed, klen, vlen, LEN, secs); + + printf("wait for async jobs\n"); + kv_async_wait_jobs(); +} + +/** + ******************************************************************************* + * \brief + ******************************************************************************/ +TEST(FVT_KV_GOOD_PATH, SYNC_ASYNC_STARVE) +{ + uint32_t klen = 256; + uint32_t vlen = KV_64K; + uint32_t LEN = 20; + uint32_t secs = 5; + + kv_async_init_ctxt_starve(ASYNC_SINGLE_CONTEXT, 20, 256, secs); + kv_async_init_job_BIG_BLOCKS(ASYNC_SINGLE_CONTEXT); + + printf("start async jobs\n"); + kv_async_start_jobs(); + + printf("start sync job\n"); + fvt_kv_utils_SGD_LOOP(kv_async_get_ark(ASYNC_SINGLE_CONTEXT), + kv_db_create_fixed, klen, vlen, LEN, secs); + + printf("wait for async jobs\n"); + kv_async_wait_jobs(); +} + void pr_perf_fail(const char *s, uint32_t x, uint32_t y) { printf("*******************************\n"); @@ -119,7 +169,6 @@ TEST(FVT_KV_GOOD_PATH, SYNC_ASYNC_PERF) { uint32_t num_ctxt = 1; uint32_t num_pth = 128; - uint32_t npool = 1; uint32_t vlen = 16; uint32_t LEN = 500; uint32_t secs = 15; @@ -128,39 +177,39 @@ TEST(FVT_KV_GOOD_PATH, SYNC_ASYNC_PERF) uint32_t e_ios = 0; uint32_t e_ops = 0; + TESTCASE_SKIP_IF_FILE; + Sync_pth sync_pth; num_ctxt = 1; e_ops=e_ios=70000; - sync_pth.run_multi_ctxt(num_ctxt, num_pth, npool, vlen, LEN,secs,&ops,&ios); + sync_pth.run_multi_ctxt(num_ctxt, num_pth, vlen, LEN,secs,&ops,&ios); if (ops < e_ops) pr_perf_fail("ops", ops, e_ops); if (ios < e_ios) pr_perf_fail("ios", ios, e_ios); num_ctxt=1; vlen=KV_64K; num_pth=40; e_ops=11500; e_ios=112000; - sync_pth.run_multi_ctxt(num_ctxt, num_pth, npool, vlen, LEN,secs,&ops,&ios); + sync_pth.run_multi_ctxt(num_ctxt, num_pth, vlen, LEN,secs,&ops,&ios); if (ops < e_ops) pr_perf_fail("ops", ops, e_ops); if (ios < e_ios) pr_perf_fail("ios", ios, e_ios); #ifndef _AIX num_ctxt = 1; vlen = KV_500K; num_pth = 20; e_ops=2200; e_ios=150000; - sync_pth.run_multi_ctxt(num_ctxt, num_pth, npool, vlen, LEN,secs,&ops,&ios); + sync_pth.run_multi_ctxt(num_ctxt, num_pth, vlen, LEN,secs,&ops,&ios); if (ops < e_ops) pr_perf_fail("ops", ops, e_ops); if (ios < e_ios) pr_perf_fail("ios", ios, e_ios); #endif - TESTCASE_SKIP_IF_FILE; - num_ctxt = 4; vlen = 16; num_pth = 128; e_ops=e_ios=100000; secs=25; - sync_pth.run_multi_ctxt(num_ctxt, num_pth, npool, vlen, LEN,secs,&ops,&ios); + sync_pth.run_multi_ctxt(num_ctxt, num_pth, vlen, LEN,secs,&ops,&ios); if (ops < e_ops) pr_perf_fail("ops", ops, e_ops); if (ios < e_ios) pr_perf_fail("ios", ios, e_ios); num_ctxt = 4; num_pth = 20; vlen = KV_64K; e_ops=19000;e_ios=180000;secs=25; - sync_pth.run_multi_ctxt(num_ctxt, num_pth, npool, vlen, LEN,secs,&ops,&ios); + sync_pth.run_multi_ctxt(num_ctxt, num_pth, vlen, LEN,secs,&ops,&ios); if (ops < e_ops) pr_perf_fail("ops", ops, e_ops); if (ios < e_ios) pr_perf_fail("ios", ios, e_ios); num_ctxt = 20; vlen = 16; num_pth = 20; e_ops=e_ios=110000; secs=25; - sync_pth.run_multi_ctxt(num_ctxt, num_pth, npool, vlen, LEN,secs,&ops,&ios); + sync_pth.run_multi_ctxt(num_ctxt, num_pth, vlen, LEN,secs,&ops,&ios); if (ops < e_ops) pr_perf_fail("ops", ops, e_ops); if (ios < e_ios) pr_perf_fail("ios", ios, e_ios); } @@ -236,11 +285,7 @@ TEST(FVT_KV_GOOD_PATH, SYNC_ASYNC_MAGNUS_DIFFICULTUS) TESTCASE_SKIP_IF_FILE; #ifdef _AIX - if (dev == NULL) - { - printf("NOT_EXECUTED for memory on AIX\n"); - return; - } + TESTCASE_SKIP_IF_MEM; #endif printf("init async %dctxt/%djobs\n", num_ctxt, ops); diff --git a/src/kv/test/fvt_kv_tst_sync_pth.C b/src/kv/test/fvt_kv_tst_sync_pth.C index 73777e0a..3c0c2839 100644 --- a/src/kv/test/fvt_kv_tst_sync_pth.C +++ b/src/kv/test/fvt_kv_tst_sync_pth.C @@ -110,30 +110,47 @@ TEST(FVT_KV_GOOD_PATH, SYNC_2_PTH_1_CONTEXT) ******************************************************************************* * \brief ******************************************************************************/ -TEST(FVT_KV_GOOD_PATH, SYNC_1_PTH_2_CONTEXT) +TEST(FVT_KV_GOOD_PATH, SYNC_64_PTH_BIG_BLOCKS) { - uint32_t num_ctxt = 2; - uint32_t num_pth = 1; - uint32_t vlen = KV_4K; - uint32_t LEN = 200; + uint32_t num_ctxt = 1; + uint32_t num_pth = 64; + uint32_t vlen = KV_500K; + uint32_t LEN = 2; + uint32_t secs = 3; + + Sync_pth sync_pth; + + sync_pth.run_multi_ctxt(num_ctxt, num_pth, vlen, LEN, secs); +} + +/** + ******************************************************************************* + * \brief + ******************************************************************************/ +TEST(FVT_KV_GOOD_PATH, SYNC_100_PTH_STARVE) +{ + uint32_t num_ctxt = 1; + uint32_t num_pth = 100; + uint32_t vlen = KV_64K; + uint32_t LEN = 100; uint32_t secs = 3; TESTCASE_SKIP_IF_FILE; Sync_pth sync_pth; - sync_pth.run_multi_ctxt(num_ctxt, num_pth, vlen, LEN, secs); + sync_pth.run_multi_ctxt(num_ctxt, num_pth, 50, 256, vlen, LEN, secs); } /** ******************************************************************************* * \brief ******************************************************************************/ -TEST(FVT_KV_GOOD_PATH, SYNC_4_PTH_40_CONTEXT) +TEST(FVT_KV_GOOD_PATH, SYNC_1_PTH_2_CONTEXT) { - uint32_t num_ctxt = 40; - uint32_t num_pth = 4; - uint32_t vlen = 128; + uint32_t num_ctxt = 2; + uint32_t num_pth = 1; + uint32_t vlen = KV_4K; uint32_t LEN = 200; uint32_t secs = 3; @@ -148,22 +165,18 @@ TEST(FVT_KV_GOOD_PATH, SYNC_4_PTH_40_CONTEXT) ******************************************************************************* * \brief ******************************************************************************/ -#ifdef _AIX -TEST(FVT_KV_GOOD_PATH, DISABLED_SYNC_4_PTH_200_CONTEXT) -#else -TEST(FVT_KV_GOOD_PATH, SYNC_4_PTH_200_CONTEXT) -#endif +TEST(FVT_KV_GOOD_PATH, SYNC_4_PTH_40_CONTEXT) { - uint32_t num_ctxt = 200; + uint32_t num_ctxt = 40; uint32_t num_pth = 4; - uint32_t npool = 1; uint32_t vlen = 128; - uint32_t LEN = 100; + uint32_t LEN = 200; uint32_t secs = 3; TESTCASE_SKIP_IF_FILE; + TESTCASE_SKIP_IF_RAID0; Sync_pth sync_pth; - sync_pth.run_multi_ctxt(num_ctxt, num_pth, npool, vlen, LEN, secs); + sync_pth.run_multi_ctxt(num_ctxt, num_pth, vlen, LEN, secs); } diff --git a/src/kv/test/fvt_kv_utils.C b/src/kv/test/fvt_kv_utils.C index bf354fe0..5f378762 100644 --- a/src/kv/test/fvt_kv_utils.C +++ b/src/kv/test/fvt_kv_utils.C @@ -43,6 +43,7 @@ extern "C" ******************************************************************************/ void fvt_kv_utils_load(ARK *ark, kv_t *db, uint32_t LEN) { + int rc = 0; uint32_t i = 0; int64_t res = 0; @@ -51,12 +52,13 @@ void fvt_kv_utils_load(ARK *ark, kv_t *db, uint32_t LEN) for (i=0; ib_mark != B_MARK) { - KV_TRC_FFDC(pFT, "FFDC: B_MARK FAILURE %p: %"PRIx64"", pCB, pCB->b_mark); + KV_TRC_FFDC(pFT, "FFDC: B_MARK FAILURE %p: %lx", pCB, pCB->b_mark); return; } if (pCB->e_mark != E_MARK) { - KV_TRC_FFDC(pFT, "FFDC: E_MARK FAILURE %p: %"PRIx64"", pCB, pCB->e_mark); + KV_TRC_FFDC(pFT, "FFDC: E_MARK FAILURE %p: %lx", pCB, pCB->e_mark); return; } if (EBUSY == errcode) {kv_async_q_retry(pCB); goto done;} @@ -312,7 +312,7 @@ static void kv_async_SET_KEY(async_CB_t *pCB) uint64_t tag = (uint64_t)pCB; int32_t rc = 0; - KV_TRC_IO(pFT, "SET_KEY: %p, %p %"PRIx64" %d", pCB, pCB->db, tag, pCB->len_i); + KV_TRC_IO(pFT, "SET_KEY: %p, %p %lx %d", pCB, pCB->db, tag, pCB->len_i); pCB->tag = tag; @@ -513,6 +513,38 @@ void kv_async_init_ctxt(uint32_t ctxt, uint32_t secs) KV_TRC(pFT, "init_ctxt ctxt:%d ark:%p secs:%d", ctxt, pCT->ark, pCT->secs); } +/** + ******************************************************************************* + * \brief + ******************************************************************************/ +void kv_async_init_ctxt_starve(uint32_t ctxt, + uint32_t nasync, + uint32_t basync, + uint32_t secs) +{ + char *env_FVT = getenv("FVT_DEV"); + async_context_t *pCT = pCTs+ctxt; + + ASSERT_TRUE(ctxt >= 0 && ctxt <= KV_ASYNC_MAX_CONTEXTS); + memset(pCT, 0, sizeof(async_context_t)); + memset(pCT->pCBs, 0, sizeof(async_CB_t)*KV_ASYNC_JOB_Q); + + ASSERT_EQ(0, ark_create_verbose(env_FVT, &pCT->ark, + 1048576, + 4096, + 1048576, + 1, + nasync, + basync, + ARK_KV_VIRTUAL_LUN)); + ASSERT_TRUE(NULL != pCT->ark); + + pCT->flags |= KV_ASYNC_CT_RUNNING; + pCT->secs = secs; + KV_TRC(pFT, "init_ctxt_starve ctxt:%d ark:%p secs:%d", + ctxt, pCT->ark, pCT->secs); +} + /** ******************************************************************************* * \brief @@ -611,7 +643,7 @@ void kv_async_job_perf(uint32_t jobs, uint32_t klen, uint32_t vlen,uint32_t len) /* do writes */ (void)ark_stats(kv_async_get_ark(ASYNC_SINGLE_CONTEXT), &ops, &ios); - KV_TRC(pFT, "PERF wr: ops:%"PRIu64" ios:%"PRIu64"", ops, ios); + KV_TRC(pFT, "PERF wr: ops:%ld ios:%ld", ops, ios); gettimeofday(&start, NULL); kv_async_run_jobs(); /* run write jobs */ KV_TRC(pFT, "writes done"); @@ -619,7 +651,7 @@ void kv_async_job_perf(uint32_t jobs, uint32_t klen, uint32_t vlen,uint32_t len) wr_us += (stop.tv_sec*mil + stop.tv_usec) - (start.tv_sec*mil + start.tv_usec); (void)ark_stats(kv_async_get_ark(ASYNC_SINGLE_CONTEXT),&post_ops,&post_ios); - KV_TRC(pFT, "PERF wr: ops:%"PRIu64" ios:%"PRIu64"", post_ops, post_ios); + KV_TRC(pFT, "PERF wr: ops:%ld ios:%ld", post_ops, post_ios); wr_ops += post_ops - ops; wr_ios += post_ios - ios; @@ -643,7 +675,7 @@ void kv_async_job_perf(uint32_t jobs, uint32_t klen, uint32_t vlen,uint32_t len) pCTs->flags |= KV_ASYNC_CT_RUNNING; (void)ark_stats(kv_async_get_ark(0), &ops, &ios); - KV_TRC(pFT, "PERF rd: ops:%"PRIu64" ios:%"PRIu64"", ops, ios); + KV_TRC(pFT, "PERF rd: ops:%ld ios:%ld", ops, ios); gettimeofday(&start, NULL); kv_async_run_jobs(); /* run read jobs */ gettimeofday(&stop, NULL); @@ -651,7 +683,7 @@ void kv_async_job_perf(uint32_t jobs, uint32_t klen, uint32_t vlen,uint32_t len) rd_us += (stop.tv_sec*mil + stop.tv_usec) - (start.tv_sec*mil + start.tv_usec); (void)ark_stats(kv_async_get_ark(0), &post_ops, &post_ios); - KV_TRC(pFT, "PERF rd: ops:%"PRIu64" ios:%"PRIu64"", post_ops, post_ios); + KV_TRC(pFT, "PERF rd: ops:%ld ios:%ld", post_ops, post_ios); rd_ops += post_ops - ops; rd_ios += post_ios - ios; @@ -1167,7 +1199,8 @@ void kv_async_run_jobs(void) (void)ark_stats(pCTs[i].ark, &ops, &ios); tops += (uint32_t)ops; tios += (uint32_t)ios; - KV_TRC(pFT, "PERF ark%p ops:%"PRIu64" ios:%"PRIu64"", pCTs[i].ark, ops, ios); + KV_TRC(pFT, "PERF ark%p ops:%ld ios:%ld", + pCTs[i].ark, ops, ios); EXPECT_EQ(0, ark_delete(pCTs[i].ark)); } diff --git a/src/kv/test/fvt_kv_utils_async_cb.h b/src/kv/test/fvt_kv_utils_async_cb.h index 382d19ad..bd8649a7 100644 --- a/src/kv/test/fvt_kv_utils_async_cb.h +++ b/src/kv/test/fvt_kv_utils_async_cb.h @@ -43,6 +43,15 @@ ******************************************************************************/ void kv_async_init_ctxt(uint32_t ctxt, uint32_t secs); +/** + ******************************************************************************* + * \brief + ******************************************************************************/ +void kv_async_init_ctxt_starve(uint32_t ctxt, + uint32_t nasync, + uint32_t basync, + uint32_t secs); + /** ******************************************************************************* * \brief diff --git a/src/kv/test/fvt_kv_utils_sync_pth.C b/src/kv/test/fvt_kv_utils_sync_pth.C index 3ed13dd3..757506df 100644 --- a/src/kv/test/fvt_kv_utils_sync_pth.C +++ b/src/kv/test/fvt_kv_utils_sync_pth.C @@ -195,7 +195,25 @@ void Sync_pth::run_multi_ctxt(uint32_t num_ctxt, uint32_t ops = 0; uint32_t ios = 0; - run_multi_ctxt(num_ctxt, num_pth, npool, vlen, LEN, secs, &ops, &ios); + run_multi_ctxt(num_ctxt, num_pth, vlen, LEN, secs, &ops, &ios); +} + +/** + ******************************************************************************* + * \brief + ******************************************************************************/ +void Sync_pth::run_multi_ctxt(uint32_t num_ctxt, + uint32_t num_pth, + uint32_t nasync, + uint32_t basync, + uint32_t vlen, + uint32_t LEN, + uint32_t secs) +{ + uint32_t ops = 0; + uint32_t ios = 0; + + run_multi_ctxt(num_ctxt,num_pth,1,nasync,basync,vlen,LEN, secs, &ops, &ios); } /** @@ -210,7 +228,7 @@ void Sync_pth::run_multi_ctxt(uint32_t num_ctxt, uint32_t *p_ops, uint32_t *p_ios) { - run_multi_ctxt(num_ctxt, num_pth, 1, vlen, LEN, secs, p_ops, p_ios); + run_multi_ctxt(num_ctxt, num_pth, 1, 512, KV_8K, vlen,LEN,secs,p_ops,p_ios); } /** @@ -220,6 +238,8 @@ void Sync_pth::run_multi_ctxt(uint32_t num_ctxt, void Sync_pth::run_multi_ctxt(uint32_t num_ctxt, uint32_t num_pth, uint32_t npool, + uint32_t nasync, + uint32_t basync, uint32_t vlen, uint32_t LEN, uint32_t secs, @@ -259,8 +279,8 @@ void Sync_pth::run_multi_ctxt(uint32_t num_ctxt, 4096, 1048576, npool, - 256, - 8*1024, + nasync, + basync, ARK_KV_VIRTUAL_LUN)); ASSERT_TRUE(NULL != ark[ctxt_i]); } diff --git a/src/kv/test/fvt_kv_utils_sync_pth.h b/src/kv/test/fvt_kv_utils_sync_pth.h index 4457584b..a2cca647 100644 --- a/src/kv/test/fvt_kv_utils_sync_pth.h +++ b/src/kv/test/fvt_kv_utils_sync_pth.h @@ -37,9 +37,10 @@ extern "C" { #include #include +#include } -#define MAX_PTH_PER_CONTEXT 256 +#define MAX_PTH_PER_CONTEXT 2048 /** ******************************************************************************* @@ -72,12 +73,18 @@ class Sync_pth kv_t *db = sg_args->db; int64_t res = 0; int32_t i = 0; + int rc = 0; /* load all key/value pairs from the fixed db into the ark */ for (i=0; iLEN; i++) { - EXPECT_EQ(0, ark_set(ark, db[i].klen, db[i].key, db[i].vlen, - db[i].value, &res)); + while (EAGAIN == (rc=ark_set(ark, + db[i].klen, + db[i].key, + db[i].vlen, + db[i].value, + &res))) {usleep(10000);} + EXPECT_EQ(0, rc); EXPECT_EQ(db[i].vlen, res); } } @@ -89,6 +96,7 @@ class Sync_pth kv_t *db = sg_args->db; int64_t res = 0; int32_t i = 0; + int rc = 0; uint8_t *gvalue = NULL; gvalue = (uint8_t*)malloc(sg_args->vlen); @@ -97,11 +105,21 @@ class Sync_pth /* query all key/value pairs from the fixed db */ for (i=sg_args->LEN-1; i>=0; i--) { - EXPECT_EQ(0, ark_get(ark, db[i].klen, db[i].key, db[i].vlen, - gvalue, 0, &res)); + while (EAGAIN == (rc=ark_get(ark, + db[i].klen, + db[i].key, + db[i].vlen, + gvalue, + 0, + &res))) {usleep(10000);} + EXPECT_EQ(0, rc); EXPECT_EQ(db[i].vlen, res); EXPECT_EQ(0, memcmp(db[i].value,gvalue,db[i].vlen)); - EXPECT_EQ(0, ark_exists(ark, db[i].klen, db[i].key, &res)); + while (EAGAIN == (rc=ark_exists(ark, + db[i].klen, + db[i].key, + &res))) {usleep(10000);} + EXPECT_EQ(0, rc); EXPECT_EQ(db[i].vlen, res); } free(gvalue); @@ -114,11 +132,16 @@ class Sync_pth kv_t *db = sg_args->db; int64_t res = 0; int32_t i = 0; + int rc = 0; /* delete all key/value pairs from the fixed db */ for (i=0; iLEN; i++) { - EXPECT_EQ(0, ark_del(ark, db[i].klen, db[i].key, &res)); + while (EAGAIN == (rc=ark_del(ark, + db[i].klen, + db[i].key, + &res))) {usleep(10000);} + EXPECT_EQ(0, rc); EXPECT_EQ(db[i].vlen, res); } } @@ -255,6 +278,16 @@ class Sync_pth void run_multi_ctxt(uint32_t num_ctxt, uint32_t num_pth, + uint32_t vlen, + uint32_t nasync, + uint32_t basync, + uint32_t LEN, + uint32_t secs); + + void run_multi_ctxt(uint32_t num_ctxt, + uint32_t num_pth, + uint32_t nasync, + uint32_t basync, uint32_t npool, uint32_t vlen, uint32_t LEN, diff --git a/src/kv/test/makefile b/src/kv/test/makefile index fea97e05..f5f25023 100644 --- a/src/kv/test/makefile +++ b/src/kv/test/makefile @@ -119,16 +119,16 @@ run_sync_async_OFILES = fvt_kv_tst_sync_async.o kv_utils_db.o \ run_err_OFILES = fvt_kv_tst_errors.o fvt_kv_utils.o kv_utils_db.o \ fvt_kv_utils_async_cb.o fvt_kv_utils_ark_io.o fvt_trace.o +include ${ROOTPATH}/config.mk + CFLAGS += \ -D__FVT__ -D__STDC_FORMAT_MACROS\ -I$(ROOTPATH)/src/kv \ - -I$(ROOTPATH)/src/test/framework/googletest/googletest/include \ + -I$(ROOTPATH)/${GTESTINC} \ $(KV_CFLAGS) CXXFLAGS+=$(CFLAGS) -include ${ROOTPATH}/config.mk - include $(ROOTPATH)/src/test/framework/gtest.objtests.mk ifeq ($(UNAME),AIX) $(GTESTS_DIR) $(GTESTS_NM_DIR) $(BIN_TESTS): $(IMGDIR)/libarkdb.a diff --git a/src/kv/test/run_kv_async.c b/src/kv/test/run_kv_async.c index 1331c974..54a94c4b 100644 --- a/src/kv/test/run_kv_async.c +++ b/src/kv/test/run_kv_async.c @@ -60,9 +60,9 @@ #include #include -uint32_t KV_ASYNC_CONTEXTS = 4; +uint32_t KV_ASYNC_CONTEXTS = 2; uint32_t KV_ASYNC_COMP_PTH = 2; -#define KV_ASYNC_JOBS_PER_CTXT 100 +#define KV_ASYNC_JOBS_PER_CTXT 40 #define KV_ASYNC_MIN_SECS 15 #define KV_ASYNC_KLEN 16 #define KV_ASYNC_VLEN 64*1024 @@ -161,19 +161,6 @@ void kv_async_DEL_KEY (async_CB_t *pCB); void kv_async_wait_jobs (void); void kv_async_completion_pth(pth_t *p); -/** -******************************************************************************** -** \brief -** hw specific sync macros -*******************************************************************************/ -#ifdef _AIX -#define SYNC() asm volatile ("lwsync") -#elif _MACOSX -#define SYNC() asm volatile ("mfence") -#else -#define SYNC() __asm__ __volatile__ ("sync") -#endif - /** ******************************************************************************** ** \brief @@ -253,9 +240,7 @@ void kv_async_cb(int errcode, uint64_t dt, int64_t res) pCB->errcode = errcode; pCB->res = res; - - SYNC(); - pCB->cb_rdy = TRUE; + pCB->cb_rdy = TRUE; } /** @@ -398,7 +383,7 @@ void kv_async_SET_KEY(async_CB_t *pCB) KV_ASYNC_VLEN, pCB->db[pCB->len_i].value, kv_async_cb, - pCB->tag))) usleep(10000); + pCB->tag))) usleep(10); if (rc) KV_ERR_STOP(pCB,"SET_KEY",rc); } @@ -421,7 +406,7 @@ void kv_async_GET_KEY(async_CB_t *pCB) pCB->gvalue, 0, kv_async_cb, - pCB->tag))) usleep(10000); + pCB->tag))) usleep(10); if (rc) KV_ERR_STOP(pCB,"GET_KEY",rc); } @@ -441,7 +426,7 @@ void kv_async_EXISTS_KEY(async_CB_t *pCB) KV_ASYNC_KLEN, pCB->db[pCB->len_i].key, kv_async_cb, - pCB->tag))) usleep(10000); + pCB->tag))) usleep(10); if (rc) KV_ERR_STOP(pCB,"EXIST_KEY",rc); } @@ -461,7 +446,7 @@ void kv_async_DEL_KEY(async_CB_t *pCB) KV_ASYNC_KLEN, pCB->db[pCB->len_i].key, kv_async_cb, - pCB->tag))) usleep(10000); + pCB->tag))) usleep(10); if (rc) KV_ERR_STOP(pCB,"DEL_KEY",rc); } diff --git a/src/kv/test/run_kv_benchmark.c b/src/kv/test/run_kv_benchmark.c index 9988db72..4cc52701 100644 --- a/src/kv/test/run_kv_benchmark.c +++ b/src/kv/test/run_kv_benchmark.c @@ -27,16 +27,19 @@ ******************************************************************************** ** \file ** \brief -** run iops benchmark for Ark synchronous IO +** run iops benchmark for the Ark Key/Value Database Sync/Async IO ** \details -** Usage: run_kv_benchmark [dev] [-p] \n -** [dev] device name to run to \n -** [-p] run to a physical lun \n +** Usage: \n +** [-d device] [-r %rd] [-q qdepth] [-n npool] [-s secs] [-p] \n ** \n -** example: run_kv_benchmark \n -** run_kv_benchmark /tmp/kvstore *kvstore must be at least 1gb \n -** run_kv_benchmark /dev/sg9 \n -** run_kv_benchmark /dev/sg9 -p \n +** \n +** examples: run_kv_benchmark \n +** run_kv_benchmark -d /tmp/kvstore *kvstore must be >= 1gb \n +** run_kv_benchmark -d /dev/sg9 \n +** run_kv_benchmark -d /dev/sg9 -q 1 -r 75 \n +** run_kv_benchmark -d /dev/sg9 -n 4 \n +** run_kv_benchmark -d /dev/sg9 -p \n +** run_kv_benchmark -d /dev/sg9 -s 120 \n ** *******************************************************************************/ #include @@ -51,15 +54,17 @@ #define KV_PTHS 128 #define KV_JOBS 128 -#define KV_NASYNC 130 +#define KV_NASYNC 260 #define KV_BASYNC 8*1024 -#define KV_NPOOL 1 -#define KV_MIN_SECS 12 +int KV_NPOOL = 1; +int KV_MIN_SECS = 12; +#define _4K (4*1024) +#define _64K (64*1024) #define KLEN 16 #define VLEN_4k 4000 #define VLEN_64k 64000 -#define LEN 100 // make this 50 to view even % r/w +int LEN = 100; // make this 50 to view even % r/w #define MAX_IO_ARKS 10 #define MAX_IO_PTHS 40 @@ -71,8 +76,6 @@ #define KV_ASYNC_EXISTS 0x00400000 #define KV_ASYNC_DEL 0x00200000 -#define KV_ASYNC_MAX_JOBS_PER_CTXT 128 - #define TRUE 1 #define FALSE 0 @@ -114,7 +117,7 @@ typedef struct uint32_t flags; uint64_t itag; uint64_t tag; - char gvalue[VLEN_64k]; + char *gvalue; } async_CB_t; /** @@ -126,7 +129,7 @@ typedef struct { ARK *ark; uint32_t flags; - async_CB_t pCBs[KV_ASYNC_MAX_JOBS_PER_CTXT]; + async_CB_t pCBs[KV_NASYNC]; } async_context_t; /** @@ -156,14 +159,15 @@ typedef struct *******************************************************************************/ db_t *dbs; +uint32_t read100_count = 0; uint32_t read100 = 0; uint32_t testcase_start = 0; uint32_t testcase_stop = 0; +uint32_t error_stop = 0; uint64_t ark_create_flag = ARK_KV_VIRTUAL_LUN; void kv_async_SET_KEY (async_CB_t *pCB); void kv_async_GET_KEY (async_CB_t *pCB); -void kv_async_EXISTS_KEY(async_CB_t *pCB); void kv_async_DEL_KEY (async_CB_t *pCB); /** @@ -194,13 +198,14 @@ void kv_async_DEL_KEY (async_CB_t *pCB); ** \brief ** print error msg, clear RUNNING flag to shutdown job, and exit the function *******************************************************************************/ -#define KV_ERR_STOP(_pCB, _msg, _rc) \ -do \ -{ \ - printf("(%s:%d)", _msg,_rc); \ - if (NULL == _pCB) return; \ - _pCB->flags &= ~KV_ASYNC_RUNNING; \ - return; \ +#define KV_ERR_STOP(_pCB, _msg, _rc) \ +do \ +{ \ + printf("(%s:%d)", _msg,_rc); \ + if (NULL == _pCB) {error_stop=1; return;} \ + if (_pCB->flags & KV_ASYNC_RUNNING) {_pCB->flags &= ~KV_ASYNC_RUNNING;} \ + else {error_stop=1;} \ + return; \ } while (0) /** @@ -212,6 +217,7 @@ do \ do \ { \ printf("\n(%s:%d)\n", _msg,_rc); \ + error_stop=1; \ return _rc; \ } while (0) @@ -231,6 +237,24 @@ do \ ark_create_flag)); \ assert(NULL != ark); \ +/** +******************************************************************************** +** \brief print the usage +** \details +** -d device *the device name to run KV IO \n +** -r %rd *the percentage of reads to issue (75 or 100) \n +** -q queuedepth *the number of outstanding ops to maintain \n +** -n npool *the number of threads in the Ark pool \n +** -s secs *the number of seconds to run the I/O \n +** -p *run in physical lun mode \n +*******************************************************************************/ +void usage(void) +{ + printf("Usage:\n"); + printf(" [-d device] [-r %%rd] [-q qdepth] [-n npool] [-s secs] [-p]\n"); + exit(0); +} + /** ******************************************************************************** ** \brief @@ -305,7 +329,7 @@ void free_kv_db(uint32_t n_db, uint32_t len) /** ******************************************************************************** ** \brief -** callback for ark functions: set,get,exists,del +** callback for ark functions: set,get,del *******************************************************************************/ void kv_async_cb(int errcode, uint64_t dt, int64_t res) { @@ -332,7 +356,16 @@ void kv_async_cb(int errcode, uint64_t dt, int64_t res) pCB->len_i = 0; pCB->flags &= ~KV_ASYNC_SET; pCB->flags |= KV_ASYNC_GET; - kv_async_GET_KEY(pCB); + if (read100_count) + { + pCB->flags &= ~KV_ASYNC_RUNNING; + --read100_count; + return; + } + else + { + kv_async_GET_KEY(pCB); + } goto done; } kv_async_SET_KEY(pCB); @@ -356,28 +389,14 @@ void kv_async_cb(int errcode, uint64_t dt, int64_t res) else { pCB->flags &= ~KV_ASYNC_GET; - pCB->flags |= KV_ASYNC_EXISTS; - kv_async_EXISTS_KEY(pCB); + pCB->flags |= KV_ASYNC_DEL; + kv_async_DEL_KEY(pCB); } goto done; } kv_async_GET_KEY(pCB); goto done; } - else if (pCB->flags & KV_ASYNC_EXISTS) - { - /* if end of db len sequence, move to next step */ - if (pCB->len_i == pCB->len) - { - pCB->len_i = 0; - pCB->flags &= ~KV_ASYNC_EXISTS; - pCB->flags |= KV_ASYNC_DEL; - kv_async_DEL_KEY(pCB); - goto done; - } - kv_async_EXISTS_KEY(pCB); - goto done; - } else if (pCB->flags & KV_ASYNC_DEL) { /* end of db len sequence, move to next step */ @@ -448,24 +467,6 @@ void kv_async_GET_KEY(async_CB_t *pCB) if (rc) KV_ERR_STOP(pCB,"GET_KEY",rc); } -/** -******************************************************************************** -** \brief -** save a key/value into the ark db -*******************************************************************************/ -void kv_async_EXISTS_KEY(async_CB_t *pCB) -{ - uint32_t rc=0; - pCB->tag = pCB->itag + pCB->len_i; - - while (EAGAIN == (rc=ark_exists_async_cb(pCB->ark, - pCB->db->klen, - pCB->db->kv[pCB->len_i].key, - kv_async_cb, - pCB->tag))) usleep(10000); - if (rc) KV_ERR_STOP(pCB,"EXIST_KEY",rc); -} - /** ******************************************************************************** ** \brief @@ -490,26 +491,29 @@ void kv_async_DEL_KEY(async_CB_t *pCB) ** do an ark asynchronous IO performance run *******************************************************************************/ void kv_async_io(char *dev, - uint32_t ctxts, - uint32_t jobs, - uint32_t vlen, - uint32_t len, - uint32_t read) + uint32_t ctxts, + uint32_t jobs, + uint32_t vlen, + uint32_t len, + uint32_t read) { async_context_t *pCT = NULL; async_CB_t *pCB = NULL; uint32_t job = 0; uint32_t ctxt = 0; - uint32_t ctxt_running = 0; + uint32_t ctxt_running = TRUE; uint32_t i = 0; uint32_t e_secs = 0; uint64_t ops = 0; uint64_t ios = 0; uint32_t tops = 0; uint32_t tios = 0; + uint32_t wr_ops = 0; + uint32_t wr_ios = 0; + uint32_t time_started = 0; - if (read == 75) read100 = FALSE; - else read100 = TRUE; + if (read == 75) {read100 = FALSE;} + else {read100 = TRUE; read100_count=ctxts*jobs;} printf("ASYNC: ctxt:%-2d QD:%-3d size:%2dk read:%3d%% ", ctxts, jobs, vlen/1000, read); fflush(stdout); @@ -535,10 +539,13 @@ void kv_async_io(char *dev, pCB->flags = KV_ASYNC_SET; pCB->db = dbs+job; pCB->len = len; + if (0==posix_memalign((void**)&pCB->gvalue, 128, vlen)) + {assert(pCB->gvalue);} } } testcase_start = time(0); + if (!read100) {time_started=TRUE;} /* start all jobs for all contexts */ for (ctxt=0; ctxtflags |= KV_ASYNC_RUNNING; + kv_async_GET_KEY(pCB); + } + } + } + else if (read100 && !time_started) + { + usleep(50000); + continue; + } + ctxt_running = FALSE; /* loop through contexts(arks) and check if all jobs are done or @@ -584,8 +618,9 @@ void kv_async_io(char *dev, } } } + usleep(50000); } - while (ctxt_running); + while (!error_stop && ctxt_running); testcase_stop = time(0); e_secs = testcase_stop - testcase_start; @@ -598,6 +633,7 @@ void kv_async_io(char *dev, tios += (uint32_t)ios; } + if (read100) {tops-=wr_ops; tios-=wr_ios;} tops = tops / e_secs; tios = tios / e_secs; printf("op/s:%-7d io/s:%-7d secs:%d\n", tops, tios, e_secs); @@ -605,6 +641,7 @@ void kv_async_io(char *dev, /* sum perf ops for all contexts/jobs and delete arks */ for (i=0; ivlen); - assert(gvalue); + assert(buf); for (i=0; ilen; i++) { @@ -657,42 +692,11 @@ uint32_t kv_query(ARK *ark, db_t *db) db->klen, db->kv[i].key, db->vlen, - gvalue, + buf, 0, &res))) usleep(10000); if (rc) KV_ERR("get1", rc); if (db->vlen != res) KV_ERR("get2",0); - - while (EAGAIN == (rc=ark_exists(ark, - db->klen, - db->kv[i].key, - &res))) usleep(10000); - if (rc) KV_ERR("exists1",rc); - if (db->vlen != res) KV_ERR("exists2",0); - } - free(gvalue); - return 0; -} - -/** -******************************************************************************** -** \brief -** validate that a k/v database exists in the ark -*******************************************************************************/ -uint32_t kv_exists(ARK *ark, db_t *db) -{ - uint32_t i = 0; - uint32_t rc = 0; - int64_t res = 0; - - for (i=0; ilen; i++) - { - while (EAGAIN == (rc=ark_exists(ark, - db->klen, - db->kv[i].key, - &res))) usleep(10000); - if (rc) KV_ERR("exists1",rc); - if (db->vlen != res) KV_ERR("exists2",0); } return 0; } @@ -728,18 +732,19 @@ uint32_t kv_del(ARK *ark, db_t *db) void read_100_percent(worker_t *w) { int start=0,cur=0,rc=0; + char *buf=NULL; - kv_load(w->ark, w->db); + if (0==posix_memalign((void**)&buf, _4K, _64K)) {assert(buf);} start = time(0); do { - if ((rc=kv_query(w->ark, w->db))) break; + if ((rc=kv_query(w->ark, w->db, buf))) break; cur = time(0); } while (cur-start < KV_MIN_SECS); - kv_del(w->ark, w->db); + free(buf); } /** @@ -750,37 +755,21 @@ void read_100_percent(worker_t *w) void read_75_percent(worker_t *w) { int start=0,cur=0,rc=0; + char *buf=NULL; - start = time(0); - do - { - if ((rc=kv_load (w->ark, w->db))) break; - if ((rc=kv_query(w->ark, w->db))) break; - if ((rc=kv_del (w->ark, w->db))) break; - cur = time(0); - } - while (cur-start < KV_MIN_SECS); -} - -/** -******************************************************************************** - -** \brief -** write and read k/v in a database from the ark to equal 50% reads 50% writes -*******************************************************************************/ -void do_50_percent_read(worker_t *w) -{ - int start=0,cur=0,rc=0; + if (0==posix_memalign((void**)&buf, _4K, _64K)) {assert(buf);} start = time(0); do { - if ((rc=kv_load (w->ark, w->db))) break; - if ((rc=kv_exists(w->ark, w->db))) break; - if ((rc=kv_del (w->ark, w->db))) break; + if ((rc=kv_load (w->ark, w->db))) break; + if ((rc=kv_query(w->ark, w->db, buf))) break; + if ((rc=kv_query(w->ark, w->db, buf))) break; + if ((rc=kv_del (w->ark, w->db))) break; cur = time(0); } while (cur-start < KV_MIN_SECS); + free(buf); } /** @@ -819,6 +808,9 @@ void kv_sync_io(char *dev, char *fp_txt = NULL; void* (*fp)(void*); + init_kv_db(pths, KLEN, vlen, len); + NEW_ARK(dev, &ark); + if (read == 75) { fp = (void*(*)(void*))read_75_percent; @@ -828,9 +820,9 @@ void kv_sync_io(char *dev, { fp = (void*(*)(void*))read_100_percent; fp_txt="read:100%"; + for (i=0; i KV_NASYNC) QD = KV_NASYNC; + if (nRD >= 100) nRD = 100; + else nRD = 75; - if (argv[1] != NULL && - argv[1][0] == '/' && - strncmp("/dev/", argv[1], 5) != 0) + if (dev !=NULL && strncmp("/dev/", dev, 5) != 0) { run_to_file = TRUE; } - if (argv[1] == NULL) + if (dev == NULL || (dev && strlen(dev)==0)) { - printf("MEMORY: SYNC: npool:%d nasync:%d basync:%d\n", + printf("MEMORY: npool:%d nasync:%d basync:%d\n", KV_NPOOL, KV_NASYNC, KV_BASYNC); } - else if (argv[2]!=NULL && 0 == strncmp(argv[2], "-p", 2)) + else if (plun) { - printf("PHYSICAL: %s: SYNC: npool:%d nasync:%d basync:%d\n", - argv[i], KV_NPOOL, KV_NASYNC, KV_BASYNC); + printf("PHYSICAL: %s: npool:%d nasync:%d basync:%d\n", + dev, KV_NPOOL, KV_NASYNC, KV_BASYNC); ark_create_flag = 0; } + else if (QD) + { + if (sync) + { + printf("VIRTUAL SYNC: %s: QD:%d npool:%d nasync:%d basync:%d ", + dev, QD, KV_NPOOL, KV_NASYNC, KV_BASYNC); + } + else + { + printf("VIRTUAL ASYNC: %s: QD:%d npool:%d nasync:%d basync:%d ", + dev, QD, KV_NPOOL, KV_NASYNC, KV_BASYNC); + } + } else { - printf("VIRTUAL: %s: SYNC: npool:%d nasync:%d basync:%d\n", - argv[i], KV_NPOOL, KV_NASYNC, KV_BASYNC); + printf("VIRTUAL: %s: npool:%d nasync:%d basync:%d\n", + dev, KV_NPOOL, KV_NASYNC, KV_BASYNC); } fflush(stdout); - kv_async_io(argv[i], 1, 1, VLEN_4k, LEN, 100); - kv_async_io(argv[i], 1, KV_JOBS, VLEN_4k, LEN, 75); - kv_async_io(argv[i], 1, KV_JOBS, VLEN_4k, LEN, 100); - kv_async_io(argv[i], 1, KV_JOBS, VLEN_64k, LEN, 100); + if (QD) + { + if (sync) {kv_sync_io(dev, VLEN_4k, LEN, QD, nRD);} + else {kv_async_io(dev, 1, QD, VLEN_4k, LEN, nRD);} + exit(0); + } + + kv_async_io(dev, 1, 1, VLEN_4k, LEN, 100); + kv_async_io(dev, 1, 64, VLEN_4k, LEN, 100); + kv_async_io(dev, 1, KV_JOBS, VLEN_4k, LEN, 75); + kv_async_io(dev, 1, KV_JOBS, VLEN_4k, LEN, 100); + kv_async_io(dev, 1, KV_JOBS, VLEN_64k, LEN, 100); if (ark_create_flag && !run_to_file) - kv_async_io(argv[i], 10, 40, VLEN_4k, LEN, 100); + kv_async_io(dev, 10, 40, VLEN_4k, LEN, 100); - kv_sync_io(argv[1], VLEN_4k, LEN, 1, 100); - kv_sync_io(argv[1], VLEN_4k, LEN, KV_PTHS, 75); - kv_sync_io(argv[1], VLEN_4k, LEN, KV_PTHS, 100); - kv_sync_io(argv[1], VLEN_64k, LEN, KV_PTHS, 100); + kv_sync_io(dev, VLEN_4k, LEN, 1, 100); + kv_sync_io(dev, VLEN_4k, LEN, KV_PTHS, 75); + kv_sync_io(dev, VLEN_4k, LEN, KV_PTHS, 100); + kv_sync_io(dev, VLEN_64k, LEN, KV_PTHS, 100); if (ark_create_flag && !run_to_file) - kv_max_sync_io(argv[i], VLEN_4k, LEN); + kv_max_sync_io(dev, VLEN_4k, LEN); - return 0; + exit(0); } diff --git a/src/kv/test/run_kv_persist.c b/src/kv/test/run_kv_persist.c index 6f3ace47..f6f9e795 100644 --- a/src/kv/test/run_kv_persist.c +++ b/src/kv/test/run_kv_persist.c @@ -350,7 +350,12 @@ int main(int argc, char **argv) } if (newark) {printf("load %d k/v\n", KV_WORKERS*LEN);} - else {printf("validate existing %d k/v\n", KV_WORKERS*LEN);} + else if (count == 0) + { + printf("no ark data found, create an ark with \'-new' option\n"); + goto exit; + } + else {printf("validate existing %d k/v\n", KV_WORKERS*LEN);} init_kv_db(MAX_KLEN, MAX_VLEN, LEN); @@ -406,7 +411,8 @@ int main(int argc, char **argv) } assert(0 == ark_count(ark, &count)); printf("persist ark with %ld keys\n", count); - assert(0 == ark_delete(ark)); +exit: + assert(0 == ark_delete(ark)); return 0; } diff --git a/src/kv/ut.c b/src/kv/ut.c index c2f3e780..c2a2d7f8 100644 --- a/src/kv/ut.c +++ b/src/kv/ut.c @@ -25,28 +25,11 @@ #include #include #include - #include - #include #include - #include -#ifndef _AIX -#include -#endif -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "am.h" -#include "ut.h" +#include uint64_t divup(uint64_t n, uint64_t m) { return n/m + (n%m==0 ? 0 : 1); @@ -68,57 +51,3 @@ char *rndalpha(int n, int m) { free(ret); return ret; } - -double time_diff(struct timeval x , struct timeval y) -{ - double x_ms , y_ms , diff; - - x_ms = (double)x.tv_sec*1000000 + (double)x.tv_usec; - y_ms = (double)y.tv_sec*1000000 + (double)y.tv_usec; - - diff = (double)y_ms - (double)x_ms; - - return diff; -} - -// return nanoseconds per tick -double time_per_tick(int n, int del) { - int i; - - double *td = malloc(n * sizeof(double)); - double *tv = malloc(n * sizeof(double)); - - struct timeval tvs; - struct timeval tve; - - ticks ts; - ticks te; - - for (i=0; iwwpn[0],&l_kw_length); @@ -632,11 +635,11 @@ uint8_t provGetAllWWPNs(prov_wwpn_info_t* io_wwpn_info, uint16_t *io_num_wwpns) { bzero(l_afu_path, sizeof(l_afu_path)); l_pci_path = NULL; - TRACEI("AFU found: '%s'\n", cxl_afu_devname(l_afu)); + TRACEI("AFU found: '%s'\n", cxl_afu_dev_name(l_afu)); //TODO: why do i have to build this myself? should we be //always appending the "master" context on the end here? - snprintf(l_afu_path, sizeof(l_afu_path), "/dev/cxl/%sm", cxl_afu_devname(l_afu)); - l_rc = cxl_afu_sysfs_pci(&l_pci_path, l_afu); + snprintf(l_afu_path, sizeof(l_afu_path), "/dev/cxl/%sm", cxl_afu_dev_name(l_afu)); + l_rc = cxl_afu_sysfs_pci(l_afu, &l_pci_path); TRACEI("sysfs rc: %d\n", l_rc); TRACEI("sysfs path: '%s'\n", l_pci_path); TRACEI("afu path: '%s'\n", l_afu_path); diff --git a/src/test/afu.c b/src/test/afu.c index 751adfac..4b1af84c 100644 --- a/src/test/afu.c +++ b/src/test/afu.c @@ -161,7 +161,7 @@ struct afu *afu_map () perror ("malloc"); return NULL; } - debug ("Allocated memory at 0x%016lx for AFU\n", (__u64) afu); + debug ("Allocated memory at 0x%p for AFU\n", afu); memset(afu, 0, sizeof(*afu)); afu->work.num_interrupts = 4; @@ -212,7 +212,7 @@ void afu_unmap (struct afu *afu) void afu_start (struct afu *afu) { /* Set WED in PSL and send start command to AFU */ - debug ("Sending WED address 0x%016lx to PSL...\n", (__u64) afu->work.work_element_descriptor); + debug ("Sending WED address 0x%016llx to PSL...\n", (__u64) afu->work.work_element_descriptor); if (ioctl (afu->fd, CXL_IOCTL_START_WORK, &afu->work) == 0) { debug ("Start command succeeded on AFU\n"); @@ -262,7 +262,7 @@ void afu_mmio_read_dw (struct afu *afu, unsigned offset, __u64 *value) { __u64 addr = 4 * (__u64) (offset & ~0x1); // Force 8byte align *value = in_be64 (afu->ps_addr + addr); - debug ("Read 0x%016lx from AFU register offset %x\n", *value, offset); + debug ("Read 0x%016llx from AFU register offset %x\n", *value, offset); } // Wait for AFU to complete job diff --git a/src/test/asyncstress.c b/src/test/asyncstress.c index 2576ae0e..da927bb5 100644 --- a/src/test/asyncstress.c +++ b/src/test/asyncstress.c @@ -38,7 +38,7 @@ #include #include #include -#include +#include //#define PRI(wh, args...) fprintf(wh, ##args) #define PRI(wh, args...) @@ -49,7 +49,7 @@ char *dev_name = "/dev/cxl/afu0.0s"; /* name of device including dev */ int num_ops = 1024; /* no of operations per thread */ int num_asyncs = 128; /* no of outstanding any given at time */ int num_threads = 1; /* no of threads */ -int num_blocks = 1000000; /* 0..num_blocks-1 lbas to be reads */ +int num_blocks = (260*1024); /* 0..num_blocks-1 lbas to be reads */ int virt_blks = 1; int history = 0; int read_block = 100; @@ -152,10 +152,12 @@ int main(int argc, char **argv) { printf("lsize = %ld, rc, = %d\n", lsize, rc); if (rc) {fprintf(stderr, "cblk_get_lun_size failed, rc=%d\n", errno); exit(errno);} - rc = cblk_set_size(id, num_blocks, 0); - printf("bsize = %d, rc, = %d\n", num_blocks, rc); - if (rc) {fprintf(stderr, "cblk_set_size failed, rc=%d\n", errno); exit(errno);} - + if (virt_blks) + { + rc = cblk_set_size(id, num_blocks, 0); + printf("bsize = %d, rc, = %d\n", num_blocks, rc); + if (rc) {fprintf(stderr, "cblk_set_size failed, rc=%d\n", errno); exit(errno);} + } int *wait_order = malloc(num_asyncs * sizeof(int)); int wait_pos = 0; int wait_next = 0; diff --git a/src/test/asyncstress2.c b/src/test/asyncstress2.c index f42adb29..eefa99b6 100644 --- a/src/test/asyncstress2.c +++ b/src/test/asyncstress2.c @@ -38,7 +38,7 @@ #include #include #include -#include +#include //#define PRI(wh, args...) fprintf(wh, ##args) #define PRI(wh, args...) diff --git a/src/test/blk_test.c b/src/test/blk_test.c index ca59d0a4..f4e7ea07 100644 --- a/src/test/blk_test.c +++ b/src/test/blk_test.c @@ -432,12 +432,8 @@ parse_args(int argc, char **argv) } break; case 'm' : -#ifndef _AIX - fprintf(stderr,"-m flag is not supported on this operating system\n"); - rc = EINVAL; -#else mpio_flag = TRUE; -#endif + break; case 'n' : @@ -781,13 +777,13 @@ void *run_loop(void *data) open_flags |= CBLK_OPN_SHARE_CTXT; } -#ifdef _AIX if (mpio_flag) { open_flags |= CBLK_OPN_MPIO_FO; } +#ifdef _AIX if (reserve_flag) { open_flags |= CBLK_OPN_RESERVE; @@ -1626,27 +1622,36 @@ void *run_loop(void *data) */ + if (virtual_lun_flag) { - if (verbose_flag && !thread_flag) { + /* + * Clone after fork is only supported for + * virtual luns. Physical luns do not need this. + */ - fprintf(stderr,"Calling cblk_clone_after_fork...\n"); - } - rc = cblk_clone_after_fork(blk_data->chunk_id,O_RDONLY,0); + if (verbose_flag && !thread_flag) { + + fprintf(stderr,"Calling cblk_clone_after_fork...\n"); + + } + rc = cblk_clone_after_fork(blk_data->chunk_id,O_RDONLY,0); - if (verbose_flag && !thread_flag) { - printf("clone completed with rc = %d, errno = %d\n",rc,errno); - } + if (verbose_flag && !thread_flag) { + printf("clone completed with rc = %d, errno = %d\n",rc,errno); + } - if (rc) { + if (rc) { - fprintf(stderr,"cblk_chunk_clone failed with rc = 0x%x errno = %d\n",rc, errno); - free(comp_data_buf); + fprintf(stderr,"cblk_chunk_clone failed with rc = 0x%x errno = %d\n",rc, errno); + free(comp_data_buf); - free(data_buf); - return (ret_code); + free(data_buf); + return (ret_code); + } + } @@ -2238,12 +2243,12 @@ int main (int argc, char **argv) flags |= CBLK_OPN_SHARE_CTXT; } -#ifdef _AIX if (mpio_flag) { flags |= CBLK_OPN_MPIO_FO; } +#ifdef _AIX if (reserve_flag) { @@ -2475,46 +2480,55 @@ int main (int argc, char **argv) */ + + if (virtual_lun_flag) { - if (verbose_flag && !thread_flag) { + /* + * Clone after fork is only supported for + * virtual luns. Physical luns do not need this. + */ + + if (verbose_flag && !thread_flag) { - fprintf(stderr,"Calling cblk_clone_after_fork...\n"); + fprintf(stderr,"Calling cblk_clone_after_fork...\n"); - } - rc = cblk_clone_after_fork(chunk_id,O_RDWR,0); + } + rc = cblk_clone_after_fork(chunk_id,O_RDWR,0); - if (verbose_flag && !thread_flag) { - printf("clone completed with rc = %d, errno = %d\n",rc,errno); - } + if (verbose_flag && !thread_flag) { + printf("clone completed with rc = %d, errno = %d\n",rc,errno); + } - if (rc) { + if (rc) { - fprintf(stderr,"cblk_chunk_clone failed with rc = 0x%x errno = %d\n",rc, errno); + fprintf(stderr,"cblk_chunk_clone failed with rc = 0x%x errno = %d\n",rc, errno); - } + } - if ((!rc) && (shared_contxt_flag)) { + if ((!rc) && (shared_contxt_flag)) { - rc2 = cblk_clone_after_fork(alt_chunk_id,O_RDWR,0); + rc2 = cblk_clone_after_fork(alt_chunk_id,O_RDWR,0); - if (verbose_flag && !thread_flag) { - printf("clone completed with rc = %d, errno = %d\n",rc,errno); - } + if (verbose_flag && !thread_flag) { + printf("clone completed with rc = %d, errno = %d\n",rc,errno); + } - if (rc2) { + if (rc2) { - fprintf(stderr,"cblk_chunk_clone failed with rc = 0x%x errno = %d\n",rc, errno); + fprintf(stderr,"cblk_chunk_clone failed with rc = 0x%x errno = %d\n",rc, errno); + } } - } + + } /* if (virtual_lun_flag) */ if ((!rc) && (!rc2) && diff --git a/src/test/ffdc/cxl_afu_dump2.c b/src/test/ffdc/cxl_afu_dump2.c index 0c9a3d0e..c4c63ff1 100644 --- a/src/test/ffdc/cxl_afu_dump2.c +++ b/src/test/ffdc/cxl_afu_dump2.c @@ -22,11 +22,11 @@ /* permissions and limitations under the License. */ /* */ /* IBM_PROLOG_END_TAG */ - #include #include #include -#include "afu.h" +#include +#include #include "afu_fc.h" #ifdef SIM #include "sim_pthread.h" @@ -34,80 +34,168 @@ #include #endif +#define DEBUG 0 +#define debug(fmt, ...) do { if (DEBUG) fprintf(stderr,fmt,__VA_ARGS__); fflush(stderr);} while (0) +#define debug0(fmt) do { if (DEBUG) fprintf(stderr,fmt); fflush(stderr);} while (0) + void print_64(FILE *f, int a, uint64_t v) { if (v != 0xFFFFFFFFFFFFFFFF) fprintf(f,"%x : %lx\n",a,v); } -void dump_range(FILE *f, struct afu *afu, int lwr, int upr) { +void dump_range(FILE *f, struct cxl_afu_h *afu, int lwr, int upr) { uint64_t r; + __u64 addr; + fprintf(f,"# mmio address range 0x%07x to 0x%07x\n",lwr, upr); while (lwr < upr) { // skip heartbeat reg otherwise mserv reports a mismatch if( lwr != 0x2010020 ) { - afu_mmio_read_dw(afu,(lwr >> 2),(__u64 *)&r); + addr = 4 * (__u64) ((lwr >> 2) & ~0x1); // Force 8byte align + cxl_mmio_read64(afu,addr,(__u64 *)&r); print_64(f,lwr,r); } lwr+=8; } } -void dump_count(FILE *f, struct afu *afu, int base, int n) { +void dump_count(FILE *f, struct cxl_afu_h *afu, int base, int n) { dump_range(f,afu,base,base+(n*8)); } -void dump_ctxt(FILE *f, struct afu* afu, int ctxt) { - int i; - uint64_t r; +void dump_ctxt(FILE *f, struct cxl_afu_h* afu, int ctxt) { dump_count(f,afu,0x10000*ctxt,16); } -void dump_cpc(FILE *f, struct afu* afu, int ctxt) { +void dump_cpc(FILE *f, struct cxl_afu_h* afu, int ctxt) { dump_count(f,afu,0x2000000+(16*8*ctxt),16); } -void dump_gbl(FILE *f, struct afu *afu) { +void dump_gbl(FILE *f, struct cxl_afu_h *afu) { dump_range(f,afu,0x2010000,0x2012000); } -void dump_afu_dbg(FILE *f, struct afu *afu){ +void dump_afu_dbg(FILE *f, struct cxl_afu_h *afu){ dump_range(f,afu,0x2040000,0x2060000); } -void dump_fc_dbg(FILE *f, struct afu* afu, int fcp) { +void dump_fc_dbg(FILE *f, struct cxl_afu_h* afu, int fcp) { dump_range(f,afu,0x2060000+(fcp*0x20000),0x2060000+(fcp*0x20000)+0x20000); } -void dump_perf (FILE *f, struct afu * afu) { +void dump_perf (FILE *f, struct cxl_afu_h * afu) { dump_range(f,afu,0x2018000,0x2020000); } -void print_version(FILE *f, struct afu *afu) +void get_str(char *out, __u64 in) +{ + int i; + for (i=0; i<8; i++) { + out[i] = (in & 0xFFl); + in = in >> 8; + } + out[8] = 0; +} + +void print_version(FILE *f, struct cxl_afu_h *afu) { uint64_t r; - afu_mmio_read_dw(afu,0x804200,&r); + __u64 addr; + char version[9]; + + addr = 4 * (__u64) (0x804200 & ~0x1); // Force 8byte align + cxl_mmio_read64(afu,addr,&r); if (r == -1l) { - fprintf(f, "# Version number not implemented\n"); + fprintf(f, "# AFU Version number not implemented\n"); } else { - char version[9]; int i; for (i=0; i<8; i++) { version[7-i] = (r & 0xFFl); r = r >> 8; } version[8] = 0; - fprintf(f,"# AFU Version = %s\n",version); + fprintf(f,"AFU Version = %s\n",version); } -} + fprintf(f,"\n"); + addr = 4 * (__u64) ((0x2060708>>2) & ~0x1); // Force 8byte align + cxl_mmio_read64(afu,addr,&r); + if (r != -1l) { + char version[9]; + get_str(version, r); + fprintf(f," NVMe0 Version = %s\n",version); + } + addr = 4 * (__u64) ((0x2012710>>2) & ~0x1); // Force 8byte align + cxl_mmio_read64(afu,addr,&r); + if (r != -1l) { + get_str(version, r); + fprintf(f," NVMe0 NEXT = %s\n",version); + } + addr = 4 * (__u64) ((0x2012700>>2) & ~0x1); // Force 8byte align + cxl_mmio_read64(afu,addr,&r); + if (r != -1l) {fprintf(f," NVMe0 STATUS = 0x%lx\n", r);} + + fprintf(f,"\n"); + addr = 4 * (__u64) ((0x2080708>>2) & ~0x1); // Force 8byte align + cxl_mmio_read64(afu,addr,&r); + if (r != -1l) { + get_str(version, r); + fprintf(f," NVMe1 Version = %s\n",version); + } + addr = 4 * (__u64) ((0x2013710>>2) & ~0x1); // Force 8byte align + cxl_mmio_read64(afu,addr,&r); + if (r != -1l) { + get_str(version, r); + fprintf(f," NVMe1 NEXT = %s\n",version); + } + addr = 4 * (__u64) ((0x2013700>>2) & ~0x1); // Force 8byte align + cxl_mmio_read64(afu,addr,&r); + if (r != -1l) {fprintf(f," NVMe1 STATUS = 0x%lx\n", r);} +} int main(int argc, char **argv) { + struct cxl_ioctl_start_work *work; + long reported = 0; + if (argc < 2) { printf("usage: %s \n",argv[0]); exit(2); } - struct afu* afu = afu_map(argv[1]); + struct cxl_afu_h *afu = cxl_afu_open_dev(argv[1]); + if (!afu) { + perror ("malloc"); + return 0; + } + debug ("Allocated memory at 0x%016lx for AFU\n", (__u64) afu); + + fprintf(stderr,"Creating and opening AFU file descriptor %s\n",argv[1]); + work = (struct cxl_ioctl_start_work*) cxl_work_alloc(); + if (work == NULL) { + perror("cxl_work_alloc"); + return 1; + } + if (cxl_afu_attach_work(afu, work)) { + debug0 ("Start command to AFU failed\n"); + perror("cxl_afu_attach_work(master)"); + return 1; + } + else { + debug0 ("Start command succeeded on AFU\n"); + } + if (cxl_mmio_map(afu, CXL_MMIO_BIG_ENDIAN) != 0) { + perror ("mmap_problem_state_registers"); + return 0; + } + + cxl_get_mmio_size(afu, &reported); + debug ("mmio_size 0x%lx for AFU\n", reported); + + debug0 ("Installing libcxl SIGBUS handler\n"); + cxl_mmio_install_sigbus_handler(); + FILE *f = stdout; print_version(f,afu); + if (argv[2] && strncmp(argv[2],"-v",2)==0) {return 0;} + int i; for(i=0; i<512; i++) dump_ctxt(f,afu,i); for(i=0; i<512; i++) dump_cpc(f,afu,i); @@ -116,5 +204,6 @@ int main(int argc, char **argv) { dump_afu_dbg(f,afu); dump_fc_dbg(f,afu,0); dump_fc_dbg(f,afu,1); + return 0; } diff --git a/src/test/ffdc/cxl_afu_inject.c b/src/test/ffdc/cxl_afu_inject.c new file mode 100644 index 00000000..aad80456 --- /dev/null +++ b/src/test/ffdc/cxl_afu_inject.c @@ -0,0 +1,311 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/test/ffdc/cxl_afu_inject.c $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#include +#include +#include +#include +#include +#include +#include + +int rc=0; +uint32_t DEBUG=0; +#define debug(fmt, ...) do { if (DEBUG) fprintf(stderr,fmt,__VA_ARGS__); fflush(stderr);} while (0) +#define debug0(fmt) do { if (DEBUG) fprintf(stderr,fmt); fflush(stderr);} while (0) + +#define RESET 0 +#define DOWN 1 +#define READ_LBA 2 +#define WRITE_LBA 3 +#define READ_DMA 4 +#define WRITE_DMA 5 +#define READ_BAD 6 +#define WRITE_BAD 7 +#define CLEAR_PERM 8 +#define PARITY_ON_WR 9 + +struct cxl_afu_h *afu; +uint64_t r,addr; +uint64_t FC_STATUS2 = 0x348; +uint64_t FC_ERRINJ_DATA = 0x3a8; +uint64_t FC_PE_ERRINJ = 0x3B0; +uint64_t FC_PE_ERRINJ_DATA = 0x3B8; +uint64_t gen1 = 0b001000000; +uint64_t gen2 = 0b010000000; +uint64_t gen3 = 0b100000000; +uint64_t compq_empty = 0b10000000; +uint64_t subq_empty = 0b01000000; +uint64_t pcie_init_cmplt = 0b00001000; +uint64_t ioq_rdy = 0b00000100; +uint64_t cntrl_rdy = 0b00000010; +uint64_t link_training_cmplt = 0b00000001; +uint64_t nvme_port_online = 0b00001111; + +/** +******************************************************************************** +** \brief usage +*******************************************************************************/ +void usage(void) +{ + printf("Usage:\n"); + printf(" [-d afuX] [-p port] [-a action] [-l lba] [-P] [-i interval]\n"); + printf(" interval: #cmds before an inject\n"); + printf(" action: 0-PORT_RESET 1-PORT_OFFLINE\n"); + printf(" 2-READ_LBA_ERR 3-WRITE_LBA_ERR\n"); + printf(" 4-READ_DMA_ERR 5-WRITE_DMA_ERR\n"); + printf(" 6-READ_BAD_DATA 7-WRITE_BAD_DATA\n"); + printf(" 8-CLEAR_PERM_RD_WR_ERR\n"); + printf(" 9-PARITY_ERR_ON_WRITE\n"); + exit(0); +} + +/** +******************************************************************************** +** \brief nvme_link +** \details +** reset an NVMe port, or set it offline +*******************************************************************************/ +int nvme_link(uint32_t port, uint32_t state) +{ + uint64_t base = port ? FC_PORT1_OFFSET : FC_PORT0_OFFSET; + + addr = base+FC_CONFIG; + + /* set offline */ + if (!state) + { + cxl_mmio_read64(afu, addr, &r); + debug("rd FC_CONFIG: 0x%016lx\n", r); + r |= 0x10; //set reset + r &= ~0x1; //clear auto-login + debug("wr FC_CONFIG: 0x%016lx\n", r); + cxl_mmio_write64(afu, addr, r); + usleep(50000); + addr=base+FC_STATUS; + cxl_mmio_read64(afu, addr, &r); + debug("rd FC_STATUS: 0x%016lx\n", r); + if ((r&nvme_port_online) == nvme_port_online) + { + printf("NVMe port offline failed\n"); + rc=-5; + } + } + /* reset or set online */ + else + { + cxl_mmio_read64(afu, addr, &r); + debug("rd FC_CONFIG: 0x%016lx\n", r); + r |= 0x11; //set reset and auto-login + debug("wr FC_CONFIG: 0x%016lx\n", r); + cxl_mmio_write64(afu, addr, r); + usleep(50000); + addr=base+FC_STATUS; + cxl_mmio_read64(afu, addr, &r); + debug("rd FC_STATUS: 0x%016lx\n", r); + if (r&nvme_port_online != nvme_port_online) + { + printf("NVMe port online failed\n"); + rc=-6; + } + } +} + +/** +******************************************************************************** +** \brief io_error +** \details +** mmio write two regs for the rd/wr inject +*******************************************************************************/ +void io_error(uint32_t port, + uint32_t type, + uint32_t lba, + uint32_t perm, + uint64_t status, + uint32_t interval) +{ + uint64_t base = port ? FC_PORT1_OFFSET : FC_PORT0_OFFSET; + uint64_t errinj = 0x80000000 | interval; + uint64_t data = (status<<32)|lba; + + if (type == CLEAR_PERM) {errinj=0; data=0;} + else {errinj|=(type<<20);} + + if (perm) {errinj|=0b110000000000000000;} + else {errinj|=0b010000000000000000;} + + debug("port:%d type:%d lba:%.8d perm:%d status:%.2lx interval:%d\n", + port,type,lba,perm,status,interval); + debug("ERRINJ:%016lx : ERRINJ_DATA:%016lx\n", errinj, data); + addr=base+FC_ERRINJ; + cxl_mmio_write64(afu, addr, errinj); + + addr=base+FC_ERRINJ_DATA; + cxl_mmio_write64(afu, addr, data); +} + +/** +******************************************************************************** +** \brief parity_error +** \details +** mmio write two regs for the parity inject +*******************************************************************************/ +void parity_err(uint32_t port, + uint32_t type, + uint32_t lba, + uint32_t perm, + uint64_t status, + uint32_t interval) +{ + uint64_t base = port ? FC_PORT1_OFFSET : FC_PORT0_OFFSET; + uint64_t errinj = 0; + uint64_t data = 0; + + debug("port:%d type:%d lba:%.8d perm:%d status:%.2lx interval:%d\n", + port,type,lba,perm,status,interval); + + addr=base+FC_PE_ERRINJ_DATA; + debug("addr:%016lx : ERRINJ_DATA:%016lx\n", addr, data); + cxl_mmio_write64(afu, addr, data); + + addr=base+FC_PE_ERRINJ; + debug("addr:%016lx : ERRINJ:%016lx\n", addr, errinj); + cxl_mmio_write64(afu, addr, errinj); + + errinj = 1; + data = 0x100400; + + addr=base+FC_PE_ERRINJ_DATA; + debug("addr:%016lx : ERRINJ_DATA:%016lx\n", addr, data); + cxl_mmio_write64(afu, addr, data); + + addr=base+FC_PE_ERRINJ; + debug("addr:%016lx : ERRINJ:%016lx\n", addr, errinj); + cxl_mmio_write64(afu, addr, errinj); +} + +/** +******************************************************************************** +** \brief main +*******************************************************************************/ +int main(int argc, char **argv) +{ + struct cxl_ioctl_start_work *work; + long mmio_size = 0; + char FF = 0xFF; + char c = '\0'; + char dev[128]; + char *_dev = NULL; + char *_port = NULL; + char *_action = NULL; + char *_lba = NULL; + char *_interval = NULL; + char *_status = NULL; + uint32_t port = 0; + uint32_t action = 0; + uint32_t lba = 0; + uint32_t interval = 0; + uint32_t status = 0; + uint32_t perm = 0; + uint32_t verbose = 0; + + /*-------------------------------------------------------------------------- + * process and verify input parms + *------------------------------------------------------------------------*/ + while (FF != (c=getopt(argc, argv, "d:p:a:l:i:s:Pvh"))) + { + switch (c) + { + case 'd': _dev = optarg; break; + case 'p': _port = optarg; break; + case 'a': _action = optarg; break; + case 'l': _lba = optarg; break; + case 'i': _interval = optarg; break; + case 's': _status = optarg; break; + case 'P': perm = 1; break; + case 'v': verbose = 1; break; + case 'h': + case '?': usage(); break; + } + } + if (verbose) {DEBUG=1;} + if (!_dev) {usage();} + if (_port) port = atoi(_port); + if (_action) action = atoi(_action); + if (_lba) lba = atoi(_lba); + if (_interval) interval = atoi(_interval); + if (_status) status = atoi(_status); + + if (strncmp(_dev,"afu",3) == 0) {sprintf(dev, "/dev/cxl/%s",_dev);} + else {sprintf(dev, "%s",_dev);} + afu = cxl_afu_open_dev(dev); + if (!afu) + { + printf("open of %s failed\n", dev); + exit(-1); + } + debug("Allocated memory at 0x%016lx for AFU\n", (__u64) afu); + + debug("Creating and opening %s\n", dev); + work = (struct cxl_ioctl_start_work*) cxl_work_alloc(); + if (work == NULL) + { + perror("cxl_work_alloc"); + exit(-2); + } + if (cxl_afu_attach_work(afu, work)) + { + debug0 ("Start command to AFU failed\n"); + perror("cxl_afu_attach_work(master)"); + exit(-3); + } + + debug0("Start command succeeded on AFU\n"); + + if (cxl_mmio_map(afu, CXL_MMIO_BIG_ENDIAN) != 0) + { + perror ("mmap_problem_state_registers"); + exit(-4); + } + + cxl_get_mmio_size(afu, &mmio_size); + debug("mmio_size 0x%lx for AFU\n", mmio_size); + + debug0("Installing libcxl SIGBUS handler\n"); + cxl_mmio_install_sigbus_handler(); + + if (action==RESET) {nvme_link(port,1);} + else if (action==DOWN) {nvme_link(port,0);} + else if (action==READ_LBA) {io_error(port,2,lba,perm,status,interval);} + else if (action==WRITE_LBA) {io_error(port,1,lba,perm,status,interval);} + else if (action==READ_DMA) {io_error(port,7,lba,perm,status,interval);} + else if (action==WRITE_DMA) {io_error(port,8,lba,perm,status,interval);} + else if (action==READ_BAD) {io_error(port,4,lba,perm,status,interval);} + else if (action==WRITE_BAD) {io_error(port,3,lba,perm,status,interval);} + else if (action==CLEAR_PERM) {io_error(port,9,lba,perm,status,interval);} + else if (action==PARITY_ON_WR){parity_err(port,9,lba,perm,status,interval);} + else {printf("invalid action: %d\n", action);} + + exit(rc); +} diff --git a/src/test/ffdc/cxl_afu_status.c b/src/test/ffdc/cxl_afu_status.c new file mode 100644 index 00000000..1d5ca565 --- /dev/null +++ b/src/test/ffdc/cxl_afu_status.c @@ -0,0 +1,230 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/test/ffdc/cxl_afu_status.c $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#include +#include +#include +#include +#include +#include +#include + +#define DEBUG 0 +#define debug(fmt, ...) do { if (DEBUG) fprintf(stderr,fmt,__VA_ARGS__); fflush(stderr);} while (0) +#define debug0(fmt) do { if (DEBUG) fprintf(stderr,fmt); fflush(stderr);} while (0) + +struct cxl_afu_h *afu; +uint64_t r,addr; +uint64_t FC_STATUS2 = 0x348; +uint64_t FC_ERRINJ_DATA = 0x3a8; +uint64_t gen1 = 0b001000000; +uint64_t gen2 = 0b010000000; +uint64_t gen3 = 0b100000000; +uint64_t compq_empty = 0b10000000; +uint64_t subq_empty = 0b01000000; +uint64_t pcie_init_cmplt = 0b00001000; +uint64_t ioq_rdy = 0b00000100; +uint64_t cntrl_rdy = 0b00000010; +uint64_t link_training_cmplt = 0b00000001; +uint64_t nvme_port_online = 0b00001111; + +/** +******************************************************************************** +** \brief usage +*******************************************************************************/ +void usage(void) +{ + printf("Usage: [-d afuX]\n"); + exit(0); +} + +/** +******************************************************************************** +** \brief show_port +** \details +** +*******************************************************************************/ +void show_port(uint32_t port) +{ + uint64_t base; + uint64_t status2; + uint32_t num; + + if (port) base=FC_PORT1_OFFSET; + else base=FC_PORT0_OFFSET; + + printf(" NVMe%d: ", port); + + addr=base+FC_STATUS2; + cxl_mmio_read64(afu, addr,(__u64 *)&status2); + if (status2&gen1) printf("GEN1 "); + if (status2&gen2) printf("GEN2 "); + if (status2&gen3) printf("GEN3 "); + uint64_t width=((status2>>9) & 0b1111); + printf("width=%ld ", width); + if (status2&0b110000000000000) printf("link up"); + + printf("\n"); + + if (status2&0b10000000000000000)printf(" user reset indication\n"); + if (status2&0b1000000000000000) printf(" core phy link down\n"); + //if (status2&0b110000000000000) printf(""); + else if (status2&0b100000000000000) printf(" link up, init in process\n"); + else if (status2&0b0100000000000000) printf(" link train in progress\n"); + else printf(" no receivers\n"); + + addr=base+FC_MTIP_STATUS; + cxl_mmio_read64(afu, addr,(__u64 *)&r); + if (r&0b10000) printf(" offline\n"); + + addr=base+FC_CONFIG; + cxl_mmio_read64(afu, addr,(__u64 *)&r); + if (r&0b10000000) printf(" timeouts disabled\n"); + if (r&0b10000) printf(" fc_reset asserted\n"); + if (!r&0b1) printf(" autologin not enabled\n"); + + addr=base+FC_CONFIG2; + cxl_mmio_read64(afu, addr,(__u64 *)&r); + if (r&0b100000) printf(" shutdown abrupt started\n"); + if (r&0b10000) printf(" shutdown normal started\n"); + + addr=base+FC_STATUS; + cxl_mmio_read64(afu, addr,(__u64 *)&r); + if (!r&link_training_cmplt) printf(" link train failed\n"); + else if (!r&cntrl_rdy) printf(" NVMe controller not rdy\n"); + else if (!r&ioq_rdy) printf(" ioq not ready\n"); + else if (!r&pcie_init_cmplt) printf(" pcie_init failed\n"); + if (r&0b100000) printf(" shutdown complete\n"); + if (r&0b10000) printf(" shutdown active\n"); + if (!r&compq_empty) printf(" compq not empty\n"); + if (!r&subq_empty) printf(" subq not empty\n"); + + addr=base+FC_CNT_LINKERR; + cxl_mmio_read64(afu, addr,(__u64 *)&r); + num=r&0xffff; + if (num) printf(" pcie_resets=%d\n", num); + num=(r&0xffff0000)>>16; + if (num) printf(" cntrlr_resets=%d\n",num); + + addr=base+FC_CNT_CRCTOT; + cxl_mmio_read64(afu, addr,(__u64 *)&r); + num=r&0xffffffff; + if (num) printf(" crc_errs=%d\n", num); + + addr=base+FC_CNT_TIMEOUT; + cxl_mmio_read64(afu, addr,(__u64 *)&r); + num=r&0xffffffff; + if (num) printf(" timeouts=%d\n", num); + + addr=base+FC_CNT_AFUABORT; + cxl_mmio_read64(afu, addr,(__u64 *)&r); + num=r&0xffffffff; + if (num) printf(" afuaborts=%d\n", num); + + addr=base+FC_ERROR; + cxl_mmio_read64(afu, addr,(__u64 *)&r); + num=r&0xffffffff; + if (num & 0x100000) printf(" PORT_LOGIN_FAILURE\n"); +} + +/** +******************************************************************************** +** \brief get_str +** \details +** +*******************************************************************************/ +void get_str(char *out, __u64 in) +{ + int i; + for (i=0; i<8; i++) { + out[i] = (in & 0xFFl); + in = in >> 8; + } + out[8] = 0; +} + +/** +******************************************************************************** +** \brief main +** \details +** +*******************************************************************************/ +int main(int argc, char **argv) +{ + struct cxl_ioctl_start_work *work; + long reported = 0; + char FF = 0xFF; + char c = '\0'; + char dev[128]; + char *_dev = NULL; + + /*-------------------------------------------------------------------------- + * process and verify input parms + *------------------------------------------------------------------------*/ + while (FF != (c=getopt(argc, argv, "d:h"))) + { + switch (c) + { + case 'd': _dev = optarg; break; + case 'h': + case '?': usage(); + } + } + if (!_dev) {usage();} + if (strncmp(_dev,"afu",3) == 0) {sprintf(dev, "/dev/cxl/%s",_dev);} + else {sprintf(dev, "%s",_dev);} + + afu = cxl_afu_open_dev(dev); + if (!afu) {printf("unable to open %s", dev); exit(-1);} + + debug("Allocated memory at %p for AFU\n", afu); + debug("Creating and opening AFU file descriptor %s\n", dev); + + work = (struct cxl_ioctl_start_work*) cxl_work_alloc(); + if (!work) {perror("cxl_work_alloc"); exit(-2);} + if (cxl_afu_attach_work(afu, work)) + { + perror("cxl_afu_attach_work failed\n"); + exit(-3); + } + + debug0("Start command succeeded on AFU\n"); + + if (cxl_mmio_map(afu, CXL_MMIO_BIG_ENDIAN) != 0) + { + perror("cxl_mmio_map"); + exit(-4); + } + + cxl_get_mmio_size(afu, &reported); + debug ("mmio_size 0x%lx for AFU\n", reported); + + debug0 ("Installing libcxl SIGBUS handler\n"); + cxl_mmio_install_sigbus_handler(); + + show_port(0); + show_port(1); + + exit(0); +} diff --git a/src/test/ffdc/makefile b/src/test/ffdc/makefile index 27104378..5d80dd65 100644 --- a/src/test/ffdc/makefile +++ b/src/test/ffdc/makefile @@ -35,8 +35,16 @@ include ${ROOTPATH}/config.mk #this application originates in surelock-afu/bringup/src and must be shipped for FFDC gathering ifeq ($(UNAME),AIX) # need to have a valid ffdc mechanism for AIX... else #Linux only -test: ${TESTDIR}/cxl_afu_dump +test: ${TESTDIR}/cxl_afu_dump ${TESTDIR}/cxl_afu_inject ${TESTDIR}/cxl_afu_status ${TESTDIR}/cxl_afu_dump: mkdir -p ${TESTDIR} - gcc -m64 -I . -I${ROOTPATH}/src/include -lpthread -o ${TESTDIR}/cxl_afu_dump -O0 afu.c capi_dev_nodes.c cxl_afu_dump2.c + gcc -m64 -I . -I${ROOTPATH}/src/include -L${ROOTPATH}/img -lpthread -o ${TESTDIR}/cxl_afu_dump -O0 afu.c capi_dev_nodes.c cxl_afu_dump2.c -lcxl + +${TESTDIR}/cxl_afu_inject: + mkdir -p ${TESTDIR} + gcc -m64 -I . -I${ROOTPATH}/src/include -L${ROOTPATH}/img -lpthread -o ${TESTDIR}/cxl_afu_inject -O0 afu.c capi_dev_nodes.c cxl_afu_inject.c -lcxl + +${TESTDIR}/cxl_afu_status: + mkdir -p ${TESTDIR} + gcc -m64 -I . -I${ROOTPATH}/src/include -L${ROOTPATH}/img -lpthread -o ${TESTDIR}/cxl_afu_status -O0 cxl_afu_status.c -lcxl endif diff --git a/src/test/flashgt_temp.c b/src/test/flashgt_temp.c new file mode 100644 index 00000000..9739f01a --- /dev/null +++ b/src/test/flashgt_temp.c @@ -0,0 +1,179 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/test/flashgt_temp.c $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +uint64_t mmio_read(char resource_file[],uint64_t offset); +uint64_t mmio_write(char resource_file[],uint64_t offset,uint64_t val64); + +int main (int argc, char *argv[]) +{ + int CFG; + char device_num[1024]; + char bar2[1024]; + char cfg_file[1024]; + + if (argc < 2) { + printf("Usage: ./flashgt_temp \nExample: ./flashgt_temp 0000:01:00.0 0\n"); + return -1; + } + strcpy (device_num, argv[1]); + strcpy(bar2, "/sys/bus/pci/devices/"); + strcat(bar2, device_num); + strcat(bar2, "/resource2"); + + strcpy(cfg_file, "/sys/class/cxl/card"); + strcat(cfg_file, argv[2]); + strcat(cfg_file, "/device/config"); + + if ((CFG = open(cfg_file, O_RDWR)) < 0) { + printf("Can not open %s\n",cfg_file); + exit(-1); + } + + int temp,vendor,device; + lseek(CFG, 0, SEEK_SET); + read(CFG, &temp, 4); + vendor = temp & 0xFFFF; + device = (temp >> 16) & 0xFFFF; + // printf("Device ID: %04X\n", device); + //printf("Vendor ID: %04X\n", vendor); + + if ( (vendor != 0x1014) || (( device != 0x0477) && (device != 0x04cf) && (device != 0x0601))) { + printf("Unknown Vendor or Device ID\n"); + exit(-1); + } + + uint64_t mmio_ptmon_addr = 0x0000000000000138; + lseek(CFG, 0x404, SEEK_SET); + read(CFG, &temp,4); + //printf(" VSEC Length/VSEC Rev/VSEC ID: 0x%08X\n", temp); + + // Set stdout to autoflush + setvbuf(stdout, NULL, _IONBF, 0); + +// ------------------------------------------------------------------------------- +// MMIOs to UCD +// ------------------------------------------------------------------------------- +uint64_t mmio_write_data = 0xFFFFFFFFFFFFFFFF; +uint64_t mmio_read_data = 0x0; +uint8_t temperature = 0x00; +float temperature2 = 0; + + //printf("Enabling Temperature Monitor\n"); + mmio_write(bar2,mmio_ptmon_addr,mmio_write_data); + + mmio_read_data = mmio_read(bar2,mmio_ptmon_addr); + //printf("PTMON data is %016llx \n", mmio_read_data); + temperature = (mmio_read_data >> 40);//16 bit mmio field is 8.8 format. Get 8 bit integer portion here. + temperature2 = temperature + ((mmio_read_data << 24) >> 56)/256.00;//Add 8 bit unsigned integer portion to 8 bit decimal portion. + printf("FPGA Temperature is %8.8f degrees Celsius\n", temperature2); + + //printf("Disabling Temperature Monitor\n"); + mmio_write_data = 0x0000000000000000; + mmio_write(bar2,mmio_ptmon_addr,mmio_write_data); + + close(CFG); + return 0; +} + + + +uint64_t mmio_read (char resource_file[],uint64_t offset) +{ + int priv2; + struct stat sb; + void *memblk, *adr; + uint64_t val64; + + if ((priv2 = open(resource_file, O_RDWR)) < 0) { + printf("Can not open %s\n",resource_file); + exit(-1); + } + + fstat(priv2, &sb); + + memblk = mmap(NULL,sb.st_size, PROT_WRITE|PROT_READ, MAP_SHARED, priv2, 0); + if (memblk == MAP_FAILED) { + printf("Can not mmap %s\n",resource_file); + exit(-1); + } + + + adr = memblk + (offset & (sb.st_size - 1)); + + val64 = *((uint64_t *)adr); + + munmap(memblk,sb.st_size); + + close(priv2); + val64 = ((val64>>56)&0xff) | ((val64>>40)&0xff00) | ((val64>>24)&0xff0000) | ((val64>>8)&0xff000000) | ((val64<<8)&0xff00000000) | ((val64<<24)&0xff0000000000) | ((val64<<40)&0xff000000000000) | ((val64<<56)&0xff00000000000000); + return val64; +} + +uint64_t mmio_write (char resource_file[],uint64_t offset, uint64_t val64) +{ + int priv2; + struct stat sb; + void *memblk, *adr; + + if ((priv2 = open(resource_file, O_RDWR)) < 0) { + printf("Can not open %s\n",resource_file); + exit(-1); + } + + fstat(priv2, &sb); + + memblk = mmap(NULL,sb.st_size, PROT_WRITE|PROT_READ, MAP_SHARED, priv2, 0); + if (memblk == MAP_FAILED) { + printf("Can not mmap %s\n",resource_file); + exit(-1); + } + + + adr = memblk + (offset & (sb.st_size - 1)); + + val64 = ((val64>>56)&0xff) | ((val64>>40)&0xff00) | ((val64>>24)&0xff0000) | ((val64>>8)&0xff000000) | ((val64<<8)&0xff00000000) | ((val64<<24)&0xff0000000000) | ((val64<<40)&0xff000000000000) | ((val64<<56)&0xff00000000000000); + *((uint64_t *)adr) = val64; + + munmap(memblk,sb.st_size); + close(priv2); + + return 0; +} + diff --git a/src/test/framework/README.md b/src/test/framework/README.md deleted file mode 100644 index 8505be7d..00000000 --- a/src/test/framework/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Test Framework # -This package relies on the [Google Test](https://github.com/google/googletest/) framework. - -It was last-tested with Google Test 1.7. Appropriate makefiles are included. - -To quickly get started: - -``` -$ pushd src/test/framework -$ git clone git@github.com:google/googletest.git -$ popd -``` - -Once an initial copy of google test is downloaded, "make tests" will automatically build and link appropriate objects. diff --git a/src/test/framework/gtest.mk b/src/test/framework/gtest.mk new file mode 100644 index 00000000..62aaa701 --- /dev/null +++ b/src/test/framework/gtest.mk @@ -0,0 +1,85 @@ +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/test/framework/gtest.mk $ +# +# IBM CONFIDENTIAL +# +# COPYRIGHT International Business Machines Corp. 2014 - 2017 +# +# p1 +# +# Object Code Only (OCO) source materials +# Licensed Internal Code Source Materials +# IBM Surelock Licensed Internal Code +# +# The source code for this program is not published or other- +# wise divested of its trade secrets, irrespective of what has +# been deposited with the U.S. Copyright Office. +# +# Origin: 30 +# +# IBM_PROLOG_END +# A sample Makefile for building Google Test and using it in user +# tests. Please tweak it to suit your environment and project. You +# may want to move it to your project's root directory. +# +# SYNOPSIS: +# +# make [all] - makes everything. +# make TARGET - makes the given target. +# make clean - removes all files generated by make. + +# Please tweak the following variable definitions as needed by your +# project, except GTEST_HEADERS, which you can use in your own targets +# but shouldn't modify. + +# Points to the root of Google Test, relative to where this file is. +# Remember to tweak this if you move this file. +GTEST_DIR =$(ROOTPATH)/${GTESTDIR} + +# Flags passed to the preprocessor. +# Set Google Test's header directory as a system directory, such that +# the compiler doesn't generate warnings in Google Test headers. +CPPFLAGS += -isystem $(GTEST_DIR)/include + + +# Flags passed to the C++ compiler. +#CXXFLAGS += -g -Wall -Wextra -pthread + + + +# All Google Test headers. Usually you shouldn't change this +# definition. +GTEST_HEADERS = $(GTEST_DIR)/include/gtest/*.h \ + $(GTEST_DIR)/include/gtest/internal/*.h + + +# Builds gtest.a and gtest_main.a. + +# Usually you shouldn't tweak such internal variables, indicated by a +# trailing _. +GTEST_SRCS_ = $(GTEST_DIR)/src/*.cc $(GTEST_DIR)/src/*.h $(GTEST_HEADERS) + +# For simplicity and to avoid depending on Google Test's +# implementation details, the dependencies specified below are +# conservative and not optimized. T + +gtest-all.o : $(GTEST_SRCS_) + $(CXX) $(CPPFLAGS) -I$(GTEST_DIR) $(CXXFLAGS) -c \ + $(GTEST_DIR)/src/gtest-all.cc + +gtest_main.o : $(GTEST_SRCS_) + $(CXX) $(CPPFLAGS) -I$(GTEST_DIR) $(CXXFLAGS) -c \ + $(GTEST_DIR)/src/gtest_main.cc + +gtest.a : gtest-all.o + $(AR) $(ARFLAGS) $@ $^ + +gtest_main.a : gtest-all.o gtest_main.o + $(AR) $(ARFLAGS) $@ $^ + + + + + diff --git a/src/test/framework/gtest.mk.aix.xlc b/src/test/framework/gtest.mk.aix.xlc new file mode 100644 index 00000000..259c1894 --- /dev/null +++ b/src/test/framework/gtest.mk.aix.xlc @@ -0,0 +1,63 @@ +# A sample Makefile for building Google Test and using it in user +# tests. Please tweak it to suit your environment and project. You +# may want to move it to your project's root directory. +# +# SYNOPSIS: +# +# make [all] - makes everything. +# make TARGET - makes the given target. +# make clean - removes all files generated by make. + +# Please tweak the following variable definitions as needed by your +# project, except GTEST_HEADERS, which you can use in your own targets +# but shouldn't modify. + +# Points to the root of Google Test, relative to where this file is. +# Remember to tweak this if you move this file. +GTEST_DIR =${GTESTDIR} + +# Flags passed to the preprocessor. +# Set Google Test's header directory as a system directory, such that +# the compiler doesn't generate warnings in Google Test headers. +CPPFLAGS += -DOLD_ANSIC_AIX_VERSION -I $(GTEST_DIR)/include + + +# Flags passed to the C++ compiler. +#CXXFLAGS += -g -Wall -Wextra -pthread + + + +# All Google Test headers. Usually you shouldn't change this +# definition. +GTEST_HEADERS = $(GTEST_DIR)/include/gtest/*.h \ + $(GTEST_DIR)/include/gtest/internal/*.h + + +# Builds gtest.a and gtest_main.a. + +# Usually you shouldn't tweak such internal variables, indicated by a +# trailing _. +GTEST_SRCS_ = $(GTEST_DIR)/src/*.cc $(GTEST_DIR)/src/*.h $(GTEST_HEADERS) + +# For simplicity and to avoid depending on Google Test's +# implementation details, the dependencies specified below are +# conservative and not optimized. T + +gtest-all.o : $(GTEST_SRCS_) + $(CXX) $(CPPFLAGS) -I$(GTEST_DIR) $(CXXFLAGS) -c \ + $(GTEST_DIR)/src/gtest-all.cc + +gtest_main.o : $(GTEST_SRCS_) + $(CXX) $(CPPFLAGS) -I$(GTEST_DIR) $(CXXFLAGS) -c \ + $(GTEST_DIR)/src/gtest_main.cc + +gtest.a : gtest-all.o + $(AR) $(ARFLAGS) $@ $^ + +gtest_main.a : gtest-all.o gtest_main.o + $(AR) $(ARFLAGS) $@ $^ + + + + + diff --git a/src/test/framework/gtest.objtests.mk b/src/test/framework/gtest.objtests.mk index fdc7adc6..176a71d5 100644 --- a/src/test/framework/gtest.objtests.mk +++ b/src/test/framework/gtest.objtests.mk @@ -16,7 +16,7 @@ # Points to the root of Google Test, relative to where this file is. # Remember to tweak this if you move this file. -GTEST_DIR =$(ROOTPATH)/src/test/framework/googletest/googletest +GTEST_DIR =$(ROOTPATH)/${GTESTDIR} UNAME=$(shell uname) diff --git a/src/test/ioppt.pl b/src/test/ioppt.pl new file mode 100755 index 00000000..2411ec86 --- /dev/null +++ b/src/test/ioppt.pl @@ -0,0 +1,378 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/test/ioppt.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +## +# ioppt.pl -c "blockio -d/dev/sgX -q1" -n 100 + +use strict; +use warnings; +use Fcntl; +use Fcntl ':seek'; +use Getopt::Long; +use Time::HiRes qw(usleep); + +my $cmdin=""; +my $ncmds; +my $cfile=""; +my @devs; +my $devstr=""; +my $prthelp=""; +my $verbose=""; +my $vverbose=""; +my $running=1; +my $outfn; +my @uticks; +my @sticks; +my $STAT; +my $ut=0; +my $st=0; +my $tt=0; +my $i=0; +my @pids; +my $pids_in=""; +my $npid=0; +my $iops=0; +my $ioppt=0; +my $tstart; +my $tstop; +my $tdiff; +my $tps; + +#------------------------------------------------------------------------------- +# print usage +#------------------------------------------------------------------------------- +sub usage() +{ + print "\n"; + print "Usage: ioppt.pl [-n num_to_exec] [-c cmd] [-d \"dev1 dev2 ...\"] [-v|-vv]\n"; + print " -n : number of cmds to execute \n"; + print " -c : cmd string to execute \n"; + print " -d : list of devices to use \n"; + print " -v : verbose, Print debug messages \n"; + print " -h or --help : Help - Print this message \n"; + exit 0; +} + +#------------------------------------------------------------------------------- +# run commands +#------------------------------------------------------------------------------- +sub run_cmds +{ + my $d=0; + my $pdev=""; + my $jobs=0; + + if ($cmdin =~ /fio/ && defined $ENV{'NUMJOBS'}) {$jobs=$ENV{'NUMJOBS'};} + if ($verbose && $jobs) {print "n=$ncmds NUMJOBS=$jobs\n";} + + $SIG{CHLD} = 'IGNORE'; + unlink $outfn; + + for ($i=0; $i<$ncmds; $i++) + { + if ($cmdin ne "") + { + if ($devstr) + { + $ENV{'DEV'}=$devs[$d]; + $ENV{'FVT_DEV'}=$devs[$d]; + $pdev="DEV=$ENV{'DEV'}"; + } + if ($verbose) {print "exec: $pdev $cmdin\n";} + if ($devstr && ++$d==@devs) {$d=0;} + if (my $pid=fork()) + { + chomp $pid; + if ($pid =~ /^\s*$/) {die "fork failed\n";} + if ($cmdin =~ /fio/) + { + for (my $cnt=0; $cnt<100000; $cnt++) + { + my $pidstr=qx(pgrep -P $pid); + if ($pidstr =~ /^\s*$/) {next;} + my @cpids=split /\s+/, $pidstr; + my @newpids; + foreach my $lpid (@cpids) + { + chomp $lpid; + if ($lpid =~ /^\s*$/) {next;} + push @newpids, $lpid; + } + if (($jobs>=1 && scalar @newpids >= $jobs) || + !$jobs && scalar @newpids > 0) + { + push(@pids, @newpids); + last; + } + } + } + else + { + push @pids, $pid; + } + } + elsif ($pid==0) + { + if (! defined $ENV{'FVT_DEV'}) {$ENV{'FVT_DEV'}="";} + exec "FVT_DEV=$ENV{'FVT_DEV'} exec $cmdin >> $outfn"; + exit 0; + } + else {print "fork failed\n";} + } + } + if ( $cmdin =~ /fio/ && $verbose) + { + my $rpids=$ncmds; + if ($jobs) {$rpids*=$jobs;} + if (scalar @pids != $rpids) + { + print "npids(" . scalar @pids . ") != request($rpids)\n"; + } + } +} + +#------------------------------------------------------------------------------- +# get linux ticks for list of processes +#------------------------------------------------------------------------------- +sub get_linux +{ + if ($vverbose) {print "get_linux\n";} + + while ( $running==1 ) + { + $running=0; + + for ($i=0; $i<@pids; $i++) + { + my $open=0; + + my $file = "/proc/$pids[$i]/stat"; + + open(STAT, $file) or next; + + $running=1; + + seek (STAT, SEEK_SET, 0); + my $fstring = ; + if (defined $fstring == 0 || length $fstring <= 0) {close(STAT); next;} + + my @stat = split /\s+/ , $fstring; + $uticks[$i] = $stat[13]; + $sticks[$i] = $stat[14]; + close(STAT); + + if ($vverbose) {print "$pids[$i]: $uticks[$i] $sticks[$i]\n";} + } + usleep(100000); + } +} + +#------------------------------------------------------------------------------- +# get aix ticks for list of processes +#------------------------------------------------------------------------------- +sub get_aix +{ + my $pdata=0; + my $nb=0; + + if ($vverbose) {print "get_aix\n";} + + while ( $running==1 ) + { + $running=0; + + for (my $i=0; $i<@pids; $i++) + { + my $open=0; + + open my $fh, '<:raw', "/proc/$pids[$i]/status" or next; + + seek $fh, 124, 0; + $nb = read $fh, $pdata, 48; + close $fh; + + my ($utv_sec, $utv_nsec, undef, undef, $stv_sec, $stv_nsec) = unpack 'N N N N N N', $pdata; + + $uticks[$i] = ($utv_sec*100) + ($utv_nsec/10000000); + $sticks[$i] = ($stv_sec*100) + ($stv_nsec/10000000); + if ($vverbose) {printf "%d: %.2f %.2f\n", $pids[$i], $uticks[$i], $sticks[$i];} + + $running=1; + } + usleep(100000); + } +} + +#------------------------------------------------------------------------------- +# Main +#------------------------------------------------------------------------------- +if (@ARGV==0) {usage();} + +if (! GetOptions ("n=i" => \$ncmds, + "c=s" => \$cmdin, + "d=s" => \$devstr, + "p=s" => \$pids_in, + "h|help!" => \$prthelp, + "v" => \$verbose, + "vv" => \$vverbose + )) +{ + usage(); +} +if ($ARGV[0]) +{ + print "\nUnknown Command Line Options:\n"; + foreach(@ARGV) + { + print " $_\n"; + } + print "\n"; + usage(); +} +if ($vverbose) +{ + $verbose=1; + if ($cmdin) {print "-c $cmdin ";} + if ($ncmds) {print "-n $ncmds ";} + if ($devstr) {print "-d $devstr ";} + if ($pids_in) {print "-p $pids_in ";} + print "\n"; +} +if ($cmdin =~ /^\s*$/) {print "-c option required\n"; usage();} +if ($ncmds =~ /^\s*$/) {print "-n option required\n"; usage();} +if ($prthelp) {usage();} +if ($devstr) {@devs=split / /, $devstr;} + + +#------------------------------------------------------------------------------- +# fork the commands +#------------------------------------------------------------------------------- +$outfn="/tmp/out_p.$$"; +$tstart=time(); +usleep(10000); +run_cmds(); +if ($verbose) {print scalar @pids . " pids: " . join(", ", @pids) . "\n";} + +#------------------------------------------------------------------------------- +# gather ticks for processes, then sum them +#------------------------------------------------------------------------------- +if (qx(uname -a) =~ /AIX/) {get_aix();} +else {get_linux();} + +$tstop=time(); +$tdiff=$tstop-$tstart; + +for ($i=0; $i<@pids; $i++) +{ + if (defined($uticks[$i])) + { + $ut+=$uticks[$i]; + if ($verbose) {printf "%d: uticks:%.2f\n", $pids[$i], $uticks[$i];} + } + if (defined($sticks[$i])) + { + $st+=$sticks[$i]; + if ($verbose) {printf "%d: sticks:%.2f\n", $pids[$i], $sticks[$i];} + } +} +$tt=$ut+$st; +$tps=$tt/$tdiff; + +#------------------------------------------------------------------------------- +# parse and sum iops from the command(s) output file +#------------------------------------------------------------------------------- +open FDATA, $outfn; +foreach my $line () +{ + if ($cmdin =~ /block/) + { + if ($line =~ /iops/) + { + foreach my $str (split(/ /,$line)) + { + if ($str =~ /iops/) + { + my @ss=split(/iops:/,$str); + if ($verbose) {print "iops=$ss[1]\n";} + $iops+=$ss[1]; + last; + } + } + } + } + elsif ($cmdin =~ /fio/) + { + if ($line =~ /iops/) + { + foreach my $str (split(/ /,$line)) + { + if ($str =~ /iops/) + { + my @ss=split(/iops=/,$str); + my $n=$ss[1]; + $n=~ tr/,/ /; + if ($verbose) {print "iops=$n\n";} + $iops+=$n; + } + } + } + } + elsif ($cmdin =~ /run_kv_benchmark/ || $cmdin =~ /ark_perf_tool/) + { + if ($line =~ /io\/s/) + { + foreach my $str (split(/ /,$line)) + { + if ($str =~ /io\/s/) + { + my @ss=split(/io\/s:/,$str); + if ($verbose) {print "iops=$ss[1]\n";} + $iops+=$ss[1]; + } + } + } + } + elsif ($cmdin =~ /asyncstress/) + { + if ($line =~ /thru/) + { + my @str = split(/,/,$line); + if ($verbose) {print "iops=$str[28]\n";} + $iops+=$str[28]; + } + } +} +close FDATA; +unlink $outfn; + +#------------------------------------------------------------------------------- +# print results +#------------------------------------------------------------------------------- +if ($iops > 0) {$ioppt=$iops/$tps;} + +printf "usr:%4.1f%% ptps:%5d iops:%7d ioppt:%5d\n", + ($ut/$tt)*100, $tps, $iops, $ioppt; +exit 0; + diff --git a/src/test/ioppts.qd b/src/test/ioppts.qd new file mode 100755 index 00000000..3df802f4 --- /dev/null +++ b/src/test/ioppts.qd @@ -0,0 +1,56 @@ +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/test/ioppts.qd $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +# ENG=libaio NR=100 NW=0 DEVS="/dev/sdk /dev/sdl /dev/sdm /dev/sdn" PROCS=10 SECS=10 ./ppts.fio ../img ../obj/tests ../parm.fio +# ENG=ktms NR=100 NW=0 DEVS="/dev/sg4 /dev/sg5 /dev/sg6 /dev/sg7" PROCS=10 SECS=10 ./ppts.fio ../img ../obj/tests ../parm.fio +#where $2 contains: +# parm.fio: +#[global] +#buffered=0 +#bs=4k +#iodepth=${QD} +#numjobs=1 +#group_reporting=1 +#size=8g +#rw=randrw +#rwmixread=${NR} +#rwmixwrite=${NW} +#ioengine=${ENG} +#runtime=${SECS} +#time_based + +#[job1] +#filename=${DEV} + +#------- + +if [ ! -z $1 ] +then + export LD_LIBRARY_PATH=$1 +fi +echo $PROCS|awk '{printf "%dP QD1 ", $1}'; QD=1 ioppt.pl -c "$CMD" -n $PROCS -d "$DEVS" +echo $PROCS|awk '{printf "%dP QD8 ", $1}'; QD=8 ioppt.pl -c "$CMD" -n $PROCS -d "$DEVS" +echo $PROCS|awk '{printf "%dP QD16 ", $1}'; QD=16 ioppt.pl -c "$CMD" -n $PROCS -d "$DEVS" +echo $PROCS|awk '{printf "%dP QD32 ", $1}'; QD=32 ioppt.pl -c "$CMD" -n $PROCS -d "$DEVS" +echo $PROCS|awk '{printf "%dP QD128 ", $1}'; QD=128 ioppt.pl -c "$CMD" -n $PROCS -d "$DEVS" diff --git a/src/test/makefile b/src/test/makefile index effd550e..3052bae6 100644 --- a/src/test/makefile +++ b/src/test/makefile @@ -28,14 +28,8 @@ UNAME=$(shell uname) # Where to find user code.(Relative path(from the makefile) preferred for portability) ROOTPATH = ../.. USER_DIR = . -SUBDIRS = ffdc.d +SUBDIRS = usfs.d TESTDIR = ${ROOTPATH}/obj/tests - -OBJS = afu.o capi_dev_nodes.o - -OBJS64 = afu.64o capi_dev_nodes.64o - -MODULE = afu EXPFLAGS = -bexpall #test code != production code, so allow warnings here. @@ -43,10 +37,20 @@ ALLOW_WARNINGS = yes export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${ROOTPATH}/img LIBPATHS = -L${ROOTPATH}/img -LINKLIBS = -lcflshcom -lcflsh_block -lafu -larkdb -larkalloc +LINKLIBS = -lcflshcom -lcflsh_block -larkdb -larkalloc + +BTESTS = asyncstress asyncstress2 + +ifneq ($(TARGET_PLATFORM),x86_64) +SUBDIRS = ffdc.d vpd.d +BTESTS += transport_test blk_test pvtestauto xlate pblkread +OBJS = afu.o capi_dev_nodes.o +OBJS64 = afu.64o capi_dev_nodes.64o +MODULE = afu +LINKLIBS += -lafu +endif -BTESTS =transport_test blk_test pvtestauto xlate pblkread asyncstress asyncstress2 BIN_TESTS=$(addprefix ${TESTDIR}/, ${BTESTS}) GTESTS = run_fvt @@ -59,10 +63,11 @@ BTESTS64 = $(addsuffix 64, ${BTESTS}) BIN_TESTS64 = $(addprefix ${TESTDIR}/, ${BTESTS64}) GTESTS64 = $(addsuffix 64, ${GTESTS}) GTESTS64_DIR = $(addprefix $(TESTDIR)/, $(GTESTS64)) -BITS = 64 +BITS = 64 else #Linux only LINKLIBS+=-lpthread -ludev +BTESTS +=flashgt_temp endif run_fvt_OFILES = \ @@ -72,18 +77,13 @@ run_fvt_OFILES = \ fvt_kv_tst_simple.o fvt_kv_tst_scenario.o fvt_kv_tst_sync_pth.o \ fvt_kv_tst_async_cb.o fvt_kv_tst_errors.o fvt_kv_tst_sync_async.o - -MODULE = afu -OBJS = afu.o capi_dev_nodes.o -OBJS64 = afu.64o capi_dev_nodes.64o - CFLAGS += \ - -D__FVT__\ + -D__FVT__ -Wno-unused-result\ -I$(ROOTPATH)/src/kv \ -I$(ROOTPATH)/src/kv/test \ -I$(ROOTPATH)/src/block \ -I$(ROOTPATH)/src/common \ - -I$(ROOTPATH)/src/test/framework/googletest/googletest/include + -I$(ROOTPATH)/${GTESTINC} CXXFLAGS+=$(CFLAGS) VPATH += \ diff --git a/src/test/multi_process_perf b/src/test/multi_process_perf index 8100f874..01c00f99 100755 --- a/src/test/multi_process_perf +++ b/src/test/multi_process_perf @@ -24,11 +24,9 @@ if [[ $1 = "-h" ]] then - echo "Usage: multi_process_perf [-csv] [libpath] [binpath]" + echo "Usage: multi_process_perf [libpath] [binpath]" echo " ex: multi_process_perf" - echo " ex: multi_process_perf -csv" echo " ex: multi_process_perf /.../surelock-sw/img /.../surelock-sw/obj/tests" - echo " ex: multi_process_perf -csv /.../surelock-sw/img /.../surelock-sw/obj/tests" exit fi @@ -49,18 +47,11 @@ fi DEVS=4 if [[ $_uname = "AIX" ]] then - DEV1=/dev/hdisk0 - DEV2=/dev/hdisk1 - DEV3=/dev/hdisk2 - DEV4=/dev/hdisk4 + DEV1=/dev/hdisk18 + DEV2=/dev/hdisk18 + DEV3=/dev/hdisk19 + DEV4=/dev/hdisk19 _64=64 - for d in 0 1 2 3 - do - if [[ $(lsmpio -l hdisk$d|grep hdisk$d|wc -l) -ne 2 ]]; - then - echo "running to only one port for hdisk$d" - fi - done else # use only /dev/sg* devices if [[ $(hostname) == cougar* ]] @@ -90,13 +81,7 @@ else _64= fi -csv=0 -if [[ $1 = "-csv" ]] -then - csv=1 -fi - -if [[ ! -z $1 && $csv -ne 1 ]] +if [[ ! -z $1 ]] then if [[ $_uname = "AIX" ]] then @@ -112,46 +97,39 @@ fi function ark { -echo "ARK Performance, using fvt_ark_perf_tool" +echo "ARK Performance, using run_kv_benchmark" for len in 100; do #len, 4096 131072 - for q in 1 8 16; do #QD, nasync, ops per ark - for j in 1; do #npool, ark threads - for ctx in 1 25 50; do #ctxts/processes (x's 4) - rm -f /tmp/out_p - c=0 - while [ $c -lt $ctx ]; do - cmd="FVT_DEV=$DEV1 $cmd_dir/fvt_ark_perf_tool$_64 -A -c 1 -j $q -n $j -k $len -v $len -s 15 >> /tmp/out_p" - eval $cmd& - cmd="FVT_DEV=$DEV2 $cmd_dir/fvt_ark_perf_tool$_64 -A -c 1 -j $q -n $j -k $len -v $len -s 15 >> /tmp/out_p" - eval $cmd& - cmd="FVT_DEV=$DEV3 $cmd_dir/fvt_ark_perf_tool$_64 -A -c 1 -j $q -n $j -k $len -v $len -s 15 >> /tmp/out_p" - eval $cmd& - cmd="FVT_DEV=$DEV4 $cmd_dir/fvt_ark_perf_tool$_64 -A -c 1 -j $q -n $j -k $len -v $len -s 15 >> /tmp/out_p" - eval $cmd& - let c=c+1 - PID=$! - done - wait - iops=0 - for d in $(cat /tmp/out_p|awk -F "io/s:" '{print $2}'|grep sec|awk -F " secs" '{print $1}'); do ((iops+=$d)); done - if [[ $iops -gt 800000 ]]; then iops=800000; fi - if [[ $csv -eq 1 ]] - then - echo "2Devs,KV,$len,contexts,$(($ctx*$DEVS)),arkthPerContext,$j,queue,$q,iops,$iops" - else - printf "Processes:%-3d npool:%-2d QD:%-3d klen:%-2d vlen:%-4d IOPS:%-6d\n" $(($ctx*$DEVS)) $j $q $len $len $iops - fi - if [[ $iops -gt 700000 ]]; then break; fi - done - done + for q in 1 8 16; do #QD, nasync, ops per ark + for ctx in 1 25 50; do #ctxts/processes (x's 4) + rm -f /tmp/out_p + c=0 + while [ $c -lt $ctx ]; do + cmd="$cmd_dir/run_kv_benchmark$_64 -d $DEV1 -q $q -s 25 >> /tmp/out_p" + eval $cmd& + cmd="$cmd_dir/run_kv_benchmark$_64 -d $DEV2 -q $q -s 25 >> /tmp/out_p" + eval $cmd& + cmd="$cmd_dir/run_kv_benchmark$_64 -d $DEV3 -q $q -s 25 >> /tmp/out_p" + eval $cmd& + cmd="$cmd_dir/run_kv_benchmark$_64 -d $DEV4 -q $q -s 25 >> /tmp/out_p" + eval $cmd& + let c=c+1 + PID=$! done + wait + iops=0 + for d in $(cat /tmp/out_p|awk -F "io/s:" '{print $2}'|grep sec|awk -F " secs" '{print $1}'); do ((iops+=$d)); done + if [[ $iops -gt 800000 ]]; then iops=800000; fi + printf "Processes:%-3d QD:%-3d IOPS:%-6d\n" $(($ctx*$DEVS)) $q $iops + if [[ $iops -gt 700000 ]]; then break; fi + done + done done } function block { -echo "BLOCK Performance, using blockplistio" +echo "BLOCK Performance, using blockio" for rd in 100; do #%reads for q in 1 8 16; do #QD for j in 1 25 50 #Processes @@ -159,33 +137,28 @@ for rd in 100; do #%reads rm -f /tmp/out_p c=0 while [ $c -lt $j ]; do - cmd="$cmd_dir/blockplistio$_64 -d $DEV1 -l 1 -c $q -s 10 $_intrps >> /tmp/out_p" + cmd="$cmd_dir/blockio$_64 -d $DEV1 -q $q -r $rd -s 10 $_intrps >> /tmp/out_p" eval $cmd& - cmd="$cmd_dir/blockplistio$_64 -d $DEV2 -l 1 -c $q -s 10 $_intrps >> /tmp/out_p" + cmd="$cmd_dir/blockio$_64 -d $DEV2 -q $q -r $rd -s 10 $_intrps >> /tmp/out_p" eval $cmd& - cmd="$cmd_dir/blockplistio$_64 -d $DEV3 -l 1 -c $q -s 10 $_intrps >> /tmp/out_p" + cmd="$cmd_dir/blockio$_64 -d $DEV3 -q $q -r $rd -s 10 $_intrps >> /tmp/out_p" eval $cmd& - cmd="$cmd_dir/blockplistio$_64 -d $DEV4 -l 1 -c $q -s 10 $_intrps >> /tmp/out_p" + cmd="$cmd_dir/blockio$_64 -d $DEV4 -q $q -r $rd -s 10 $_intrps >> /tmp/out_p" eval $cmd& let c=c+1 PID=$! done wait iops=0 - for d in $(cat /tmp/out_p|grep iops|awk '{print $9}'|awk -F : '{print $2}'); do ((iops+=$d)); done; + for d in $(cat /tmp/out_p|grep iops|awk -F"iops:" '{print $2}'); do ((iops+=$d)); done; if [[ $iops -gt 800000 ]]; then iops=800000; fi - if [[ $csv -eq 1 ]] - then - echo "2Devs,QdPerJob,$q,JobPerDev,$j,Rd,$rd,Wt,$((100-$rd)),iops,$iops" - else - printf "Processes:%-3d QD:%-4d Rd:%-3d Wt:%-3d iops:%-6d\n" $(($j*$DEVS)) $q $rd $((100-$rd)) $iops - fi + printf "Processes:%-3d QD:%-4d Rd:%-3d Wt:%-3d iops:%-6d\n" $(($j*$DEVS)) $q $rd $((100-$rd)) $iops if [[ $iops -gt 700000 ]]; then break; fi done done done echo "" -echo "BLOCK Performance, using blockio" +echo "BLOCK Performance, using blockplistio" for rd in 100; do #%reads for q in 1 8 16; do #QD for j in 1 25 50 #Processes @@ -193,27 +166,22 @@ for rd in 100; do #%reads rm -f /tmp/out_p c=0 while [ $c -lt $j ]; do - cmd="$cmd_dir/blockio$_64 -d $DEV1 -q $q -r $rd -s 10 $_intrps >> /tmp/out_p" + cmd="$cmd_dir/blockplistio$_64 -d $DEV1 -l 1 -c $q -s 10 $_intrps >> /tmp/out_p" eval $cmd& - cmd="$cmd_dir/blockio$_64 -d $DEV2 -q $q -r $rd -s 10 $_intrps >> /tmp/out_p" + cmd="$cmd_dir/blockplistio$_64 -d $DEV2 -l 1 -c $q -s 10 $_intrps >> /tmp/out_p" eval $cmd& - cmd="$cmd_dir/blockio$_64 -d $DEV3 -q $q -r $rd -s 10 $_intrps >> /tmp/out_p" + cmd="$cmd_dir/blockplistio$_64 -d $DEV3 -l 1 -c $q -s 10 $_intrps >> /tmp/out_p" eval $cmd& - cmd="$cmd_dir/blockio$_64 -d $DEV4 -q $q -r $rd -s 10 $_intrps >> /tmp/out_p" + cmd="$cmd_dir/blockplistio$_64 -d $DEV4 -l 1 -c $q -s 10 $_intrps >> /tmp/out_p" eval $cmd& let c=c+1 PID=$! done wait iops=0 - for d in $(cat /tmp/out_p|grep iops|awk '{print $11}'|awk -F : '{print $2}'); do ((iops+=$d)); done; + for d in $(cat /tmp/out_p|grep iops|awk '{print $9}'|awk -F : '{print $2}'); do ((iops+=$d)); done; if [[ $iops -gt 800000 ]]; then iops=800000; fi - if [[ $csv -eq 1 ]] - then - echo "2Devs,QdPerJob,$q,JobPerDev,$j,Rd,$rd,Wt,$((100-$rd)),iops,$iops" - else - printf "Processes:%-3d QD:%-4d Rd:%-3d Wt:%-3d iops:%-6d\n" $(($j*$DEVS)) $q $rd $((100-$rd)) $iops - fi + printf "Processes:%-3d QD:%-4d Rd:%-3d Wt:%-3d iops:%-6d\n" $(($j*$DEVS)) $q $rd $((100-$rd)) $iops if [[ $iops -gt 700000 ]]; then break; fi done done diff --git a/src/test/pticks.pl b/src/test/pticks.pl new file mode 100644 index 00000000..fb51530a --- /dev/null +++ b/src/test/pticks.pl @@ -0,0 +1,120 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/test/pticks.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +## +use strict; +use warnings; +use Fcntl; +use Fcntl ':seek'; +use Getopt::Long; +use Time::HiRes qw(usleep); + +#foreach (split(//, $d1)) { +# printf("%02x", ord($_)); +#} +#print "\n"; + +my $running=1; +my @uticks; +my @sticks; +my $STAT; +my $ut=0; +my $st=0; +my $i=0; + +#input string of process ids, output sum of ticks for all pids + +sub get_linux +{ + while ( $running==1 ) + { + $running=0; + + for ($i=0; $i<@ARGV; $i++) + { + my $open=0; + + my $file = "/proc/$ARGV[$i]/stat"; + + open(STAT, $file) or next; + + $running=1; + + seek (STAT, SEEK_SET, 0); + + my @stat = split /\s+/ , ; + $uticks[$i] = $stat[13]; + $sticks[$i] = $stat[14]; + close(STAT); + } + usleep(100000); + } +} + +sub get_aix +{ + my $pdata=0; + my $nb=0; + + while ( $running==1 ) + { + $running=0; + + for (my $i=0; $i<@ARGV; $i++) + { + my $open=0; + + open my $fh, '<:raw', "/proc/$ARGV[$i]/status" or next; + + seek $fh, 124, 0; + $nb = read $fh, $pdata, 48; + close $fh; + + my ($utv_sec, $utv_nsec, undef, undef, $stv_sec, $stv_nsec) = unpack 'N N N N N N', $pdata; + + $uticks[$i] = ($utv_sec*100) + ($utv_nsec/10000000); + $sticks[$i] = ($stv_sec*100) + ($stv_nsec/10000000); +# printf("pid:%d utime:%d stime:%d\n", $ARGV[$i], $uticks[$i, $sticks[$i]); + + $running=1; + } + usleep(1000000); + } +} + +#MAIN +`uname -a|grep AIX`; +if ($? == 0) {get_aix();} +else {get_linux();} + +for ($i=0; $i<@ARGV; $i++) +{ + if (defined($uticks[$i])) {print "$ARGV[$i] : sys: $sticks[$i] usr:$uticks[$i]\n"; $ut+=$uticks[$i];} + if (defined($sticks[$i])) {$st+=$sticks[$i];} +} + +printf "uticks:%d sticks:%d\n", $ut, $st; +exit 0; + diff --git a/src/test/run_ioppts b/src/test/run_ioppts new file mode 100755 index 00000000..aa5cef26 --- /dev/null +++ b/src/test/run_ioppts @@ -0,0 +1,116 @@ +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/test/run_ioppts $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +# run_ioppts [2] +# *use 2 as a parm to run a second time to the separate list of devices +# (which could include more adapters) + +if [[ $1 -eq -2 ]] +then + DO_2ADAP=1 +fi + +_AIX=$(uname -a|awk '{print $1}') + +SECS=20 +DEVS_LEG_1ADAP="/dev/sdk /dev/sdl /dev/sdq /dev/sdr /dev/sdm /dev/sds /dev/sdo /dev/sdp /dev/sdv /dev/sdu" +DEVS_LEG_2ADAP="/dev/sdk /dev/sdl /dev/sdq /dev/sdr /dev/sdm /dev/sds /dev/sdo /dev/sdp /dev/sdv /dev/sdu" +DEVS_SUP_1ADAP="/dev/sg34 /dev/sg35 /dev/sg40 /dev/sg41" +DEVS_SUP_2ADAP="/dev/sg34 /dev/sg35 /dev/sg40 /dev/sg41" + +if [[ $_AIX = AIX ]] +then + _64=64 + DEVS_SUP_1ADAP="/dev/hdisk19" + DEVS_SUP_2ADAP="/dev/hdisk18 /dev/hdisk19" +else + if [[ $(cat /proc/sys/fs/aio-max-nr) -ne 500000 ]] + then + echo "500000" > /proc/sys/fs/aio-max-nr + fi +fi + +DEVS_LEG=$DEVS_LEG_1ADAP; +DEVS_SUP=$DEVS_SUP_1ADAP; + +for loop in 1 2 +do + if [[ $loop = 2 && $1 != 2 ]]; then break; fi + + if [[ $loop = 2 && $1 = 2 ]] + then + DEVS_LEG=$DEVS_LEG_2ADAP; + DEVS_SUP=$DEVS_SUP_2ADAP; + fi + echo "LEG="$DEVS_LEG; + echo "SUP="$DEVS_SUP; + if [[ $_AIX != AIX ]] + then + echo "Legacy, fio:libaio" + PROCS=2 NUMJOBS=1 NR=100 NW=0 ENG=libaio QD=1 SECS=$SECS DEVS=$DEVS_LEG CMD="fio parm.fio" ioppts.qd + echo "Legacy, fio:ktms" + PROCS=2 NUMJOBS=1 NR=100 NW=0 ENG=ktms QD=1 SECS=$SECS DEVS=$DEVS_SUP CMD="fio parm.fio" ioppts.qd + fi + echo "blockio, polling" + PROCS=2 CMD="blockio$_64 -d \$DEV -q\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd + echo "blockio, bckgrnd thrds" + PROCS=2 CMD="blockio$_64 -i -d \$DEV -q\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd + echo "blockplistio" + PROCS=2 CMD="blockplistio$_64 -d \$DEV -l1 -c\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd + echo "run_kv_benchmark" + PROCS=2 CMD="run_kv_benchmark$_64 -d \$DEV -q\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd + + if [[ $_AIX != AIX ]] + then + echo "Legacy, fio:libaio" + PROCS=16 NUMJOBS=1 NR=100 NW=0 ENG=libaio QD=1 SECS=$SECS DEVS=$DEVS_LEG CMD="fio parm.fio" ioppts.qd + echo "Legacy, fio:ktms" + PROCS=16 NUMJOBS=1 NR=100 NW=0 ENG=ktms QD=1 SECS=$SECS DEVS=$DEVS_SUP CMD="fio parm.fio" ioppts.qd + fi + echo "blockio, polling" + PROCS=16 CMD="blockio$_64 -d \$DEV -q\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd + echo "blockio, bckgrnd thrds" + PROCS=16 CMD="blockio$_64 -i -d \$DEV -q\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd + echo "blockplistio" + PROCS=16 CMD="blockplistio$_64 -d \$DEV -l1 -c\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd + echo "run_kv_benchmark" + PROCS=16 CMD="run_kv_benchmark$_64 -d \$DEV -q\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd + + if [[ $_AIX != AIX ]] + then + echo "Legacy, fio:libaio" + PROCS=100 NUMJOBS=1 NR=100 NW=0 ENG=libaio QD=1 SECS=$SECS DEVS=$DEVS_LEG CMD="fio parm.fio" ioppts.qd + echo "Legacy, fio:ktms" + PROCS=100 NUMJOBS=1 NR=100 NW=0 ENG=ktms QD=1 SECS=$SECS DEVS=$DEVS_SUP CMD="fio parm.fio" ioppts.qd + fi + echo "blockio, polling" + PROCS=100 CMD="blockio$_64 -d \$DEV -q\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd + echo "blockio, bckgrnd thrds" + PROCS=100 CMD="blockio$_64 -i -d \$DEV -q\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd + echo "blockplistio" + PROCS=100 CMD="blockplistio$_64 -d \$DEV -l1 -c\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd + echo "run_kv_benchmark" + PROCS=100 CMD="run_kv_benchmark$_64 -d \$DEV -q\$QD -s$SECS" DEVS=$DEVS_SUP ioppts.qd +done diff --git a/src/test/usfs/cusfs.c b/src/test/usfs/cusfs.c new file mode 100644 index 00000000..0acb74b5 --- /dev/null +++ b/src/test/usfs/cusfs.c @@ -0,0 +1,2374 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* bos720 src/bos/usr/ccs/lib/libcflsh_block/tools/blk_test.c 1.2 */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#ifdef _AIX +static char sccsid[] = "%Z%%M% %I% %W% %G% %U%"; +#endif + + +/* + * COMPONENT_NAME: (sysxcflashblock) CAPI Flash block library + * + * FUNCTIONS: + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +#ifndef _AIX +#define _GNU_SOURCE +#endif /* !_AIX */ + + +#include +#include +#include +#include +#include +#include +#include +#ifndef _MACOSX +#include +#endif /* !_MACOS */ +#include +#include +#ifdef _AIX +#include +#else +#include +#endif +#if !defined(_AIX) && !defined(_MACOSX) +#include +#endif +#include +#include +#include +#include +#include + + +#ifndef FALSE +#define FALSE 0 +#endif + +#ifndef TRUE +#define TRUE 1 +#endif + +#ifndef TIMELEN +#define TIMELEN 26 /* Linux does have a define for the minium size of the a timebuf */ + /* However linux man pages say it is 26 */ +#endif + +#ifndef MIN +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#endif + +/************************************************************************/ +/* Subcommand enumerations */ +/************************************************************************/ + +typedef +enum { + CUSFS_APP_CMD_CRFS = 0x1, /* Create filesystem */ + CUSFS_APP_CMD_RMFS = 0x2, /* Remove filesystem */ + CUSFS_APP_CMD_MKDIR = 0x3, /* Make directory */ + CUSFS_APP_CMD_RMDIR = 0x4, /* Remove directory */ + CUSFS_APP_CMD_LS = 0x5, /* List files in directory */ + CUSFS_APP_CMD_CAT = 0x6, /* Cat file */ + CUSFS_APP_CMD_STAT = 0x7, /* stat a file */ + CUSFS_APP_CMD_CHMOD = 0x8, /* chmod a file */ + CUSFS_APP_CMD_CHGRP = 0x9, /* chgrp a file */ + CUSFS_APP_CMD_CHOWN = 0xa, /* chown a file */ + CUSFS_APP_CMD_MV = 0xb, /* move a file */ + CUSFS_APP_CMD_COPY = 0xc, /* copy a file */ + CUSFS_APP_CMD_LN = 0xd, /* link a file */ + CUSFS_APP_CMD_TRUNCATE = 0xe, /* truncate a file */ + CUSFS_APP_CMD_UTIME = 0xf, /* set mod time on a file */ + CUSFS_APP_CMD_FSCK = 0x10, /* fsck the filesystem */ + CUSFS_APP_CMD_TOUCH = 0x11, /* set mod time on a file to now. */ + CUSFS_APP_CMD_RMFILE = 0x12, /* Remove a file. */ + CUSFS_APP_CMD_QUERYFS = 0x13, /* Query a filesystem */ + CUSFS_APP_CMD_STATFS = 0x14, /* Query a filesystem */ + CUSFS_APP_CMD_AIO_COPY = 0x15, /* copy a file doing writes via AIO */ + CUSFS_APP_CMD_LAST = 0x16, /* Not valid command */ +} cusfs_app_cmdline_t; + + +struct cmd_lookup_s { + cusfs_app_cmdline_t cmd; + char *cmd_string; +}; + + +struct cmd_lookup_s cmds[] = { + {CUSFS_APP_CMD_CRFS,"crfs"}, + {CUSFS_APP_CMD_RMFS,"rmfs"}, + {CUSFS_APP_CMD_MKDIR,"mkdir"}, + {CUSFS_APP_CMD_RMDIR,"rmdir"}, + {CUSFS_APP_CMD_LS,"ls"}, + {CUSFS_APP_CMD_CAT,"cat"}, + {CUSFS_APP_CMD_STAT,"stat"}, + {CUSFS_APP_CMD_CHMOD,"chmod"}, + {CUSFS_APP_CMD_CHGRP,"chgrp"}, + {CUSFS_APP_CMD_CHOWN,"chown"}, + {CUSFS_APP_CMD_MV,"mv"}, + {CUSFS_APP_CMD_COPY,"cp"}, + {CUSFS_APP_CMD_AIO_COPY,"aiocp"}, + {CUSFS_APP_CMD_LN,"ln"}, + {CUSFS_APP_CMD_TRUNCATE,"truncate"}, + {CUSFS_APP_CMD_UTIME,"utime"}, + {CUSFS_APP_CMD_TOUCH,"touch"}, + {CUSFS_APP_CMD_RMFILE,"rm"}, + {CUSFS_APP_CMD_QUERYFS,"queryfs"}, + {CUSFS_APP_CMD_STATFS,"statfs"}, + {CUSFS_APP_CMD_FSCK,"fsck"}, +}; + + + + +/************************************************************************/ +/* Global variables adjusted by parsing command line arguments. */ +/************************************************************************/ + +static char *command = NULL; + +static int bflag; /* indicates the b flag was passed on command + * line which indicates a block/transfer size + * us specified. + */ + +static int block_size = 0; /* point to the arg for -b flag */ + +static int cflag; /* indicates the c flag was passed on command + * line which indicates a command was + * specified. + */ + +static cusfs_app_cmdline_t cmd_op = CUSFS_APP_CMD_LAST; /* point to the arg for the -c flag */ + +static int dflag; /* indicates the d flag was passed on command + * line which indicates a device name was + * specified. + */ +static char *device_name = NULL; /* point to the arg for -d flag */ + +static int Dflag; /* indicates the D flag was passed on command + * line which indicates another device name was + * specified. + */ +static char *Device_name = NULL; /* point to the arg for -D flag */ + + +static int force_flag; /* indicates the f flag was passed on command + * line which indicates a force option is + * specified. + */ + + +static int gflag; /* indicates the g flag was passed on command + * line which indicates a group was + * specified. + */ +static int hflag; /* indicates the h flag was passed on command + * line which indicates a help was + * specified. + */ +static int group_id = 0; /* point to the arg for -g flag */ + +static int lflag; /* indicates the l flag was passed on command + * line which indicates a length was + * specified. + */ +static int length = 0; /* point to the arg for -l flag */ + +static int mflag; /* indicates the m flag was passed on command + * line which indicates a mode was + * specified. + */ +static int mode = 0; /* point to the arg for -m flag */ + +static int oflag; /* indicates the o flag was passed on command + * line which indicates to preserve old settings + * as specified. + */ +static int pflag; /* indicates the p flag was passed on command + * line which indicates a path was + * specified. + */ +static char *path = NULL; /* point to the arg for -p flag */ + +static int Pflag; /* indicates the P flag was passed on command + * line which indicates an (old) path was + * specified. + */ +static char *Path = NULL; /* point to the arg for -P flag */ + + +static int sflag; /* indicates the s flag was passed on command + * line which indicates a path was + * specified. + */ + +static int tflag; /* indicates the t flag was passed on command + * line which indicates a time was + * specified. + */ +static int time_val = 0; /* point to the arg for -t flag */ + +static int uflag; /* indicates the u flag was passed on command + * line which indicates a user id was + * specified. + */ +static int user_id = 0; /* point to the arg for -u flag */ + +static int fork_flag = 0; /* indicates the z flag was passed on command + * line which indicates a to fork + */ + + +char disk_pathname[PATH_MAX]; +char disk_pathname2[PATH_MAX]; + + +/* + * NAME: Usage + * + * FUNCTION: print usage message and returns + * + * + * INPUTS: + * argc -- INPUT; the argc parm passed into main(). + * argv -- INPUT; the argv parm passed into main(). + * + * RETURNS: + * 0: Success + * -1: Error + * + */ +static void +usage(void) +{ + + fprintf(stderr,"\n"); + + fprintf(stderr,"Usage:cusfs -c command\n\n"); + fprintf(stderr," -c crfs -d disk_name [ -f ]\n"); + fprintf(stderr," where -f is force flag\n"); + fprintf(stderr," -c queryfs -d disk_name\n"); + fprintf(stderr," -c statfs -d disk_name\n"); + fprintf(stderr," -c rmfs -d disk_name\n"); + fprintf(stderr," -c mkdir -p path -d disk_name\n"); + fprintf(stderr," -c rmdir -p path -d disk_name\n"); + fprintf(stderr," -c rm -p path -d disk_name\n"); + fprintf(stderr," -c ls -p path -d disk_name\n"); + fprintf(stderr," -c cat -p path -d disk_name\n"); + fprintf(stderr," -c stat -p path -d disk_name\n"); + fprintf(stderr," -c chmod -m mode -p path -d disk_name\n"); + fprintf(stderr," -c chgrp -g group -p path -d disk_name\n"); + fprintf(stderr," -c chown -u user_id -p path -d disk_name\n"); + fprintf(stderr," -c mv -P old_path -p new_path -d disk_name\n"); + fprintf(stderr," -c cp -P old_path [ -D old_disk_name] -p new_path [ -d new_disk_name ] [ -o ] [ -z ] [ -b block_size ]\n"); + fprintf(stderr," where -o is to preserve mode and time stamps\n"); + fprintf(stderr," where -z indicates to fork after opening and stating\n"); + fprintf(stderr," -c aiocp -P old_path [ -D old_disk_name] -p new_path [ -d new_disk_name ] [ -o ] [ -z ] [ -b block_size ]\n"); + fprintf(stderr," where -o is to preserve mode and time stamps\n"); + fprintf(stderr," where -z indicates to fork after opening and stating\n"); + fprintf(stderr," -c ln -P old_path -p new_path -d new_disk_name [ -s ]\n"); + fprintf(stderr," -c truncate -p path -d disk_name -l length_bytes\n"); + fprintf(stderr," -c utime -p path -d disk_name -t times\n"); + fprintf(stderr," -c touch -p path -d disk_name\n"); + fprintf(stderr," -c fsck -d disk_name\n"); + fprintf(stderr," -h : help\n"); + + + return; +} + + +/* + * NAME: parse_args + * + * FUNCTION: Parse command line arguments + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ + +static int +parse_args(int argc, char **argv) +{ + extern int optind; + extern char *optarg; + int rc = 0; + int c; + int i; + + + bflag = FALSE; + cflag = FALSE; + dflag = FALSE; + Dflag = FALSE; + gflag = FALSE; + lflag = FALSE; + mflag = FALSE; + oflag = FALSE; + pflag = FALSE; + Pflag = FALSE; + sflag = FALSE; + tflag = FALSE; + uflag = FALSE; + force_flag = FALSE; + fork_flag = FALSE; + + + /* + * Get parameters + */ + while ((c = getopt(argc,argv,"b:c:d:D:g:l:m:p:P:t:u:fhsz")) != EOF) + { + switch (c) + { + + case 'b' : + if (optarg) { + + block_size = atoi(optarg); + + + bflag = TRUE; + } else { + + fprintf(stderr,"-b flag requires block_size \n"); + rc = EINVAL; + + } + + break; + case 'c' : + if (optarg) { + + command = optarg; + + i = 0; + while (i < CUSFS_APP_CMD_LAST) { + + if (!strcmp(cmds[i].cmd_string,command)) { + + cmd_op = cmds[i].cmd; + break; + } + i++; + } + if (cmd_op == CUSFS_APP_CMD_LAST) { + fprintf(stderr,"Invalid cmd %s for -c flag\n",command); + + usage(); + rc = -1; + } else { + cflag = TRUE; + } + } else { + + + fprintf(stderr,"-c flag requires a value to be supplied\n"); + + } + break; + case 'd' : + device_name = optarg; + if (device_name) { + dflag = TRUE; + } else { + + fprintf(stderr,"-d flag requires a device name \n"); + rc = EINVAL; + + } + break; + + case 'D' : + Device_name = optarg; + if (Device_name) { + Dflag = TRUE; + } else { + + fprintf(stderr,"-D flag requires a device name \n"); + rc = EINVAL; + + } + break; + + case 'f' : + + force_flag = TRUE; + break; + + + case 'h' : + + usage(); + break; + + + case 'g' : + if (optarg) { + + group_id = atoi(optarg); + + gflag = TRUE; + } else { + + fprintf(stderr,"-g flag requires mode \n"); + rc = EINVAL; + + } + break; + + + case 'l' : + if (optarg) { + + length = atoi(optarg); + + lflag = TRUE; + } else { + + fprintf(stderr,"-l flag requires length \n"); + rc = EINVAL; + + } + break; + + + case 'm' : + if (optarg) { + + mode = atoi(optarg); + + mflag = TRUE; + } else { + + fprintf(stderr,"-m flag requires mode \n"); + rc = EINVAL; + + } + break; + + case 'o' : + + oflag = TRUE; + break; + + case 'p' : + path = optarg; + if (path) { + pflag = TRUE; + } else { + + fprintf(stderr,"-p flag requires a path \n"); + rc = EINVAL; + + } + break; + case 'P' : + Path = optarg; + if (Path) { + Pflag = TRUE; + } else { + + fprintf(stderr,"-P flag requires a path \n"); + rc = EINVAL; + + } + break; + + + case 's' : + + sflag = TRUE; + break; + + case 't' : + if (optarg) { + + time_val = atoi(optarg); + + tflag = TRUE; + } else { + + fprintf(stderr,"-t flag requires length \n"); + rc = EINVAL; + + } + break; + + case 'u' : + if (optarg) { + + user_id = atoi(optarg); + + uflag = TRUE; + } else { + + fprintf(stderr,"-u flag requires mode \n"); + rc = EINVAL; + + } + break; + case 'z' : + + fork_flag = TRUE; + break; + default: + usage(); + break; + }/*switch*/ + }/*while*/ + + if (!cflag) { + fprintf(stderr,"The -c flag is required to specify a command\n"); + usage(); + rc = EINVAL; + return rc; + } + + if ((!dflag) && + (cmd_op != CUSFS_APP_CMD_COPY) && + (cmd_op != CUSFS_APP_CMD_AIO_COPY)) { + fprintf(stderr,"The -d flag is required to specify a device name\n"); + usage(); + rc = EINVAL; + return rc; + } + + if ((!pflag) && + ((cmd_op == CUSFS_APP_CMD_MKDIR) || + (cmd_op == CUSFS_APP_CMD_RMDIR) || + (cmd_op == CUSFS_APP_CMD_RMFILE) || + (cmd_op == CUSFS_APP_CMD_LS) || + (cmd_op == CUSFS_APP_CMD_CAT) || + (cmd_op == CUSFS_APP_CMD_STAT) || + (cmd_op == CUSFS_APP_CMD_CHMOD) || + (cmd_op == CUSFS_APP_CMD_CHGRP) || + (cmd_op == CUSFS_APP_CMD_CHOWN) || + (cmd_op == CUSFS_APP_CMD_MV) || + (cmd_op == CUSFS_APP_CMD_COPY) || + (cmd_op == CUSFS_APP_CMD_AIO_COPY) || + (cmd_op == CUSFS_APP_CMD_LN) || + (cmd_op == CUSFS_APP_CMD_TRUNCATE) || + (cmd_op == CUSFS_APP_CMD_TOUCH) || + (cmd_op == CUSFS_APP_CMD_UTIME))) { + + + fprintf(stderr,"The -p flag is required to specify the path for this command %s\n",command); + usage(); + rc = EINVAL; + + } + + if ((!Pflag) && + ((cmd_op == CUSFS_APP_CMD_MV) || + (cmd_op == CUSFS_APP_CMD_COPY) || + (cmd_op == CUSFS_APP_CMD_AIO_COPY) || + (cmd_op == CUSFS_APP_CMD_LN))) { + + + fprintf(stderr,"The -P flag is required to specify the old path for this command %s\n",command); + usage(); + rc = EINVAL; + return rc; + } + + if ((!tflag) && + (cmd_op == CUSFS_APP_CMD_UTIME)) { + + fprintf(stderr,"The -t flag is required to specify the time for this command %s\n",command); + usage(); + rc = EINVAL; + return rc; + + } + + if ((!lflag) && + (cmd_op == CUSFS_APP_CMD_TRUNCATE)) { + + fprintf(stderr,"The -l flag is required to specify the length for this command %s\n",command); + usage(); + rc = EINVAL; + return rc; + + } + if (!(Dflag || dflag) && + ((cmd_op == CUSFS_APP_CMD_COPY) || + (cmd_op == CUSFS_APP_CMD_AIO_COPY))) { + + fprintf(stderr,"Either -d or -D flag is required to specify the for this command %s\n",command); + usage(); + rc = EINVAL; + return rc; + + } + + if ((!mflag) && + (cmd_op == CUSFS_APP_CMD_CHMOD)) { + + fprintf(stderr,"The -m flag is required to specify the mode for this command %s\n",command); + usage(); + rc = EINVAL; + return rc; + + } + + if ((!gflag) && + (cmd_op == CUSFS_APP_CMD_CHGRP)) { + + fprintf(stderr,"The -g flag is required to specify the group for this command %s\n",command); + usage(); + rc = EINVAL; + return rc; + + } + + if ((!uflag) && + (cmd_op == CUSFS_APP_CMD_CHOWN)) { + + fprintf(stderr,"The -u flag is required to specify the user_id for this command %s\n",command); + usage(); + rc = EINVAL; + return rc; + + } + + + return (rc); + + +}/*parse_args*/ + + +/* + * NAME: common_seek + * + * FUNCTION: Seeks either a file in either the + * CAPI Flash filesystem or the traditional + * filesystem. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +#ifdef _AIX +offset_t common_seek(int fd, offset_t Offset, int Whence, int capi_ufs) +{ + offset_t rc; + + if (capi_ufs) { + rc = cusfs_llseek(fd,Offset,Whence); + + } else { + rc = llseek(fd,Offset,Whence); + + } + + + return rc; +} +#else + +off_t common_seek(int fd, off_t Offset, int Whence, int capi_ufs) +{ + off_t rc; + + if (capi_ufs) { + rc = cusfs_lseek(fd,Offset,Whence); + + } else { + rc = lseek(fd,Offset,Whence); + + } + + + return rc; +} +#endif + +/* + * NAME: common_fstat + * + * FUNCTION: fstat either a file in either the + * CAPI Flash filesystem or the traditional + * filesystem. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int common_fstat(int fd, struct stat *Buffer, int capi_ufs) +{ + int rc; + + if (capi_ufs) { + rc = cusfs_fstat(fd,Buffer); + + } else { + rc = fstat(fd,Buffer); + + } + + + return rc; +} + +/* + * NAME: common_read + * + * FUNCTION: read either a file in either the + * CAPI Flash filesystem or the traditional + * filesystem. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int common_read(int fd, void *buffer,size_t nbytes,int flags, int capi_ufs) +{ + int rc; + + if (capi_ufs) { + rc = cusfs_read(fd,buffer,nbytes); + + } else { + rc = read(fd,buffer,nbytes); + + } + + + return rc; +} + +/* + * NAME: common_write + * + * FUNCTION: write either a file in either the + * CAPI Flash filesystem or the traditional + * filesystem. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int common_write(int fd, void *buffer,size_t nbytes,int flags, int capi_ufs) +{ + int rc; + + if (capi_ufs) { + rc = cusfs_write(fd,buffer,nbytes); + + } else { + rc = write(fd,buffer,nbytes); + + } + + + return rc; +} + + +/* + * NAME: common_write + * + * FUNCTION: write either a file in either the + * CAPI Flash filesystem or the traditional + * filesystem. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +#ifdef _AIX +int common_aio_write(int fd, void *buffer,size_t nbytes,offset_t aio_offset,int flags, int capi_ufs) +#else +int common_aio_write(int fd, void *buffer,size_t nbytes,off_t aio_offset,int flags, int capi_ufs) +#endif +{ + int rc; + struct aiocb64 aiocb_arg; + int retry = 0; + + bzero(&aiocb_arg,sizeof(aiocb_arg)); + + aiocb_arg.aio_offset = aio_offset; + aiocb_arg.aio_buf = buffer; + aiocb_arg.aio_nbytes = nbytes; + aiocb_arg.aio_fildes = fd; + + if (capi_ufs) { + rc = cusfs_aio_write64(&aiocb_arg); + + } else { + rc = aio_write64(&aiocb_arg); + + } + + + if (rc < 0) { + + return rc; + } + + while (retry < 10000) { + + if (capi_ufs) { + rc = cusfs_aio_error64(&aiocb_arg); + + } else { + rc = aio_error64(&aiocb_arg); + + } + + if ((rc == 0) || + (rc != EINPROGRESS)) { + + break; + } + + usleep(1000); + retry++; + + } + + if (!rc) { + + + if (capi_ufs) { + rc = cusfs_aio_return64(&aiocb_arg); + + } else { + rc = aio_return64(&aiocb_arg); + + } + } + return rc; +} + + +/* + * NAME: common_fstat + * + * FUNCTION: close either a file in either the + * CAPI Flash filesystem or the traditional + * filesystem. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int common_close(int fd, int capi_ufs) +{ + int rc; + + if (capi_ufs) { + rc = cusfs_close(fd); + + } else { + rc = close(fd); + + } + + + return rc; +} + +/* + * NAME: create_fs + * + * FUNCTION: Create Userspace filesystem on a CAPI flash disk + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int create_fs(void) +{ + int rc = 0; + int flags = 0; + + + if (force_flag) { + + flags = CUSFS_FORCE_CREATE_FS; + } + + rc = cusfs_create_fs(device_name,flags); + + + return rc; +} + +/* + * NAME: query_fs + * + * FUNCTION: Query a userspace filesystem from a CAPI flash disk + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int query_fs(void) +{ + int rc = 0; + struct cusfs_query_fs query; + char timebuf[TIMELEN+1]; + + bzero(&query,sizeof(query)); + + + rc = cusfs_query_fs(device_name,&query,0); + + if ((!rc) || +#ifdef _AIX + (errno == ECORRUPT)) { +#else + (errno == EBADR)) { +#endif + + printf("version = 0x%x\n",query.version); + + if (query.flags & CUSFS_QRY_DIF_ENDIAN) { + + printf("flags = 0x%x: !Different endianess!\n",query.flags); + } else { + printf("flags = 0x%x\n",query.flags); + } + + + printf("os_type = 0x%x\n",query.os_type); + printf("fs_block_size = 0x%"PRIX64"\n",query.fs_block_size); + printf("disk_block_size = 0x%"PRIX64"\n",query.disk_block_size); + printf("num_blocks = 0x%"PRIX64"\n",query.num_blocks); + printf("free_table_size = 0x%"PRIX64"\n",query.free_block_table_size); + printf("inode_table_size = 0x%"PRIX64"\n",query.inode_table_size); + printf("create_time = %s",ctime_r(&(query.create_time),timebuf)); + printf("write_time = %s",ctime_r(&(query.write_time),timebuf)); + printf("mount_time = %s",ctime_r(&(query.mount_time),timebuf)); + printf("fsck_time = %s",ctime_r(&(query.fsck_time),timebuf)); + + } + + return rc; +} + + +/* + * NAME: stat_fs + * + * FUNCTION: Query a userspace filesystem from a CAPI flash disk + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int stat_fs(void) +{ + int rc = 0; + struct statfs statfs; + + + bzero(&statfs,sizeof(statfs)); + + + rc = cusfs_statfs(device_name,&statfs); + + if (!rc) { +#ifdef _AIX + printf("version = 0x%x\n",statfs.f_version); + printf("f_type = 0x%x\n",statfs.f_type); + printf("f_fsize = 0x%lx\n",statfs.f_fsize); + printf("f_fname = %s\n",statfs.f_fname); +#else + printf("f_frsize = 0x%lx\n",statfs.f_frsize); + +#endif + printf("f_bsize = 0x%lx\n",statfs.f_bsize); + printf("f_blocks = 0x%lx\n",statfs.f_blocks); + printf("f_bfree = 0x%lx\n",statfs.f_bfree); + printf("f_bavail = 0x%lx\n",statfs.f_bavail); + printf("f_files = 0x%lx\n",statfs.f_files); + printf("f_ffree = 0x%lx\n",statfs.f_ffree); + + } + + return rc; +} + +/* + * NAME: remove_fs + * + * FUNCTION: Remove Userspace filesystem from a CAPI flash disk + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int remove_fs(void) +{ + int rc = 0; + + rc = cusfs_remove_fs(device_name,0); + + return rc; +} + + +/* + * NAME: mk_directory + * + * FUNCTION: Make a directory + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int mk_directory(void) +{ + int rc = 0; + + rc = cusfs_mkdir(disk_pathname,mode); + + return rc; +} + +/* + * NAME: rm_directory + * + * FUNCTION: Remove a directory + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int rm_directory(void) +{ + int rc = 0; + + + rc = cusfs_rmdir(disk_pathname); + + return rc; +} + + +/* + * NAME: rm_file + * + * FUNCTION: Remove a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int rm_file(void) +{ + int rc = 0; + + + rc = cusfs_unlink(disk_pathname); + + return rc; +} + + +/* + * NAME: list_files + * + * FUNCTION: List files (ls) + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int list_files(void) +{ + int rc = 0; +#ifdef _AIX + DIR64 *directory; + struct dirent64 file; + struct dirent64 *result = &file; + struct stat64 stats; +#else + DIR *directory; + struct dirent file; + struct dirent *result = &file; + struct stat stats; +#endif + char stat_name[PATH_MAX]; + char timebuf[TIMELEN+1]; + char time_string[TIMELEN+1]; + +#ifdef _AIX + directory = cusfs_opendir64(disk_pathname); + +#else + directory = cusfs_opendir(disk_pathname); +#endif + + if (directory == NULL) { + + fprintf(stderr,"opendir64 returned null pointer with errno = %d\n",errno); + return -1; + } + + printf("Directory name = %s\n",path); + + printf("mode \tlinks\tuid\tgid\tmtime \tsize\tinode\tfilename\n"); + +#ifdef _AIX + while ((cusfs_readdir64_r(directory,&file,&result) == 0) && (result != NULL)) { +#else + while ((cusfs_readdir_r(directory,&file,&result) == 0) && (result != NULL)) { +#endif + + bzero(&stats,sizeof(stats)); + + sprintf(stat_name,"%s%s",disk_pathname,file.d_name); + + +#ifdef _AIX + rc = cusfs_stat64(stat_name,&stats); +#else + rc = cusfs_stat(stat_name,&stats); +#endif + + if (rc) { + break; + } + + sprintf(time_string,"%s",ctime_r(&(stats.st_mtime),timebuf)); + + /* + * Remove newline from time_string + */ + + time_string[strlen(time_string) - 1] = '\0'; + + + /* + * ?? This needs to be cleaned up + */ + + if (stats.st_mode & S_IFDIR) { + printf("d"); + } else if ( S_ISLNK(stats.st_mode)) { + printf("l"); + } else { + printf("-"); + } + + if (stats.st_mode & S_IRUSR) { + printf("r"); + } else { + printf("-"); + } + + if (stats.st_mode & S_IWUSR) { + printf("w"); + } else { + printf("-"); + } + + + if (stats.st_mode & S_IXUSR) { + printf("x"); + } else { + printf("-"); + } + + if (stats.st_mode & S_IRGRP) { + printf("r"); + } else { + printf("-"); + } + + if (stats.st_mode & S_IWGRP) { + printf("w"); + } else { + printf("-"); + } + + + if (stats.st_mode & S_IXGRP) { + printf("x"); + } else { + printf("-"); + } + + + if (stats.st_mode & S_IROTH) { + printf("r"); + } else { + printf("-"); + } + + if (stats.st_mode & S_IWOTH) { + printf("w"); + } else { + printf("-"); + } + + + if (stats.st_mode & S_IXOTH) { + printf("x"); + } else { + printf("-"); + } + +#if !defined(__64BIT__) && defined(_AIX) + printf("\t%d\t%d\t%d\t%s\t0x%"PRIX64"\t0x%x\t%s\n", + (int)stats.st_nlink,stats.st_uid, stats.st_gid, + time_string, + stats.st_size,stats.st_ino,file.d_name); +#else + printf("\t%d\t%d\t%d\t%s\t0x%"PRIX64"\t0x%"PRIX64"\t%s\n", + (int)stats.st_nlink,stats.st_uid, stats.st_gid, + time_string, + stats.st_size,stats.st_ino,file.d_name); +#endif + + } /*while */ + +#ifdef _AIX + if (cusfs_closedir64(directory)) { +#else + if (cusfs_closedir(directory)) { +#endif + + fprintf(stderr,"closedir64 failed with errno = %d\n",errno); + } + + +#ifdef _REMOVE + rc = cusfs_list_files(disk_pathname,0); +#endif + + return rc; +} + +/* + * NAME: move_file + * + * FUNCTION: Move a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int move_file(void) +{ + int rc = 0; + + rc = cusfs_rename(disk_pathname,disk_pathname2); + + return rc; +} + + +/* + * NAME: copy_file + * + * FUNCTION: Copy a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int copy_file(void) +{ + int rc = 0; + int src_fd; + int dest_fd; + int src_capi; + int dest_capi; + void *buf; + int buf_size = 4096; + int64_t remaining_size; + mode_t mode; + struct statfs usfs_statfs; + struct stat stats; + pid_t pid; +#ifdef _AIX + offset_t offset; +#else + off_t offset; +#endif + + + + + + bzero(&stats,sizeof(stats)); + + if (Dflag) { + /* + * A device name was specified, so this + * file is on a CAPI flash fileystem. + */ + + + /* + * Get filesystem optimal block size + */ + + bzero(&usfs_statfs,sizeof(usfs_statfs)); + + rc = cusfs_statfs(disk_pathname2,&usfs_statfs); + + if (rc) { + + fprintf(stderr,"statfs of %s failed with errno = %d\n", + disk_pathname2,errno); + return -1; + } + + if (bflag) { + buf_size = block_size; + } else { + buf_size = usfs_statfs.f_bsize; + } + + src_capi = TRUE; + + src_fd = cusfs_open(disk_pathname2,O_RDONLY,0); + + } else { + + /* + * A device was not specified. So + * this is from a traditional + * filesystem. + */ + + src_capi = FALSE; + + src_fd = open(Path,O_RDONLY,0); + } + + if (src_fd < 0) { + + fprintf(stderr,"Open of %s failed with errno = %d\n", + disk_pathname,errno); + return -1; + + } + + rc = common_fstat(src_fd,&stats,src_capi); + + if (dflag) { + /* + * A device name was specified, so this + * file is on a CAPI flash fileystem. + */ + + dest_capi = TRUE; + + /* + * Get filesystem optimal block size + */ + + bzero(&usfs_statfs,sizeof(usfs_statfs)); + + rc = cusfs_statfs(disk_pathname,&usfs_statfs); + + if (rc) { + + fprintf(stderr,"statfs of %s failed with errno = %d\n", + disk_pathname,errno); + return -1; + } + + if (Dflag) { + buf_size = MIN(usfs_statfs.f_bsize,buf_size); + } else { + buf_size = usfs_statfs.f_bsize; + } + + if (oflag) { + + /* + * Caller has requested + * we preserve permissions. + */ + + // TODO?? Need more work here to preserve time stamps, uid, gid etc. + + + mode = stats.st_mode; + } else { + + mode = (S_IRWXU | S_IRWXG | S_IRWXO); + } + + dest_fd = cusfs_open(disk_pathname,(O_WRONLY|O_CREAT),mode); + + rc = cusfs_ftruncate64(dest_fd,stats.st_size); + + if (rc) { + + fprintf(stderr,"cusf_ftruncate64 of %s failed with errno = %d\n", + disk_pathname2,errno); + common_close(src_fd,src_capi); + return -1; + + } + } else { + + /* + * A device was not specified. So + * this is from a traditional + * filesystem. + */ + + dest_capi = FALSE; + + dest_fd = open(path,(O_WRONLY|O_CREAT),0); + + rc = ftruncate64(dest_fd,stats.st_size); + + if (rc) { + + fprintf(stderr,"ftruncate64 of %s failed with errno = %d\n", + path,errno); + common_close(src_fd,src_capi); + return -1; + + } + } + + if (dest_fd < 0) { + + fprintf(stderr,"Open of %s failed with errno = %d\n", + disk_pathname2,errno); + return -1; + + } + + + if (fork_flag) { + + if ((pid = fork()) < 0) { /* fork failed */ + perror("Fork failed \n"); + + } + else if (pid > 0) { /* parents fork call */ + + fprintf(stderr,"Fork succeeded \n"); + + + /* + * Let parent sleep long enough for child to start up. + */ + + sleep(100); + + return 0; + } + + /* + * Only child get here + */ + + + + } + + offset = common_seek(src_fd,0,SEEK_SET,src_capi); + offset = common_seek(dest_fd,0,SEEK_SET,dest_capi); + + if (offset != 0) { + + fprintf(stderr,"seek to 0 of %s failed with errno = %d,\n", + disk_pathname2,errno); + return -1; + + } + + buf = malloc(buf_size); + + if (buf == NULL) { + + fprintf(stderr,"failed to malloc buf with errno = %d",errno); + } + + bzero(buf,buf_size); + + remaining_size = stats.st_size; + + while (remaining_size > 0) { + + rc = common_read(src_fd,buf,buf_size,0,src_capi); + + if (rc <= 0) { + fprintf(stderr,"Read failed with rc = %d, but buf_size = %d with errno = %d\n", + rc,buf_size,errno); + + break; + + } + + remaining_size -= rc; + + rc = common_write(dest_fd,buf,rc,0,dest_capi); + + } + + if (remaining_size == 0) { + + rc = 0; + } + + if (common_close(src_fd,src_capi)) { + + + fprintf(stderr,"Close source file failed with errno = %d\n",errno); + + } + + + if (common_close(dest_fd,dest_capi)) { + + + fprintf(stderr,"Close destination file failed with errno = %d\n",errno); + + } + + return rc; +} + +/* + * NAME: aio_copy_file + * + * FUNCTION: Copy a file using asynchronous writes + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int aio_copy_file(void) +{ + int rc = 0; + int src_fd; + int dest_fd; + int src_capi; + int dest_capi; + void *buf; + int buf_size = 4096; + int64_t remaining_size; + mode_t mode; + struct statfs usfs_statfs; + struct stat stats; + pid_t pid; +#ifdef _AIX + offset_t offset; + offset_t aio_offset; +#else + off_t offset; + off_t aio_offset; +#endif + + + + + + bzero(&stats,sizeof(stats)); + + if (Dflag) { + /* + * A device name was specified, so this + * file is on a CAPI flash fileystem. + */ + + + /* + * Get filesystem optimal block size + */ + + bzero(&usfs_statfs,sizeof(usfs_statfs)); + + rc = cusfs_statfs(disk_pathname2,&usfs_statfs); + + if (rc) { + + fprintf(stderr,"statfs of %s failed with errno = %d\n", + disk_pathname2,errno); + return -1; + } + + buf_size = usfs_statfs.f_bsize; + + src_capi = TRUE; + + src_fd = cusfs_open(disk_pathname2,O_RDONLY,0); + + } else { + + /* + * A device was not specified. So + * this is from a traditional + * filesystem. + */ + + src_capi = FALSE; + + src_fd = open(Path,O_RDONLY,0); + } + + if (src_fd < 0) { + + fprintf(stderr,"Open of %s failed with errno = %d\n", + disk_pathname,errno); + return -1; + + } + + rc = common_fstat(src_fd,&stats,src_capi); + + if (dflag) { + /* + * A device name was specified, so this + * file is on a CAPI flash fileystem. + */ + + dest_capi = TRUE; + + /* + * Get filesystem optimal block size + */ + + bzero(&usfs_statfs,sizeof(usfs_statfs)); + + rc = cusfs_statfs(disk_pathname,&usfs_statfs); + + if (rc) { + + fprintf(stderr,"statfs of %s failed with errno = %d\n", + disk_pathname,errno); + return -1; + } + + if (Dflag) { + buf_size = MIN(usfs_statfs.f_bsize,buf_size); + } else { + buf_size = usfs_statfs.f_bsize; + } + + if (oflag) { + + /* + * Caller has requested + * we preserve permissions. + */ + + // TODO?? Need more work here to preserve time stamps, uid, gid etc. + + + mode = stats.st_mode; + } else { + + mode = (S_IRWXU | S_IRWXG | S_IRWXO); + } + + dest_fd = cusfs_open(disk_pathname,(O_WRONLY|O_CREAT),mode); + + rc = cusfs_ftruncate64(dest_fd,stats.st_size); + + if (rc) { + + fprintf(stderr,"cusf_ftruncate64 of %s failed with errno = %d\n", + disk_pathname2,errno); + common_close(src_fd,src_capi); + return -1; + + } + } else { + + /* + * A device was not specified. So + * this is from a traditional + * filesystem. + */ + + dest_capi = FALSE; + + dest_fd = open(path,(O_WRONLY|O_CREAT),0); + + rc = ftruncate64(dest_fd,stats.st_size); + + if (rc) { + + fprintf(stderr,"ftruncate64 of %s failed with errno = %d\n", + path,errno); + common_close(src_fd,src_capi); + return -1; + + } + } + + if (dest_fd < 0) { + + fprintf(stderr,"Open of %s failed with errno = %d\n", + disk_pathname2,errno); + return -1; + + } + + + + if (fork_flag) { + + if ((pid = fork()) < 0) { /* fork failed */ + perror("Fork failed \n"); + + } + else if (pid > 0) { /* parents fork call */ + + + fprintf(stderr,"Fork succeeded \n"); + + + /* + * Let parent sleep long enough for child to start up. + */ + + sleep(100); + + return 0; + } + + /* + * Only child get here + */ + + } + + + + offset = common_seek(src_fd,0,SEEK_SET,src_capi); + aio_offset = common_seek(dest_fd,0,SEEK_SET,dest_capi); + + if (offset != 0) { + + fprintf(stderr,"seek to 0 of %s failed with errno = %d,\n", + disk_pathname2,errno); + return -1; + + } + + buf = malloc(buf_size); + + if (buf == NULL) { + + fprintf(stderr,"failed to malloc buf with errno = %d",errno); + } + + bzero(buf,buf_size); + + remaining_size = stats.st_size; + + while (remaining_size > 0) { + + rc = common_read(src_fd,buf,buf_size,0,src_capi); + + if (rc <= 0) { + fprintf(stderr,"Read failed with rc = %d, but buf_size = %d with errno = %d\n", + rc,buf_size,errno); + + break; + + } + + remaining_size -= rc; + + rc = common_aio_write(dest_fd,buf,rc,aio_offset,0,dest_capi); + + if (rc < 0) { + + break; + } else { + aio_offset += remaining_size; + } + + } + + if (remaining_size == 0) { + + rc = 0; + } + + if (common_close(src_fd,src_capi)) { + + + fprintf(stderr,"Close source file failed with errno = %d\n",errno); + + } + + + if (common_close(dest_fd,dest_capi)) { + + + fprintf(stderr,"Close destination file failed with errno = %d\n",errno); + + } + + return rc; +} + + +/* + * NAME: stat_file + * + * FUNCTION: Get statistics of a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int stat_file(void) +{ + int rc = 0; + int fd; + char timebuf[TIMELEN+1]; + struct stat stat; + + bzero(&stat,sizeof(stat)); + + fd = cusfs_open(disk_pathname,O_RDONLY,0); + + if (fd < 0) { + + fprintf(stderr,"Open failed with errno = %d\n",errno); + return -1; + + } + + rc = cusfs_fstat(fd,&stat); + + if (!rc) { + + printf("st_ino = 0x%"PRIX64"\n",stat.st_ino); + printf("st_mode = %o\n",stat.st_mode); + printf("st_nlink = 0x%"PRIX64"\n",stat.st_nlink); + printf("st_uid = %d\n",stat.st_uid); + printf("st_gid = %d\n",stat.st_gid); + printf("st_blksize = 0x%"PRIX64"\n",stat.st_blksize); + printf("st_atime = %s\n",ctime_r(&(stat.st_atime),timebuf)); + printf("st_mtime = %s\n",ctime_r(&(stat.st_mtime),timebuf)); + printf("st_ctime = %s\n",ctime_r(&(stat.st_ctime),timebuf)); +#if !defined(__64BIT__) && defined(_AIX) + printf("st_size = 0x%x\n",stat.st_size); +#else + printf("st_size = 0x%"PRIX64"\n",stat.st_size); +#endif + + + } + if (cusfs_close(fd)) { + + + fprintf(stderr,"Close failed with errno = %d\n",errno); + } + + return rc; +} + + + +/* + * NAME: link_file + * + * FUNCTION: Link (hard/sym) a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int link_file(void) +{ + int rc = 0; + + rc = cusfs_create_link(disk_pathname,disk_pathname2,mode,user_id,group_id,0); + + return rc; +} + +/* + * NAME: chmod_file + * + * FUNCTION: Change mode of a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int chmod_file(void) +{ + int rc = 0; + int fd; + + + fd = cusfs_open(disk_pathname,O_RDWR,0); + + if (fd < 0) { + + fprintf(stderr,"Open failed with errno = %d\n",errno); + return -1; + + } + + rc = cusfs_fchmod(fd,mode ); + + if (cusfs_close(fd)) { + + + fprintf(stderr,"Close failed with errno = %d\n",errno); + } + + + return rc; +} + + +/* + * NAME: chgrp_file + * + * FUNCTION: Change grp of a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int chgrp_file(void) +{ + int rc = 0; + int fd; + + + fd = cusfs_open(disk_pathname,O_RDWR,0); + + if (fd < 0) { + + fprintf(stderr,"Open failed with errno = %d\n",errno); + return -1; + + } + + rc = cusfs_fchown(fd,-1,group_id); + + if (cusfs_close(fd)) { + + + fprintf(stderr,"Close failed with errno = %d\n",errno); + } + + + return rc; +} + + +/* + * NAME: chown_file + * + * FUNCTION: Change owner of a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int chown_file(void) +{ + int rc = 0; + int fd; + + + fd = cusfs_open(disk_pathname,O_RDWR,0); + + if (fd < 0) { + + fprintf(stderr,"Open failed with errno = %d\n",errno); + return -1; + + } + + rc = cusfs_fchown(fd,user_id,-1); + + if (cusfs_close(fd)) { + + + fprintf(stderr,"Close failed with errno = %d\n",errno); + } + + return rc; +} + + +/* + * NAME: truncate_file + * + * FUNCTION: Truncate a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int truncate_file(void) +{ + int rc = 0; + int fd; + + fd = cusfs_open(disk_pathname,O_RDONLY,0); + + if (fd < 0) { + + fprintf(stderr,"Open failed with errno = %d\n",errno); + return -1; + + } + + rc = cusfs_ftruncate(fd,length); + + if (cusfs_close(fd)) { + + + fprintf(stderr,"Close failed with errno = %d\n",errno); + } + + return rc; +} + + +/* + * NAME: utime_file + * + * FUNCTION: Change time of a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int utime_file(void) +{ + int rc = 0; + struct utimbuf times; + + + //?? convert time to utimbuf + + times.actime = time_val; + times.modtime = time_val; + + rc = cusfs_utime(disk_pathname,×); + + return rc; +} + +/* + * NAME: touch_file + * + * FUNCTION: Change time of a file to now + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int touch_file(void) +{ + int rc = 0; + struct utimbuf times; + time_t curtime; + + + + curtime = time(NULL); + times.actime = curtime; + times.modtime = curtime; + + rc = cusfs_utime(disk_pathname,×); + + return rc; +} + +/* + * NAME: fsck + * + * FUNCTION: fsck on filesystem + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int fsck(void) +{ + int rc = 0; + + rc = cusfs_fsck(device_name,0); + + return rc; +} + + +/* + * NAME: main + * + * FUNCTION: + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error + * + */ + +int main (int argc, char **argv) +{ + int rc = 0; /* Return code */ + /* parse the input args & handle syntax request quickly */ + rc = parse_args( argc, argv ); + if (rc) + { + return (0); + } + + + if ( hflag) + { + usage(); + return (0); + } + + if (pflag & dflag) { + sprintf(disk_pathname,"%s%s%s",device_name, + CUSFS_DISK_NAME_DELIMINATOR,path); + } + + if (Pflag & Dflag) { + sprintf(disk_pathname2,"%s%s%s",Device_name, + CUSFS_DISK_NAME_DELIMINATOR,Path); + } + + cusfs_init(NULL,0); + + switch(cmd_op) { + case CUSFS_APP_CMD_CRFS: + rc = create_fs(); + break; + case CUSFS_APP_CMD_RMFS: + rc = remove_fs(); + break; + case CUSFS_APP_CMD_QUERYFS: + rc = query_fs(); + break; + case CUSFS_APP_CMD_STATFS: + rc = stat_fs(); + break; + case CUSFS_APP_CMD_MKDIR: + rc = mk_directory(); + break; + case CUSFS_APP_CMD_RMDIR: + rc = rm_directory(); + break; + case CUSFS_APP_CMD_LS: + rc = list_files(); + break; + case CUSFS_APP_CMD_MV: + rc = move_file(); + break; + case CUSFS_APP_CMD_COPY: + rc = copy_file(); + break; + case CUSFS_APP_CMD_AIO_COPY: + rc = aio_copy_file(); + break; + case CUSFS_APP_CMD_STAT: + rc = stat_file(); + break; + case CUSFS_APP_CMD_LN: + rc = link_file(); + break; + case CUSFS_APP_CMD_CHMOD: + rc = chmod_file(); + break; + case CUSFS_APP_CMD_CHGRP: + rc = chgrp_file(); + break; + case CUSFS_APP_CMD_CHOWN: + rc = chown_file(); + break; + case CUSFS_APP_CMD_TRUNCATE: + rc = truncate_file(); + break; + case CUSFS_APP_CMD_UTIME: + rc = utime_file(); + break; + case CUSFS_APP_CMD_TOUCH: + rc = touch_file(); + break; + case CUSFS_APP_CMD_RMFILE: + rc = rm_file(); + break; + case CUSFS_APP_CMD_FSCK: + rc = fsck(); + break; + default: + fprintf(stderr,"Invalid command 0x%x\n",cmd_op); + } + + if (rc) { + fprintf(stderr,"Errno = %d\n",errno); + } + + cusfs_term(NULL,0); + + + return rc; + +} diff --git a/src/test/usfs/cusfs_fuse.c b/src/test/usfs/cusfs_fuse.c new file mode 100644 index 00000000..525080fd --- /dev/null +++ b/src/test/usfs/cusfs_fuse.c @@ -0,0 +1,1596 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* bos720 src/bos/usr/ccs/lib/libcflsh_block/tools/blk_test.c 1.2 */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#ifdef _AIX +static char sccsid[] = "%Z%%M% %I% %W% %G% %U%"; +#endif + + +/* + * COMPONENT_NAME: (sysxcflashusfs) CAPI Flash userspace filesystem + * + * FUNCTIONS: + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + + +/* + * Set FUSE version to the level that corresponds to the 2.9.5 + * version that we built this against. + */ + +#define FUSE_USE_VERSION 26 + + +#define _FILE_OFFSET_BITS 64 + +#ifndef _AIX +#define _GNU_SOURCE +#endif /* !_AIX */ + + +#include +#include +#include +#include +#include +#include +#include +#ifndef _MACOSX +#include +#endif /* !_MACOS */ +#include +#include +#ifdef _AIX +#include +#else +#include +#endif +#if !defined(_AIX) && !defined(_MACOSX) +#include +#endif +#include +#include +#include +#include +#include +#include +#include + + +#ifndef FALSE +#define FALSE 0 +#endif + +#ifndef TRUE +#define TRUE 1 +#endif + +#ifndef TIMELEN +#define TIMELEN 26 /* Linux does have a define for the minium size of the a timebuf */ + /* However linux man pages say it is 26 */ +#endif + + + +static char *device_name = NULL; /* point to device name */ + + + +/* + * NOTE: The paths passed to each of the callback functions + * are relative to the root directory of that filesystem + */ + + + +/* + * NOTE3: The return codes of the call back fuse functions needs + * to either be 0 (good completion) or -errno for erro. + */ + + +/* + * NAME: cusfs_fuse_convert_path + * + * FUNCTION: Convert a fuse relative path into one that contains + * the device_name. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +static char *cusfs_fuse_convert_path(const char *path) +{ + char *disk_pathname; + + disk_pathname = malloc(strlen(path) + strlen(device_name) + 1); + + if (disk_pathname == NULL) { + + return NULL; + } + + sprintf(disk_pathname,"%s%s%s",device_name, + CUSFS_DISK_NAME_DELIMINATOR,path); + + fprintf(stderr,"disk_pathname = %s\n",disk_pathname); + return disk_pathname; +} + +/* + * NAME: cusfs_fuse_getattr + * + * FUNCTION: Get statistics of a file + * Similar to stat(). The 'st_dev' and 'st_blksize' fields are + * ignored. The 'st_ino' field is ignored except if the 'use_ino' + * mount option is given. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_getattr(const char *pathname, struct stat *stats) +{ + int rc = 0; + char *disk_pathname; +#ifdef _AIX + struct stat64 stats64; + + + + bzero(&stats64,sizeof(stats64)); +#endif + + + disk_pathname = cusfs_fuse_convert_path(pathname); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + +#ifdef _AIX + rc = cusfs_stat64(disk_pathname,&stats64); + + stats->st_ino = stats64.st_ino; + stats->st_mode = stats64.st_mode; + stats->st_nlink = stats64.st_nlink; + stats->st_uid = stats64.st_uid; + stats->st_gid = stats64.st_gid; + stats->st_atime = stats64.st_atime; + stats->st_ctime = stats64.st_ctime; + stats->st_mtime = stats64.st_mtime; + stats->st_blksize = stats64.st_blksize; + stats->st_blocks = stats64.st_blocks; +#ifndef __64BIT__ + + stats->st_size = (int)stats64.st_size; +#else + stats->st_ssize = (int)stats64.st_size; +#endif + +#else + rc = cusfs_stat(disk_pathname,stats); +#endif + + free(disk_pathname); + + + fprintf(stderr,"getattr: rc = %d, errno = %d\n",rc,errno); + + if (rc) { + + rc = -errno; + } + return rc; +} + + +/* + * NAME: cusfs_fuse_mkdir + * + * FUNCTION: Make a directory + * + * + * NOTE: The mode argument may not have the type specification + * bits set, i.e. S_ISDIR(mode) can be false. To obtain the + * correct directory type bits use mode|S_IFDIR + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_mkdir(const char *pathname, mode_t mode) +{ + int rc = 0; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(pathname); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + rc = cusfs_mkdir(disk_pathname,mode); + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + + + +/* + * NAME: cusfs_fuse_rmdir + * + * FUNCTION: Remove a directory + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_rmdir(const char *pathname) +{ + int rc = 0; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(pathname); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + rc = cusfs_rmdir(disk_pathname); + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + + +/* + * NAME: cusfs_fuse_unlink + * + * FUNCTION: Remove a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_unlink(const char *pathname) +{ + int rc = 0; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(pathname); + + if (disk_pathname == NULL) { + + return ENOMEM; + } + + rc = cusfs_unlink(disk_pathname); + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + +/* + * NAME: cusfs_fuse_rename + * + * FUNCTION: Rename a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_rename(const char *from_pathname, const char *to_pathname) +{ + int rc = 0; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(from_pathname); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + rc = cusfs_rename(disk_pathname,(char *)to_pathname); + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + +/* + * NAME: cusfs_fuse_link + * + * FUNCTION: Sym link a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_symlink(const char *from_pathname, const char *to_pathname) +{ + int rc = 0; + uid_t uid; + gid_t gid; + char *disk_pathname; + mode_t mode = (S_IRWXU|S_IRWXG|S_IRWXO); + + + disk_pathname = cusfs_fuse_convert_path(from_pathname); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + uid = getuid(); + + gid = getgid(); + + rc = cusfs_create_link(disk_pathname,(char *)to_pathname,mode,uid,gid,CFLSH_USFS_LINK_SYM_FLAG); + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + + + +/* + * NAME: cusfs_fuse_link + * + * FUNCTION: Link a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_link(const char *from_pathname, const char *to_pathname) +{ + int rc = 0; + uid_t uid; + gid_t gid; + char *disk_pathname; + mode_t mode = (S_IRWXU|S_IRWXG|S_IRWXO); + + + disk_pathname = cusfs_fuse_convert_path(from_pathname); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + uid = getuid(); + + gid = getgid(); + + rc = cusfs_create_link(disk_pathname,(char *)to_pathname,mode,uid,gid,0); + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + + +/* + * NAME: cusfs_fuse_chmod + * + * FUNCTION: Change mode of a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_chmod(const char *path, mode_t mode) +{ + int rc = 0; + int fd; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(path); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + + fd = cusfs_open(disk_pathname,O_RDWR,0); + + if (fd < 0) { + + fprintf(stderr,"Open failed with errno = %d\n",errno); + + free(disk_pathname); + return -ENOMEM; + + } + + rc = cusfs_fchmod(fd,mode); + + if (cusfs_close(fd)) { + + + fprintf(stderr,"Close failed with errno = %d\n",errno); + } + + + free(disk_pathname); + + if (rc) { + + rc = -errno; + } + return rc; +} + + +/* + * NAME: cusfs_fuse_chown + * + * FUNCTION: Change owner/group of a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_chown(const char *path, uid_t uid, gid_t gid) +{ + int rc = 0; + int fd; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(path); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + + fd = cusfs_open(disk_pathname,O_RDWR,0); + + if (fd < 0) { + + fprintf(stderr,"Open failed with errno = %d\n",errno); + + free(disk_pathname); + return -errno; + + } + + rc = cusfs_fchown(fd,uid,gid); + + if (cusfs_close(fd)) { + + + fprintf(stderr,"Close failed with errno = %d\n",errno); + } + + + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + +/* + * NAME: cusfs_fuse_truncate + * + * FUNCTION: Change file size + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_truncate(const char *path, off_t file_size) +{ + int rc = 0; + int fd; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(path); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + + fd = cusfs_open(disk_pathname,O_RDWR,0); + + if (fd < 0) { + + fprintf(stderr,"Open failed with errno = %d\n",errno); + + free(disk_pathname); + return -errno; + + } + + rc = cusfs_ftruncate(fd,file_size); + + if (cusfs_close(fd)) { + + + fprintf(stderr,"Close failed with errno = %d\n",errno); + } + + + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + + +/* + * NAME: cusfs_fuse_utime + * + * FUNCTION: Change time of file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_utime(const char *path, struct utimbuf *timebuf) +{ + int rc = 0; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(path); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + + rc = cusfs_utime(disk_pathname,timebuf); + + + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + +#ifdef _NOT_YET + +/* + * NAME: cusfs_fuse_utimens + * + * FUNCTION: Change time of a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_utimes(const char *path, const struct timespec tv[2]) +{ + int rc = 0; + int fd; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(path); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + + fd = cusfs_open(disk_pathname,O_RDWR,0); + + if (fd < 0) { + + fprintf(stderr,"Open failed with errno = %d\n",errno); + + free(disk_pathname); + return -errno; + + } + + rc = cusfs_futimens(fd,tv); + + if (cusfs_close(fd)) { + + + fprintf(stderr,"Close failed with errno = %d\n",errno); + } + + + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + +#endif + + +/* + * NAME: cusfs_fuse_ftruncate + * + * FUNCTION: Change file size + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_ftruncate(const char *path, off_t file_size,struct fuse_file_info *finfo) +{ + + int fd; + int rc = 0; + + if (finfo == NULL) { + errno = EINVAL; + return -1; + } + + fd = finfo->fh; + + rc = cusfs_ftruncate(fd,file_size); + if (rc) { + + rc = -errno; + } + + return rc; +} + + +/* + * NAME: cusfs_fuse_open + * + * FUNCTION: Open a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_open(const char *path, struct fuse_file_info *finfo) +{ + int rc = 0; + int fd; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(path); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + if (finfo == NULL) { + + return -EINVAL; + } + + + fd = cusfs_open(disk_pathname,O_RDWR,0); + + if (fd < 0) { + + fprintf(stderr,"Open failed %s with errno = %d\n", + disk_pathname,errno); + rc = -errno; + + } + + finfo->fh = fd; + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + + return rc; +} + +/* + * NAME: cusfs_fuse_create + * + * FUNCTION: Open a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_create(const char *path, mode_t mode,struct fuse_file_info *finfo) +{ + int rc = 0; + int fd; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(path); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + if (finfo == NULL) { + + return -EINVAL; + } + + + fd = cusfs_creat(disk_pathname,mode); + + if (fd < 0) { + + fprintf(stderr,"Creat failed %s with errno = %d\n", + disk_pathname,errno); + rc = -errno; + + } + + finfo->fh = fd; + + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + + +/* + * NAME: cusfs_fuse_read + * + * FUNCTION: read a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_read(const char *path, char *buffer, size_t nbytes, off_t offset, + struct fuse_file_info *finfo) +{ + int fd; + int rc = 0; + + if (finfo == NULL) { + + return -EINVAL; + } + + fd = finfo->fh; + + rc = cusfs_lseek(fd,offset,0); + + if (rc != offset) { + + return -errno; + } + + rc = cusfs_read(fd,buffer,nbytes); + if (rc < 0) { + + rc = -errno; + } + + return rc; +} + +/* + * NAME: cusfs_fuse_write + * + * FUNCTION: read a file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_write(const char *path, const char *buffer, size_t nbytes, off_t offset, + struct fuse_file_info *finfo) +{ + int fd; + int rc; + + if (finfo == NULL) { + errno = EINVAL; + return -1; + } + + fd = finfo->fh; + + rc = cusfs_lseek(fd,offset,0); + + if (rc != offset) { + + return -errno; + } + + rc = cusfs_write(fd,(char *)buffer,nbytes); + if (rc < 0) { + + rc = -errno; + } + + return rc; +} + + +/* + * NAME: cusfs_fuse_statfs + * + * FUNCTION: Gets stats for a filesystem. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_statfs(const char *path, struct statvfs *statvfs) +{ + int rc; + struct statfs statfs; + + //? add statvfs to library to do this directly + + if (statvfs == NULL) { + errno = EINVAL; + return -1; + } + + bzero(&statfs,sizeof(statfs)); + + bzero(statvfs,sizeof(*statvfs)); + + rc = cusfs_statfs(device_name,&statfs); + + if (rc == 0) { + + statvfs->f_blocks = statfs.f_blocks; + statvfs->f_bfree = statfs.f_bfree; + statvfs->f_bavail = statfs.f_bavail; + statvfs->f_blocks = statfs.f_blocks; + statvfs->f_files = statfs.f_files; + statvfs->f_ffree = statfs.f_ffree; + statvfs->f_favail = statvfs->f_ffree; + + } + + + if (rc) { + + rc = -errno; + } + return rc; +} + + +/* + * NAME: cusfs_fuse_flush + * + * FUNCTION: flush data for a file + * + * NOTES: Currently this just does the same thing as fsync. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_flush(const char *path, struct fuse_file_info *finfo) +{ + int fd; + int rc = 0; + + if (finfo == NULL) { + + return -EINVAL; + } + + fd = finfo->fh; + + rc = cusfs_fsync(fd); + + if (rc) { + + rc = -errno; + } + return rc; +} + +/* + * NAME: cusfs_fuse_release + * + * FUNCTION: release(close) an open file + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_release(const char *path, struct fuse_file_info *finfo) +{ + int fd; + int rc = 0; + + if (finfo == NULL) { + + return -EINVAL; + } + + fd = finfo->fh; + + rc = cusfs_close(fd); + + if (rc) { + + rc = -errno; + } + return rc; +} + + +/* + * NAME: cusfs_fuse_fsync + * + * FUNCTION: Sync a file's contents + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_fsync(const char *path, int data_sync,struct fuse_file_info *finfo) +{ + int fd; + int rc = 0; + + if (finfo == NULL) { + + return -EINVAL; + } + + fd = finfo->fh; + + rc = cusfs_fsync(fd); + + if (rc) { + + rc = -errno; + } + return rc; +} + + + +/* + * NAME: cusfs_fuse_opendir + * + * FUNCTION: Open a directory + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_opendir(const char *path, struct fuse_file_info *finfo) +{ + int rc = 0; +#ifdef _AIX + DIR64 *directory; +#else + DIR *directory; +#endif + char *disk_pathname; + + + if (finfo == NULL) { + + return -EINVAL; + } + + disk_pathname = cusfs_fuse_convert_path(path); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + +#ifdef _AIX + directory = cusfs_opendir64(disk_pathname); + +#else + directory = cusfs_opendir(disk_pathname); +#endif + + if (directory == NULL) { + + fprintf(stderr,"opendir returned null pointer with errno = %d\n",errno); + rc = -errno; + } + + finfo->fh = (uint64_t)directory; + + + free(disk_pathname); + if (rc) { + + rc = -errno; + } + return rc; +} + + +/* + * NAME: cusfs_fuse_releasedir + * + * FUNCTION: Close a directory + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_releasedir(const char *path, + struct fuse_file_info *finfo) +{ + int rc = 0; +#ifdef _AIX + DIR64 *directory; +#else + DIR *directory; +#endif + + if (finfo == NULL) { + errno = EINVAL; + return -1; + } + +#ifdef _AIX + directory = (DIR64 *) finfo->fh; + + if (directory) { + rc= cusfs_closedir64(directory); + } + +#else + directory = (DIR *) finfo->fh; + if (directory) { + rc = cusfs_closedir(directory); + } +#endif + + if (rc) { + + rc = -errno; + } + return rc; +} + + + + +/* + * NAME: cusfs_fuse_readdir + * + * FUNCTION: Read a directory entry + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_readdir(const char *path, void *buf, + fuse_fill_dir_t filler, off_t offset, + struct fuse_file_info *finfo) +{ + int rc = 0; + char stat_name[PATH_MAX]; +#ifdef _AIX + DIR64 *directory; + struct dirent64 file; + struct dirent64 *result = &file; + struct stat64 stats64; +#else + DIR *directory; + struct dirent file; + struct dirent *result = &file; +#endif + struct stat stats; + off_t nextoff; + + + if (finfo == NULL) { + + return -EINVAL; + } + +#ifdef _AIX + directory = (DIR64 *) finfo->fh; +#else + directory = (DIR *) finfo->fh; +#endif + + while (TRUE) { +#ifdef _AIX + rc = cusfs_readdir64_r(directory,&file,&result); + nextoff = file.d_offset; + bzero(&stats64,sizeof(stats64)); +#else + rc = cusfs_readdir_r(directory,&file,&result); + nextoff = file.d_off; +#endif + + if (rc) { + + + return -errno; + } + + if (result == NULL) { + + return 0; + } + + bzero(&stats,sizeof(stats)); + + sprintf(stat_name,"%s%s%s%s",device_name, + CUSFS_DISK_NAME_DELIMINATOR,path,file.d_name); + + +#ifdef _AIX + rc = cusfs_stat64(stat_name,&stats64); + + stats.st_ino = stats64.st_ino; + stats.st_mode = stats64.st_mode; + stats.st_nlink = stats64.st_nlink; + stats.st_uid = stats64.st_uid; + stats.st_gid = stats64.st_gid; + stats.st_atime = stats64.st_atime; + stats.st_ctime = stats64.st_ctime; + stats.st_mtime = stats64.st_mtime; + stats.st_blksize = stats64.st_blksize; + stats.st_blocks = stats64.st_blocks; +#ifndef __64BIT__ + + stats.st_size = (int)stats64.st_size; +#else + stats.st_ssize = (int)stats64.st_size; +#endif +#else + rc = cusfs_stat(stat_name,&stats); +#endif + + if (rc) { + + return -errno; + } + + //?? For AIX stat64 is not correct here. + + if (filler(buf, file.d_name, &stats, nextoff)) { + + /* + * Buffer is full + */ + break; + } + + + } + + + return 0; +} + + +/* + * NAME: cusfs_fuse_readlink + * + * FUNCTION: Read symbollic link filename + * + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_readlink(const char *pathname, char *buffer, size_t buffer_size) +{ + int rc = 0; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(pathname); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + rc = cusfs_readlink(disk_pathname,buffer,buffer_size); + + + free(disk_pathname); + + + if (rc) { + + rc = -errno; + } + return rc; +} + + +/* + * NAME: cusfs_fuse_access + * + * FUNCTION: Determines accessibility of a file + * + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_access(const char *pathname, int mode) +{ + int rc = 0; + char *disk_pathname; + + + disk_pathname = cusfs_fuse_convert_path(pathname); + + if (disk_pathname == NULL) { + + return -ENOMEM; + } + + rc = cusfs_access(disk_pathname,mode); + + + free(disk_pathname); + + + fprintf(stderr,"access: rc = %d, errno = %d\n",rc,errno); + if (rc) { + + rc = -errno; + } + return rc; +} + + +/* + * NAME: cusfs_fuse_fgetattr + * + * FUNCTION: Get statistics of a file + * Similar to stat(). The 'st_dev' and 'st_blksize' fields are + * ignored. The 'st_ino' field is ignored except if the 'use_ino' + * mount option is given. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int cusfs_fuse_fgetattr(const char *pathname, struct stat *stats, + struct fuse_file_info *finfo) +{ + int fd; + int rc = 0; + + if (finfo == NULL) { + errno = EINVAL; + return -1; + } + + fd = finfo->fh; + + + rc = cusfs_fstat(fd,stats); + + fprintf(stderr,"access: rc = %d, errno = %d\n",rc,errno); + + if (rc) { + + rc = -errno; + } + + return rc; +} + + +/************************************************************************/ +/* Fuse operations supported */ +/************************************************************************/ + + +static struct fuse_operations cflsh_ufs_oper = { + .getattr = cusfs_fuse_getattr, + .mkdir = cusfs_fuse_mkdir, + .unlink = cusfs_fuse_unlink, + .rmdir = cusfs_fuse_rmdir, + .symlink = cusfs_fuse_link, + .rename = cusfs_fuse_rename, + .link = cusfs_fuse_link, + .chmod = cusfs_fuse_chmod, + .chown = cusfs_fuse_chown, + .truncate = cusfs_fuse_truncate, + .utime = cusfs_fuse_utime, + .open = cusfs_fuse_open, + .read = cusfs_fuse_read, + .write = cusfs_fuse_write, + .statfs = cusfs_fuse_statfs, + .flush = cusfs_fuse_flush, + .release = cusfs_fuse_release, + .fsync = cusfs_fuse_fsync, + .opendir = cusfs_fuse_opendir, + .readdir = cusfs_fuse_readdir, + .releasedir = cusfs_fuse_releasedir, + .readlink = cusfs_fuse_readlink, + .access = cusfs_fuse_access, + .create = cusfs_fuse_create, + .ftruncate = cusfs_fuse_ftruncate, + .fgetattr = cusfs_fuse_fgetattr, +}; + +void cusfs_fuse_usage() +{ + + fprintf(stderr,"usage: cusfs_fuse [ fuse_options ] device_name mount_point\n"); + return; +} + +/* + * NAME: Main + * + * FUNCTION: fuse_main turns this into a daemon. + * + * NOTES: Last argument with no "-" is mount point. + * -o fsname=/dev/sg5:/ mount_point + * INPUTS: + * NONE + * + * RETURNS: + * 0 success, otherwise error. + * + */ +int main(int argc, char *argv[]) +{ + int rc = 0; + + cusfs_init(NULL,0); + + if (argc < 3) { + + cusfs_fuse_usage(); + return EINVAL; + } + + if ((argv[argc-2][0] == '-') || + (argv[argc-1][0] == '-')) { + + + cusfs_fuse_usage(); + return EINVAL; + } + + + device_name = strdup(argv[argc-2]); + + if (device_name == NULL) { + + + fprintf(stderr,"device_name is null"); + + cusfs_fuse_usage(); + + return EINVAL; + } + + /* + * Remove device name from the argument stream + */ + + argv[argc-2] = argv[argc-1]; + argv[argc-1] = NULL; + argc--; + + rc = fuse_main(argc, argv, &cflsh_ufs_oper, NULL); + + cusfs_term(NULL,0); + + + free(device_name); + + return rc; +} diff --git a/src/test/usfs/makefile b/src/test/usfs/makefile new file mode 100644 index 00000000..6fa5c14c --- /dev/null +++ b/src/test/usfs/makefile @@ -0,0 +1,71 @@ +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/test/makefile $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +UNAME=$(shell uname) + +# Where to find user code.(Relative path(from the makefile) preferred for portability) +ROOTPATH = ../../.. +USER_DIR = . + +TESTDIR = ${ROOTPATH}/obj/tests + +EXPFLAGS = -bexpall + +#test code != production code, so allow warnings here. +ALLOW_WARNINGS = yes + +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${ROOTPATH}/img +LIBPATHS = -L${ROOTPATH}/img +LINKLIBS = -lcflsh_block -lcflsh_usfs + + +BTESTS =cusfs +BIN_TESTS=$(addprefix ${TESTDIR}/, ${BTESTS}) + + +ifeq ($(UNAME),AIX) # AIX only +LINKLIBS+=-lpthreads + +BTESTS64 = $(addsuffix 64, ${BTESTS}) +BIN_TESTS64 = $(addprefix ${TESTDIR}/, ${BTESTS64}) +BITS = 64 + +else #Linux only +LINKLIBS+=-lpthread -ludev +endif + +CFLAGS += \ + -D__FVT__\ + -I$(ROOTPATH)/src/block \ + -I$(ROOTPATH)/src/common +CXXFLAGS+=$(CFLAGS) + +VPATH += \ + ${ROOTPATH}/src/kv \ + ${ROOTPATH}/src/kv/test \ + ${ROOTPATH}/src/block/test \ + +include ${ROOTPATH}/config.mk + diff --git a/src/test/vpd/flashgt_vpd_access.c b/src/test/vpd/flashgt_vpd_access.c new file mode 100644 index 00000000..62e07025 --- /dev/null +++ b/src/test/vpd/flashgt_vpd_access.c @@ -0,0 +1,529 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/test/vpd/flashgt_vpd_access.c $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +uint64_t mmio_read(char resource_file[],uint64_t offset); +uint64_t mmio_write(char resource_file[],uint64_t offset,uint64_t val64); +uint64_t query_i2c_ready(char resource_file[]); +uint64_t query_i2c_error(char resource_file[]); +uint64_t query_i2c_readdata_val(char resource_file[]); +uint64_t poll_vpd_read_done(int basecfg, int cfgaddr); +uint64_t poll_vpd_write_done(int basecfg, int cfgaddr); + +int main (int argc, char *argv[]) +{ + int priv1,priv2; + int dat, dif, edat; + int CFG; + int RBF; + int VPDRBF; + time_t ct, lt, st, et, eet, set, ept, spt, svt, evt; + int address, raddress; + char device_num[1024]; + char bar2[1024]; + + char rbf_file[1024]; + char cfg_file[1024]; + char vpdrbf_file[1024]; + + int print_cnt = 0; + uint32_t vpd[32]; + + if (argc < 3) { + printf("Usage: ./flashgt_vpd_access \nExample: ./flashgt_vpd_access 0000:01:00.0 0 vpd_FlashGT_050916.vpdrbf\n"); + return -1; + } + strcpy (device_num, argv[1]); + strcpy(bar2, "/sys/bus/pci/devices/"); + strcat(bar2, device_num); + strcat(bar2, "/resource2"); + + strcpy(cfg_file, "/sys/class/cxl/card"); + strcat(cfg_file, argv[2]); + strcat(cfg_file, "/device/config"); + + if ((CFG = open(cfg_file, O_RDWR)) < 0) { + printf("Can not open %s\n",cfg_file); + exit(-1); + } + + + strcpy(vpdrbf_file, argv[3]); + + //printf("Opening VPD FILE, %s\n",vpdrbf_file); + if ((VPDRBF = open(vpdrbf_file, O_RDWR)) < 0) { + printf("Can not open %s\n",vpdrbf_file); + exit(-1); + } + + int temp,vendor,device; + lseek(CFG, 0, SEEK_SET); + read(CFG, &temp, 4); + vendor = temp & 0xFFFF; + device = (temp >> 16) & 0xFFFF; + printf("Device ID: %04X\n", device); + printf("Vendor ID: %04X\n", vendor); + + if ( (vendor != 0x1014) || (( device != 0x0477) && (device != 0x04cf) && (device != 0x0601))) { + printf("Unknown Vendor or Device ID\n"); + exit(-1); + } + + int addr_reg, size_reg, cntl_reg, data_reg; + uint64_t mmio_i2c_addr = 0x0000000000000130; + lseek(CFG, 0x404, SEEK_SET); + read(CFG, &temp,4); + printf(" VSEC Length/VSEC Rev/VSEC ID: 0x%08X\n", temp); + addr_reg = 0x40; + data_reg = 0x44; + + // Set stdout to autoflush + setvbuf(stdout, NULL, _IONBF, 0); + +// ------------------------------------------------------------------------------- +// MMIOs to UCD +// ------------------------------------------------------------------------------- +uint64_t mmio_write_data = 0x00000000000000AB; +uint64_t mmio_read_data = 0x0; +uint8_t i2c_read_data = 0x00; +uint64_t i2c_access_clear = 0; +uint64_t i = 0; +int ready_status = 1; + + for(i = 0; i < 100; i++) { + ready_status = query_i2c_ready(bar2); + if(ready_status == 0) { + break; + } + else if((i == 99) && (ready_status != 0)) { + printf("I2C Failed to be ready\n"); + exit -1; + } + else { + printf("I2C Not ready yet...\r"); + } + } + + mmio_write_data = 0x00030001FAD00006; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + mmio_write_data = 0x00000001FAD00006; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + + ready_status = 1; + for(i = 0; i < 100; i++) { + ready_status = query_i2c_ready(bar2); + if(ready_status == 0) { + break; + } + else if((i == 99) && (ready_status != 0)) { + printf("I2C Failed to be ready\n"); + exit -1; + } + else { + printf("I2C Not ready yet...\r"); + } + } + + query_i2c_error(bar2); + + mmio_write_data = 0x00010001FAD10006; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + mmio_write_data = 0x00000001FAD10006; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + + ready_status = 1; + for(i = 0; i < 100; i++) { + mmio_read_data = query_i2c_readdata_val(bar2); + if((mmio_read_data & 0x8000000000000000) == 0x8000000000000000) { + ready_status = 0; + } + if(ready_status == 0) { + i2c_read_data = ((mmio_read_data & 0x000000000000FF00) >> 8); + break; + } + else if((i == 99) && (ready_status != 0)) { + printf("I2C Failed to be ready\n"); + exit -1; + } + else { + printf("I2C Not ready yet...\r"); + } + } + + query_i2c_error(bar2); + + //Final Check to make sure GPIO select took effect + if(i2c_read_data == 0x06) { + printf("Successfully targetting proper GPIO\n"); + } + else { + printf("ERROR: I2C to target EEPROM wp GPIO did not work\n"); + } + + mmio_write_data = 0x00030001FBD00003; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + mmio_write_data = 0x00000001FBD00003; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + + ready_status = 1; + for(i = 0; i < 100; i++) { + ready_status = query_i2c_ready(bar2); + if(ready_status == 0) { + break; + } + else if((i == 99) && (ready_status != 0)) { + printf("I2C Failed to be ready\n"); + exit -1; + } + else { + printf("I2C Not ready yet...\r"); + } + } + + query_i2c_error(bar2); + + mmio_write_data = 0x00010001FBD10003; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + mmio_write_data = 0x00000001FBD10003; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + + ready_status = 1; + for(i = 0; i < 100; i++) { + mmio_read_data = query_i2c_readdata_val(bar2); + if((mmio_read_data & 0x8000000000000000) == 0x8000000000000000) { + ready_status = 0; + } + if(ready_status == 0) { + i2c_read_data = ((mmio_read_data & 0x000000000000FF00) >> 8); + break; + } + else if((i == 99) && (ready_status != 0)) { + printf("I2C Failed to be ready\n"); + exit -1; + } + else { + printf("I2C Not ready yet...\r"); + } + } + + query_i2c_error(bar2); + + //Final Check to make sure GPIO took down the write protect + if(i2c_read_data == 0x03) { + printf("Successfully changed write protect GPIO value\n"); + } + else { + printf("ERROR: I2C to alter EEPROM wp value did not work\n"); + } + + mmio_write_data = 0x00030001FBD00002; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + mmio_write_data = 0x00000001FBD00002; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + + ready_status = 1; + for(i = 0; i < 100; i++) { + ready_status = query_i2c_ready(bar2); + if(ready_status == 0) { + break; + } + else if((i == 99) && (ready_status != 0)) { + printf("I2C Failed to be ready\n"); + exit -1; + } + else { + printf("I2C Not ready yet...\r"); + } + } + + query_i2c_error(bar2); + //lseek(CFG, cntl_reg, SEEK_SET); + mmio_write_data = 0x00010001FBD10002; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + mmio_write_data = 0x00000001FBD10002; + mmio_write(bar2,mmio_i2c_addr,mmio_write_data); + + ready_status = 1; + for(i = 0; i < 100; i++) { + mmio_read_data = query_i2c_readdata_val(bar2); + if((mmio_read_data & 0x8000000000000000) == 0x8000000000000000) { + ready_status = 0; + } + if(ready_status == 0) { + i2c_read_data = ((mmio_read_data & 0x000000000000FF00) >> 8); + break; + } + else if((i == 99) && (ready_status != 0)) { + printf("I2C Failed to be ready\n"); + exit -1; + } + else { + printf("I2C Not ready yet...\r"); + } + } + + query_i2c_error(bar2); + + //Final Check to make sure GPIO change is made temporary + if(i2c_read_data == 0x02) { + printf("Successfully made write protect change temporary\n"); + } + else { + printf("ERROR: I2C to make wp change temporary did not work. Please do not do further i2c operations to UCD\n"); + } + + //lseek(CFG, cntl_reg, SEEK_SET); + //write(CFG,&temp,4); + + +// ------------------------------------------------------------------------------- +// Read and Write VPD Segments +// ------------------------------------------------------------------------------- + + uint32_t vpd_data = 0x0; + uint32_t vpd_f_addr_struct = 0x00000000; + int rawbinaddr = 0; + int readvalid = 1; + + //Loop to read original vpd contents + printf("Current VPD contents are: \n"); + for(int j = 0; j < 64; j++) { + lseek(CFG, addr_reg, SEEK_SET); + write(CFG, &vpd_f_addr_struct,4); + + poll_vpd_read_done(CFG, addr_reg); + + lseek(CFG, data_reg, SEEK_SET); + read(CFG, &vpd_data,4); + vpd[j] = vpd_data; + //lseek(VPDRBF, rawbinaddr, SEEK_SET); + //write(VPDRBF, &vpd[j], 4); + printf("0x%08x\n",vpd_data); + vpd_f_addr_struct = vpd_f_addr_struct + 0x00040000; + } + + //Loop to write new vpd contents + printf("Writing in new VPD contents:\n"); + vpd_f_addr_struct = 0x80000000; + for(int j = 0; j < 64; j++) { + if (readvalid) + { + lseek(VPDRBF, rawbinaddr, SEEK_SET); + readvalid = read(VPDRBF, &vpd_data, 4); + } + else {vpd_data = 0xFFFFFFFF;} + + printf("0x%08x\n",vpd_data); + lseek(CFG, data_reg, SEEK_SET); + write(CFG, &vpd_data, 4); + lseek(CFG, addr_reg, SEEK_SET); + write(CFG, &vpd_f_addr_struct,4); + + poll_vpd_write_done(CFG, addr_reg); + + //lseek(CFG, data_reg, SEEK_SET); + //read(CFG, &vpd_data,4); + //vpd[j] = vpd_data; + //lseek(VPDRBF, rawbinaddr, SEEK_SET); + //write(VPDRBF, &vpd[j], 4); + + rawbinaddr = rawbinaddr + 4; + vpd_f_addr_struct = vpd_f_addr_struct + 0x00040000; + } + + close(VPDRBF); + close(CFG); + exit(0); +} + + + +uint64_t mmio_read (char resource_file[],uint64_t offset) +{ + int priv2; + struct stat sb; + void *memblk, *adr; + int type; + uint64_t val64; + + if ((priv2 = open(resource_file, O_RDWR)) < 0) { + printf("Can not open %s\n",resource_file); + exit(-1); + } + + fstat(priv2, &sb); + + memblk = mmap(NULL,sb.st_size, PROT_WRITE|PROT_READ, MAP_SHARED, priv2, 0); + if (memblk == MAP_FAILED) { + printf("Can not mmap %s\n",resource_file); + exit(-1); + } + + //offset = strtoul(argv[2], 0, 0); + + adr = memblk + (offset & (sb.st_size - 1)); + + //printf("\n"); + val64 = *((uint64_t *)adr); + //printf("Reading double : [%jx]= 0x%016jx\n",adr, val64); + + munmap(memblk,sb.st_size); + + //printf("\n"); + close(priv2); + val64 = ((val64>>56)&0xff) | ((val64>>40)&0xff00) | ((val64>>24)&0xff0000) | ((val64>>8)&0xff000000) | ((val64<<8)&0xff00000000) | ((val64<<24)&0xff0000000000) | ((val64<<40)&0xff000000000000) | ((val64<<56)&0xff00000000000000); + //printf("Reading double : [%jx]= 0x%016jx\n",adr, val64); + return val64; +} + +uint64_t mmio_write (char resource_file[],uint64_t offset, uint64_t val64) +{ + int priv2; + struct stat sb; + void *memblk, *adr; + int type; + + + if ((priv2 = open(resource_file, O_RDWR)) < 0) { + printf("Can not open %s\n",resource_file); + exit(-1); + } + + fstat(priv2, &sb); + + memblk = mmap(NULL,sb.st_size, PROT_WRITE|PROT_READ, MAP_SHARED, priv2, 0); + if (memblk == MAP_FAILED) { + printf("Can not mmap %s\n",resource_file); + exit(-1); + } + + + adr = memblk + (offset & (sb.st_size - 1)); + + //printf("\n"); +//printf("Writing double : [%jx]= 0x%016jx\n",adr, val64); + val64 = ((val64>>56)&0xff) | ((val64>>40)&0xff00) | ((val64>>24)&0xff0000) | ((val64>>8)&0xff000000) | ((val64<<8)&0xff00000000) | ((val64<<24)&0xff0000000000) | ((val64<<40)&0xff000000000000) | ((val64<<56)&0xff00000000000000); + *((uint64_t *)adr) = val64; + //printf("Writing double : [%jx]= 0x%016jx\n",adr, val64); + + munmap(memblk,sb.st_size); + close(priv2); + + //printf("\n"); + return 0; +} + +uint64_t query_i2c_ready(char resource_file[]) +{ + uint64_t offset = 0x0000000000000130; + int rv; + uint64_t read_data; + + read_data = mmio_read(resource_file, offset); + if((read_data & 0x8000000000000000) == 0x8000000000000000) { + printf("I2C controller is ready\n"); + return 0; + } + else return 1; +} + +uint64_t query_i2c_error(char resource_file[]) +{ + uint64_t offset = 0x0000000000000130; + int rv; + uint64_t read_data; + + read_data = mmio_read(resource_file, offset); + if((read_data & 0x2000000000000000) == 0x2000000000000000) { + printf("I2C controller is in error\n"); + exit -1; + } + else return 1; +} + +uint64_t query_i2c_readdata_val(char resource_file[]) +{ + uint64_t offset = 0x0000000000000130; + int rv; + uint64_t read_data; + uint8_t i2c_rdata = 0x00; + + read_data = mmio_read(resource_file, offset); + i2c_rdata = ((read_data & 0x000000000000FF00) >> 8); + if((read_data & 0x4000000000000000) == 0x4000000000000000) { + printf("I2C controller read data is valid\n"); + return read_data; + } + else return 1; +} + +uint64_t poll_vpd_read_done(int basecfg, int cfgaddr) { + int i = 0; + uint32_t temp; + + for(i = 0; i < 10000; i++) { + lseek(basecfg, cfgaddr, SEEK_SET); + read(basecfg,&temp,4); + if((temp & 0x80000000) == 0x80000000) { + return 0; + } + else if(i >= 9999) { + printf("ERROR: VPD Read is not finishing\n"); + printf("VPD reg value is %08x\n",temp); + exit -1; + } + } +} + +uint64_t poll_vpd_write_done(int basecfg, int cfgaddr) { + int i = 0; + uint32_t temp; + + for(i = 0; i < 1000000; i++) { + lseek(basecfg, cfgaddr, SEEK_SET); + read(basecfg,&temp,4); + if((temp & 0x80000000) == 0x00000000) { + return 0; + } + else if(i >= 999999) { + printf("ERROR: VPD Write is not finishing\n"); + exit -1; + } + } +} diff --git a/src/test/vpd/makefile b/src/test/vpd/makefile new file mode 100644 index 00000000..5d904f89 --- /dev/null +++ b/src/test/vpd/makefile @@ -0,0 +1,48 @@ +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/test/ffdc/makefile $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +UNAME=$(shell uname) + +# Where to find user code.(Relative path(from the makefile) preferred for portability) +ROOTPATH = ../../.. + +#test code != production code, so allow warnings here. +ALLOW_WARNINGS = yes + +include ${ROOTPATH}/config.mk + +#this application originates in surelock-afu/bringup/src and must be shipped for FFDC gathering +ifeq ($(UNAME),AIX) # need to have a valid ffdc mechanism for AIX... +else #Linux only + +test: ${TESTDIR}/mmio ${TESTDIR}/flashgt_vpd_access + +${TESTDIR}/mmio: + mkdir -p ${TESTDIR} + gcc -I . -lpthread -o ${TESTDIR}/mmio -O0 mmio.c + +${TESTDIR}/flashgt_vpd_access: + mkdir -p ${TESTDIR} + gcc -I . -o ${TESTDIR}/flashgt_vpd_access -std=c99 flashgt_vpd_access.c +endif diff --git a/src/test/vpd/mmio.c b/src/test/vpd/mmio.c new file mode 100644 index 00000000..81bbfb4d --- /dev/null +++ b/src/test/vpd/mmio.c @@ -0,0 +1,108 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/test/vpd/mmio.c $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main (int argc, char *argv[]) +{ + int priv2; + struct stat sb; + void *memblk, *adr; + off_t offset; + int type; + uint64_t val64; + uint32_t val32; + + if (argc < 3) { + printf("Usage: mmio [dat]\n\n"); + exit(-1); + } + + if ((priv2 = open(argv[3], O_RDWR)) < 0) { + printf("Can not open %s\n",argv[3]); + exit(-1); + } + + fstat(priv2, &sb); + + memblk = mmap(NULL,sb.st_size, PROT_WRITE|PROT_READ, MAP_SHARED, priv2, 0); + if (memblk == MAP_FAILED) { + printf("Can not mmap %s\n",argv[1]); + exit(-1); + } + + type = 0; + if (tolower(argv[1][0] == 'd')) { + type++; + } + if (argc == 5) { + type += 2; + } + + offset = strtoul(argv[2], 0, 0); + + adr = memblk + (offset & (sb.st_size - 1)); + + switch (type) { + case 0 : // Read word + val32 = *((uint32_t *)adr); + printf("Reading word : [%jx]= 0x%08x\n",offset, val32); + break; + case 1 : // Read double + val64 = *((uint64_t *)adr); + printf("Reading double : [%jx]= 0x%016lx\n",offset, val64); + break; + case 2 : // Write word + val32 = strtoul(argv[4], 0, 0); + *((uint32_t *)adr) = val32; + printf("Writing word : [%jx]= 0x%08x\n",offset, val32); + break; + case 3 : // Write double + val64 = strtoul(argv[4], 0, 0); + *((uint64_t *)adr) = val64; + printf("Writing double : [%jx]= 0x%016lx\n",offset, val64); + break; + other : // Error + printf("Invalid Parms\n"); + exit(-1); + break; + } + + munmap(memblk,sb.st_size); + + close(priv2); +} diff --git a/src/test/vpd/surelock_vpd2rbf.pl b/src/test/vpd/surelock_vpd2rbf.pl new file mode 100755 index 00000000..0fd2be57 --- /dev/null +++ b/src/test/vpd/surelock_vpd2rbf.pl @@ -0,0 +1,368 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/test/vpd/surelock_vpd2rbf.pl $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +# + use Fcntl; + use Getopt::Long; +## +# +# ------------------------------------------------------------------------------- +# Variables +# ------------------------------------------------------------------------------- +my @parms; # Parameters from .csv file +my $p; # Current Parameter +my $pc; # Current Parameter position +my $dt; # Data Type +my $dl; # Data Length +my $comment; # Current VPD Key Word is a comment +my $serial_num; # Replacement Serial Number +my $card_identifier; # Replacement Card Idetification String +my $filename; # Source VPD File +my $tempfile; # Temporary Binary File + +my $bv; # Binary Value +my $fs; # File Size +my $cs; # Checksum +my $tcs; # Temporary Checksum +my $vcs; # Checksum of VPD information +my $bfs; # Binary File Size +my $padb; # Number of pad bytes + +# ------------------------------------------------------------------------------- +# Defaults: +# ------------------------------------------------------------------------------- +my $sn = 0; # Current VPD Key Word is the Serial Number +my $prthelp = 0; # Print Help Menu +my $comp = 0; # Complete Flag +my $optOK = 0; # Options are OK + +# ------------------------------------------------------------------------------- +# Parse Options +# ------------------------------------------------------------------------------- + $optOK = GetOptions ( "f|file=s"=>\$filename, + "s|sn=s"=>\$serial_num, + "i|id=s"=>\$card_identifier, + "h|help!",\$prthelp + ); + + if ($ARGV[0]) { + print "\nUnknown Command Line Options:\n"; + foreach(@ARGV) { + print " $_\n"; + } + print "\n"; + $prthelp = 1; + } + + + if (!($optOK) | $prthelp) { + print "\n"; + print "Usage: vpd2rbf.pl [-h | --help] [-f | --file] \n"; + print " -f or --file : Comma Separated Value File Containing The VPD Key Words.\n"; + print " -s or --sn : Serial Number for the adapter (SN Key Word MUST be in .csv file. This value replaces the .csv contents.\n"; + print " -i or --id : Card Identification String (Default: \"Corsa-CAPI PCIe Adapter\").\n"; + print " -h or --help : Help - Print this message. \n"; + print " \n"; + print " Version 1.1 \n"; + die "\n"; + } + + print("\n"); + + if ($serial_num) { + print "Serial Number Provided : $serial_num\n"; + } + + if ($card_identifier) { + print "Card Identifier Provided : $card_identifier Length = $p\n"; + } else { + $card_identifier = "Corsa-CAPI PCIe Adapter"; + } + + print "The VPD source file is: $filename\n\n" if $filename; + + print "VPD File ==> $filename\n"; + open(VPD, "<", $filename) or die $!; + + $tempfile = substr($filename,0,index($filename,".")) . ".tempvpd"; + $vpdfile = substr($filename,0,index($filename,".")) . ".vpdrbf"; + + print "The VPD binary image is: $vpdfile\n"; + + if (-e $tempfile) { + print("ERROR ==> Temporary File $tempfile exist. Please remove / rename and rerun script.\n"); + exit; + } + + open(VPDTB, ">",$tempfile) or die $!; + binmode VPDTB; + + print("\n"); + #------------------------------------------------------------------------------- + # Phase 0: Read and parse the VPD CSV File. Write out a temp binary image. + # ------------------------------------------------------------------------------- + + while () { + chomp; + my @parms = split(/,/); + + $pc = 1; + $dt = 0; + $dl = 0; + $sn = 0; + $comment = 0; + + if ($comp == 0) { + foreach $parm (@parms) { + + #------------------------------------------------------------------------------- + # Skip comments and lines past the end of the VPD information + #------------------------------------------------------------------------------- + if ( ((substr($parm,0,2) eq "//") || (substr($parm,1,2) eq "//")) && ($pc == 1) ) { + $comment = 1; + if ( (substr($parm,3,3) eq "END") || (substr($parm,4,3) eq "END") || (substr($parm,2,3) eq "END") ) { + $comp = 1; + last; + } else { + last; + } + } + + #------------------------------------------------------------------------------- + # Parse parameters + #------------------------------------------------------------------------------- + # First Parameter (Key Word) + if ($pc == 1) { + if (substr($parm,0,1) eq '"') { + $p = substr($parm,1,2); + } else { + $p = $parm; + } + print ("| $p | "); + $bv = pack("a*",$p); + print (VPDTB "$bv"); + if ($p eq "SN") { + $sn = 1; + } else { + $sn = 0; + } + } + + # Second Parameter (Length) + if ($pc == 2) { + $dl = $parm; + $bv = pack("C",$dl); + print (VPDTB "$bv"); + print ("| $dl | "); + } + + # Third Parameter (Data Type) + if ($pc == 3) { + if (substr($parm,0,1) eq '"') { + $p = substr($parm,1,1); + } else { + $p = $parm; + } + if ($p eq "A") { $dt = 1; } + else { $dt = 0;} + } + + # Fourth Parameter (Key Word Information) + if ($pc == 4) { + if ($dt == 1) { + if (substr($parm,0,1) eq '"') { + $p = substr($parm,1,length($parm) - 2); + } else { + $p = $parm; + } + if (($serial_num) && ($sn == 1)) { + $p = $serial_num; + } + print ("| $p | "); + $bv = pack("a*",$p); + print (VPDTB "$bv"); + } else { + if (substr($parm,0,1) eq '"') { + $parm = substr($parm,1,length($parm)-2); + } + $cr = length($parm); + $cs = $parm; + while ($cr > 0) { # moved this to 0 from -1 to get rid of extra field at the back?? + $p = substr($cs,length($cs)-2); + print ("| $p | "); + $bv = pack("H*",$p); + print (VPDTB "$bv"); + $cr-=2; + $cs = substr($cs,0,length($cs)-2); + } + } + } + + $pc++; + } + } + + if (($comment == 1) || ($comp == 1)) { + $comment = 0; + } else { + print ("\n"); + } + } + + #------------------------------------------------------------------------------- + # Add Checksum Keyword. Checksum value will be added in next phase. + #------------------------------------------------------------------------------- + $bv = pack("a*","RV"); + print (VPDTB "$bv"); + + #$dl = 1; + #$bv = pack("C",$dl); + #print (VPDTB "$bv"); + + close(VPD); + close(VPDTB); + # ------------------------------------------------------------------------------- + # End Phase 0: Temp VPD bin file created. + # ------------------------------------------------------------------------------- + + # ------------------------------------------------------------------------------- + # Phase 1: Create final VPD image with tags and checksums. + # ------------------------------------------------------------------------------- + $fs = -s $tempfile; + + $cs = 0; + $vcs = 0; + + $bfs = 0; + + open(VPDTB, "<",$tempfile) or die $!; + binmode VPDTB; + open(VPDB, ">",$vpdfile) or die $!; + binmode VPDB; + + # ------------------------------------------------------------------------------- + # Write the Identifier String + # ------------------------------------------------------------------------------- + $p = 0x82; + $bv = pack("C",$p); + print (VPDB "$bv"); + $cs += unpack("C",$bv); + + $p = length($card_identifier); + $bv = pack("C",$p); + print (VPDB "$bv"); + $cs += unpack("C",$bv); + + $p = "00"; + $bv = pack("H*",$p); + print (VPDB "$bv"); + $cs += unpack("C",$bv); + + $p = $card_identifier; + for ($i=0; $i>8) % 256); + + # ------------------------------------------------------------------------------- + # Write the VPD Data + # ------------------------------------------------------------------------------- + + while ( read(VPDTB, my $bv, 1) ) { + print (VPDB "$bv"); + $cs += unpack("C",$bv); + $vcs += unpack("C",$bv); + } + + $dl = $padb + 1; + $bv = pack("C",$dl); + print (VPDB "$bv"); + $cs += unpack("C",$bv); + + $p = 256 - ($cs & 255); + #$p = 256 - ($vcs & 255); + $bv = pack("C",$p); + $cs += unpack("C",$bv); + print (VPDB "$bv"); + $tcs = $p; + + $dl = 0; + for ($i=0; $i<$padb; $i++) { + $bv = pack("C",$dl); + print (VPDB "$bv"); + } + + # ------------------------------------------------------------------------------- + # Write the End Tag + # ------------------------------------------------------------------------------- + $p = "78"; + $bv = pack("H*",$p); + print (VPDB "$bv"); + $cs += unpack("C",$bv); + + close(VPDTB); + close(VPDB); + + # ------------------------------------------------------------------------------- + # Check the Checksum + # ------------------------------------------------------------------------------- + open(VPDB, "<",$vpdfile) or die $!; + binmode VPDB; + + $cs = 0; + while ( read(VPDB, my $bd, 1) ) { + $cs += unpack("C",$bd); + } + + $cs -= 0x78; + $cs = $cs % 256; + print ("\n\nThe checksum for $vpdfile : $cs (expected 0)\n"); + + close(VPDB); + + system("rm $tempfile"); + + exit; diff --git a/src/test/vpd/vpd_FlashGT.csv b/src/test/vpd/vpd_FlashGT.csv new file mode 100755 index 00000000..c1af26ee --- /dev/null +++ b/src/test/vpd/vpd_FlashGT.csv @@ -0,0 +1,12 @@ +// VPD Key Word,Length,Type,Data,Comment, +// Comment Line -- FlashGT,,,,, +PN,7,A,01DH750,Adapter Part Number, +EC,6,A,P11003,Adapter EC Level, +FN,7,A,01DH749,IBM FRU, +SN,8,A,17800823,Adapter Serial Number, +V1,4,A,EJ1K,Feature Code - 2U FlashGT, +V2,4,A,58CD,IBM CCIN, +V3,4,A,0000,Configuration,# Use 0000 +V5,16,A,0000000000000000,WWPN 1,# Use 0000000000000000 +V6,16,A,0000000000000000,WWPN 2,# Use 0000000000000000 +// END,,,,, diff --git a/src/test/xlate.c b/src/test/xlate.c index 8e31bcf2..dd9330c2 100644 --- a/src/test/xlate.c +++ b/src/test/xlate.c @@ -591,11 +591,11 @@ void rw_cmp_buf(struct ctx *p_ctx, __u64 start_lba, __u64 stride) { send_readm(p_ctx, start_lba, stride); // sends NUM_CMDS reads sprintf(buf, "read.%d", pr_id); - read_fd = open(buf, O_RDWR|O_CREAT); + read_fd = open(buf, O_RDWR|O_CREAT, 0600); sprintf(buf, "write.%d", pr_id); - write_fd = open(buf, O_RDWR|O_CREAT); + write_fd = open(buf, O_RDWR|O_CREAT, 0600); sprintf(buf, "readm.%d", pr_id); - readm_fd = open(buf, O_RDWR|O_CREAT); + readm_fd = open(buf, O_RDWR|O_CREAT, 0600); write(read_fd, &p_ctx->rbuf[i][0], sizeof(p_ctx->rbuf[i])); write(write_fd, &p_ctx->wbuf[i][0], sizeof(p_ctx->wbuf[i])); @@ -721,7 +721,7 @@ main(int argc, char *argv[]) #else sprintf(ctx_file, "ctx.%d", pr_id); unlink(ctx_file); - ctx_fd = open(ctx_file, O_RDWR|O_CREAT); + ctx_fd = open(ctx_file, O_RDWR|O_CREAT, 0600); if (ctx_fd < 0) { fprintf(stderr, "open failed: file %s, errno %d", ctx_file, errno); exit(-1); diff --git a/src/usfs/Makefile.ade b/src/usfs/Makefile.ade new file mode 100644 index 00000000..3bf42f01 --- /dev/null +++ b/src/usfs/Makefile.ade @@ -0,0 +1,81 @@ +# @(#)86 1.4 src/bos/usr/ccs/lib/libcflsh_block/Makefile, sysxcflashblock, bos720, o2016_07A2 2/16/16 12:03:22 +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# bos720 src/bos/usr/ccs/lib/libcflsh_block/Makefile 1.4 +# +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +# COMPONENT_NAME: (syscflashusfs) +# +# FUNCTIONS: +# ADE Makefile +# +# ORIGINS: 27 +# + + +CFLAGS += -qcpluscmt #allow C++ style comments +CFLAGS += -Dinline=__inline #alloc inline functions +CFLAGS += -qlist -qsource #Create listings +#CFLAGS += -qattr=full #includes attribute listing for all identifiers + +# Information on: no explicit return values from non-void function; +# redundant unsigned comparisons; constants; conversions; inconsistent +# declarations; inconsistent enumerations; language-level effects; +# variable names truncated by the compiler. + +CFLAGS += -D__FULL_PROTO -DINFULLPROTO -qinfo=pro -qproto +CFLAGS += -qinfo=ret -qinfo=cmp -qinfo=cns -qinfo=cnv +CFLAGS += -qinfo=dcl -qinfo=enu -qinfo=lan -qinfo=tru +CFLAGS += -qinfo=eff -qinfo=ini -qinfo=inl -qinfo=ord +#CFLAGS += -qinfo=par +CFLAGS += -DTARGET_ARCH_PPC64BE #User POWER Big Endian +#CFLAGS += -D_MASTER_LOCK + + +#disable optimization +CC_OPT_LEVEL = + +VPATH +=./common +INCFLAGS += -I./common + +EXPORTS = -bE:libcflsh_usfs.exp + +SHARED_LIBRARIES = libcflsh_usfs.a + +SHARED_OFILES = \ + cflsh_usfs.o \ + cflsh_usfs_utils.o \ + cflsh_usfs_disk.o \ + cflsh_usfs_inode.o \ + cflsh_usfs_client.o \ + cflsh_usfs_wrapper.o + +SHARED64_OFILES = ${SHARED_OFILES:.o=.64o} + +LIBS = -lc -lpthreads -lcflsh_block + +EXPLIB_TARGETS = export_libcflsh_usfs.a + +ILIST = libcflsh_usfs.a +IDIR = /usr/lib/ + +.include <${RULES_MK}> diff --git a/src/usfs/cflsh_usfs.c b/src/usfs/cflsh_usfs.c new file mode 100644 index 00000000..ef94a10e --- /dev/null +++ b/src/usfs/cflsh_usfs.c @@ -0,0 +1,6710 @@ +// IBM_PROLOG_BEGIN_TAG +// This is an automatically generated prolog. +// +// $Source: src/usfs/cflsh_usfs.c $ +// +// IBM CONFIDENTIAL +// +// COPYRIGHT International Business Machines Corp. 2016 +// +// p1 +// +// Object Code Only (OCO) source materials +// Licensed Internal Code Source Materials +// IBM Surelock Licensed Internal Code +// +// The source code for this program is not published or other- +// wise divested of its trade secrets, irrespective of what has +// been deposited with the U.S. Copyright Office. +// +// Origin: 30 +// +// IBM_PROLOG_END_TAG */ +#ifdef _AIX +static char sccsid[] = "%Z%%M% %I% %W% %G% %U%"; +#endif + +/* + * COMPONENT_NAME: (sysxcflashusfs) CAPI Flash user space filesystem library + * + * FUNCTIONS: + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +#define CFLSH_USFS_FILENUM 0x0100 + +#include "cflsh_usfs_internal.h" +#include "cflsh_usfs_protos.h" +#include "cflsh_usfs_client.h" +#ifdef _AIX +#include "cflsh_usfs.h" +#include "cflsh_usfs_admin.h" +#else +#include +#include +#endif + + +#ifndef _AIX +#include +REVISION_TAGS(ufs); +#ifdef CFLASH_LITTLE_ENDIAN_HOST +#ident "$Debug: LITTLE_ENDIAN $" +#else +#ident "$Debug: BIG_ENDIAN $" +#endif +#ifdef _MASTER_LOCK +#ident "$Debug: MASTER_LOCK $" +#endif +#endif + +cusfs_global_t cusfs_global; + +char *cusfs_log_filename = NULL; /* Trace log filename */ + /* This traces internal */ + /* activities */ + +int cusfs_log_verbosity = 0; /* Verbosity of traces in */ + /* log. */ + +FILE *cusfs_logfp = NULL; /* File pointer for */ + /* trace log */ +FILE *cusfs_dumpfp = NULL; /* File pointer for */ + /* dumpfile */ + +int cusfs_dump_level; /* Run time level threshold */ + /* required to initiate */ + /* live dump. */ + +int cusfs_notify_log_level; /* Run time level threshold */ + /* notify/error logging */ + + + +int dump_sequence_num; /* Dump sequence number */ + + +pthread_mutex_t cusfs_log_lock = PTHREAD_MUTEX_INITIALIZER; + +pthread_mutex_t cflsh_usfs_init_lock = PTHREAD_MUTEX_INITIALIZER; + +uint32_t num_thread_logs = 0; /* Number of thread log files */ + +#ifndef _AIX +/* + * This __attribute constructor is not valid for the AIX xlc + * compiler. + */ +static void _cflsh_usfs_init(void) __attribute__((constructor)); +static void _cflsh_usfs_free(void) __attribute__((destructor)); +#else +static int cflsh_usfs_initialize = 0; +static int cflsh_usfs_init_finish = 0; +static int cflsh_usfs_term = 0; + +#endif + +/* + * NAME: _cflsh_usfs_init + * + * FUNCTION: Internal intializer/constructor + * for the CAPI flash block library. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * NONE + * + */ + +void _cflsh_usfs_init(void) +{ + + char *env_sigsegv = getenv("CFLSH_USFS_SIGSEGV_DUMP"); + char *env_sigusr1 = getenv("CFLSH_USFS_SIGUSR1_DUMP"); + char *env_dump_level = getenv("CFLSH_USFS_DUMP_LEVEL"); + char *env_notify_log_level = getenv("CFLSH_USFS_NOTIFY_LOG_LEVEL"); + int rc; + + /* + * Require that the inode structure fits evenly into a 4K + * disk block. + */ + + CFLSH_USFS_COMPILE_ASSERT((4096 % sizeof(cflsh_usfs_inode_t)) == 0); + + +#ifdef _LINUX_MTRACE + mtrace(); +#endif + + + CUSFS_RWLOCK_INIT(cusfs_global.global_lock); + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + + + rc = pthread_atfork(cusfs_prepare_fork,cusfs_parent_post_fork,cusfs_child_post_fork); + +#ifndef _AIX + if (rc) { + fprintf(stderr,"pthread_atfork failed rc = %d, errno = %d\n",rc,errno); + + } +#endif + + /* + * ?? Initialize global data structs + */ + + cusfs_setup_trace_files(FALSE); + + + if (!cusfs_valid_endianess()) { + + CUSFS_TRACE_LOG_FILE(1,"This program is compiled for different endianess then the host is is running"); + } + + CUSFS_TRACE_LOG_FILE(2,"cusfs_global = %p",&cusfs_global); + + if (env_sigsegv) { + + /* + * The caller has requested that on SIGSEGV + * signals (segmentation fault), that we dump + * data just prior the application crash (and + * potentially coredump). This will provide + * more data to aid in analyzing the core dump. + */ + + if (cusfs_setup_sigsev_dump()) { + CUSFS_TRACE_LOG_FILE(1,"failed to set up sigsev dump handler"); + } + + } + + if (env_sigusr1) { + + /* + * The caller has requested that on SIGUSR1 + * signals that we dump data. + */ + + if (cusfs_setup_sigusr1_dump()) { + CUSFS_TRACE_LOG_FILE(1,"failed to set up sigusr1 dump handler"); + } + + } + + if (env_dump_level) { + + if (cusfs_setup_dump_file()) { + CUSFS_TRACE_LOG_FILE(1,"failed to set up dump file"); + + } else { + + cusfs_dump_level = atoi(env_dump_level); + } + + } + + + if (env_notify_log_level) { + + cusfs_notify_log_level = atoi(env_notify_log_level); + + } + + cusfs_global.caller_pid = getpid(); + + /* + * Determine host type + */ + + + cusfs_global.os_type = cusfs_get_os_type(); + + cflsh_usfs_master_init(); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return; +} + + +/* + * NAME: _cflsh_usfs_free + * + * FUNCTION: Free library resources. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * NONE + * + */ + +void _cflsh_usfs_free(void) +{ + + cflsh_usfs_master_term(); + + if (cusfs_global.flags & CFLSH_G_SYSLOG) { + + closelog(); + + } + + + if (num_thread_logs) { + + free(cusfs_global.thread_logs); + } + +#ifdef _LINUX_MTRACE + muntrace(); +#endif + + + + if (cusfs_global.process_name) { + + free(cusfs_global.process_name); + } + + + CUSFS_TRACE_LOG_FILE(3,"\nLIBRARY STATISTICS ..."); + +#ifdef CFLASH_LITTLE_ENDIAN_HOST + CUSFS_TRACE_LOG_FILE(3,"Little Endian"); +#else + CUSFS_TRACE_LOG_FILE(3,"Big Endian"); +#endif + + CUSFS_TRACE_LOG_FILE(3,"cusfs_log_verbosity 0x%x",cusfs_log_verbosity); +#if !defined(__64BIT__) && defined(_AIX) + CUSFS_TRACE_LOG_FILE(3,"32-bit app support "); +#else + CUSFS_TRACE_LOG_FILE(3,"64-bit app support "); + +#endif + CUSFS_TRACE_LOG_FILE(3,"flags 0x%x",cusfs_global.flags); + CUSFS_TRACE_LOG_FILE(3,"next_cusfs_id 0x%llx",cusfs_global.next_cusfs_id); + CUSFS_TRACE_LOG_FILE(3,"num_active_cufs 0x%x",cusfs_global.num_active_cufs); + CUSFS_TRACE_LOG_FILE(3,"num_max_active_cufs 0x%x",cusfs_global.num_max_active_cufs); + CUSFS_TRACE_LOG_FILE(3,"num_bad_cusfs_ids 0x%x",cusfs_global.num_bad_cusfs_ids); + + return; + +} + + +#ifdef _AIX +/* + * NAME: cflsh_usfs_init + * + * FUNCTION: Call library initializing code. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * NONE + * + */ + +void cflsh_usfs_init(void) +{ + /* + * The first thread will invoke this routine, but + * but subsequent threads must not be allowed to + * proceed until the _cusfs_init has completed. + * + * NOTE: the use of both a lock and a fetch_and_or + * is overkill here. A better solution would be + * to find a way to use fetch_and_or and only using + * the lock while the _cusfs_init. + */ + + //pthread_mutex_lock(&cflsh_usfs_init_lock); + + if (fetch_and_or(&cflsh_usfs_initialize,1)) { + + + + /* + * If cusfs_initialize is set, then + * we have done all the cflash block code + * initialization. As result we can return + * now provided we ensure any thread that is + * doing initializatoin for this library + * has completed. + */ + + while (!cflsh_usfs_init_finish) { + + usleep(1); + + } + + return; + } + + /* + * We get here if the cflash_block code has not been + * initialized yet. + */ + + _cflsh_usfs_init(); + + fetch_and_or(&cflsh_usfs_init_finish,1); + + //pthread_mutex_unlock(&cflsh_usfs_init_lock); + return; +} + +#endif /* _AIX */ + +/* + * NAME: cusfs_init + * + * FUNCTION: Call library initializing code. + * + * + * INPUTS: + * NONE + * + * RETURNS: + * NONE + * + */ + +int cusfs_init(void *arg,uint64_t flags) +{ + + /* + * Today, this routine is a no-op, but it provides + * future expandability options should that ever be needed. + */ + + return 0; +} + + +/* + * NAME: cusfs_term + * + * FUNCTION: Free library resources. + * + * + * INPUTS: NONE + * + * + * RETURNS: NONE + * + */ + +int cusfs_term(void *arg,uint64_t flags) +{ +#ifdef _AIX + if (fetch_and_or(&cflsh_usfs_term,1)) { + + + + /* + * If cflsh_blk_term is set, then + * we have done all the cflash block code + * cleanup. As result we can return + * now. + */ + return 0; + } + + /* + * We get here if the cflsh UFS code has not been + * terminated yet. + */ + + _cflsh_usfs_free(); + +#endif /* AIX */ + return 0; +} + + + +/* + * NAME: cusfs_create_fs + * + * FUNCTION: Create a CAPI flash user space filesystem + * on a disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_create_fs(char *device_name, int flags) +{ + int rc = 0; + int local_flags = 0; + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_data_obj_t *file; + uid_t uid; + gid_t gid; + mode_t mode; + + + CUSFS_TRACE_LOG_FILE(5,"device_name = %s", + device_name); + if (device_name == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Null device_name"); + + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + cufs = cusfs_get_cufs(device_name,0); + + if (cufs == NULL) { + + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + if (flags & CUSFS_FORCE_CREATE_FS) { + local_flags = CFLSH_USFS_FORCE_SB_CREATE; + } + + + if (cusfs_create_superblock(cufs,local_flags)) { + + cusfs_release_cufs(cufs,0); + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + + if (cusfs_create_free_block_table(cufs,0)) { + + + cusfs_release_cufs(cufs,0); + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + cufs->superblock.flags |= CFLSH_USFS_SB_FT_VAL; + + if (cusfs_create_inode_table(cufs,0)) { + + + cusfs_release_cufs(cufs,0); + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + + cufs->superblock.flags |= CFLSH_USFS_SB_IT_VAL; + + if (cusfs_create_journal(cufs,0)) { + + + cusfs_release_cufs(cufs,0); + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + cufs->superblock.flags |= CFLSH_USFS_SB_JL_VAL; + + /* + * Create root directory + */ + + uid = getuid(); + + gid = getgid(); + + mode = S_IRWXU | S_IRWXG | S_IRWXO; + + file = cusfs_create_data_obj(cufs,"/",CFLSH_USFS_INODE_FILE_DIRECTORY,0,mode,uid,gid,NULL,CFLSH_USFS_ROOT_DIR); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to create root directory on disk %s", + cufs->device_name); + } + + cufs->superblock.flags |= CFLSH_USFS_SB_ROOT_DIR; + + if (cusfs_update_superblock(cufs,0)) { + + + cusfs_release_cufs(cufs,0); + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + return rc; +} + +/* + * NAME: cusfs_query_fs + * + * FUNCTION: Query informatin on filesystem + * for the specified disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_query_fs(char *device_name, struct cusfs_query_fs *query,int flags) +{ + int rc = 0; + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_super_block_t cusfs_super_block; + + CUSFS_TRACE_LOG_FILE(5,"device_name = %s", + device_name); + if (device_name == NULL) { + + + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + cufs = cusfs_get_cufs(device_name,0); + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (cufs == NULL) { + + + return -1; + } + + bzero(query,sizeof(*query)); + bzero(&cusfs_super_block,sizeof(cusfs_super_block)); + + rc = cusfs_query_superblock(cufs,&cusfs_super_block,0); + + if (!rc) { + + + if (cusfs_super_block.start_marker != CLFSH_USFS_SB_SM) { + + /* + * Not valid super block for this host + */ + + CUSFS_TRACE_LOG_FILE(1,"Not valid super block start_marker = 0x%llx", + cusfs_super_block.start_marker); + + + if (CFLASH_REV64(cusfs_super_block.start_marker) == CLFSH_USFS_SB_SM) { + + CUSFS_TRACE_LOG_FILE(1,"Actually superblock is valid for different endianess host", + cusfs_super_block.start_marker); + query->flags |= CUSFS_QRY_DIF_ENDIAN; + query->os_type = CFLASH_REV32(cusfs_super_block.os_type); + query->fs_block_size = CFLASH_REV64(cusfs_super_block.fs_block_size); + query->disk_block_size = CFLASH_REV64(cusfs_super_block.disk_block_size); + query->num_blocks = CFLASH_REV64(cusfs_super_block.num_blocks); + query->free_block_table_size = CFLASH_REV64(cusfs_super_block.free_block_table_size); + query->inode_table_size = CFLASH_REV64(cusfs_super_block.inode_table_size); + + + if (sizeof(time_t) == 8) { + + query->create_time = CFLASH_REV64(cusfs_super_block.create_time); + query->write_time = CFLASH_REV64(cusfs_super_block.write_time); + query->mount_time = CFLASH_REV64(cusfs_super_block.mount_time); + query->fsck_time = CFLASH_REV64(cusfs_super_block.fsck_time); + + + } else { + + query->create_time = CFLASH_REV32(cusfs_super_block.create_time); + query->write_time = CFLASH_REV32(cusfs_super_block.write_time); + query->mount_time = CFLASH_REV32(cusfs_super_block.mount_time); + query->fsck_time = CFLASH_REV32(cusfs_super_block.fsck_time); + + } + +#ifdef _AIX + rc = ECORRUPT; +#else + rc = EBADR; +#endif /* ! AIX */ + + + + + } else { + + rc = ENXIO; + + } + } else { + + + query->os_type = cusfs_super_block.os_type; + query->fs_block_size = cusfs_super_block.fs_block_size; + query->disk_block_size = cusfs_super_block.disk_block_size; + query->num_blocks = cusfs_super_block.num_blocks; + query->free_block_table_size = cusfs_super_block.free_block_table_size; + query->inode_table_size = cusfs_super_block.inode_table_size; + query->create_time = cusfs_super_block.create_time; + query->write_time = cusfs_super_block.write_time; + query->mount_time = cusfs_super_block.mount_time; + query->fsck_time = cusfs_super_block.fsck_time; + + } + + } + + cusfs_release_cufs(cufs,0); + + if (rc) { + errno = rc; + + rc = -1; + + } + return rc; +} + +/* + * NAME: cusfs_statfs + * + * FUNCTION: Return statfs informatin on filesystem + * for the specified disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: NULL for failure, otherwise success. + * + * + */ + +int cusfs_statfs(char *path, struct statfs *statfs) +{ + int rc = 0; + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_super_block_t cusfs_super_block; + uint64_t num_inodes; + uint64_t num_avail_inodes; + uint64_t num_free_blocks; + uint64_t num_avail_free_blocks; + char device_name[PATH_MAX]; + char *filename; + + + CUSFS_TRACE_LOG_FILE(5,"device_name = %s", + device_name); + + if (path == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"path is null"); + errno = EINVAL; + return -1; + } + + /* + * Copy path, since we're + * going to modify it. + */ + + strcpy(device_name,path); + + filename = strchr(device_name,CFLSH_USFS_DISK_DELIMINATOR); + + if (filename) { + + filename[0] = '\0'; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + cufs = cusfs_get_cufs(device_name,0); + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"could not find filesystem for device_name = %s", + device_name); + + + return -1; + } + + bzero(statfs,sizeof(*statfs)); + bzero(&cusfs_super_block,sizeof(cusfs_super_block)); + + rc = cusfs_query_superblock(cufs,&cusfs_super_block,0); + + if (!rc) { + + if (cusfs_super_block.start_marker != CLFSH_USFS_SB_SM) { + + /* + * Not valid super block for this host + */ + + CUSFS_TRACE_LOG_FILE(1,"Not valid super block start_marker = 0x%llx", + cusfs_super_block.start_marker); + + + cusfs_release_cufs(cufs,0); + errno = ENXIO; + return -1; + + } +#ifdef _AIX + statfs->f_version = cusfs_super_block.version; + statfs->f_name_max = PATH_MAX; +#endif + + //??statfs->f_type + statfs->f_bsize = cusfs_super_block.fs_block_size; + statfs->f_blocks = cusfs_super_block.num_blocks; //?? should this be in fs_block_size? + bcopy(&(cusfs_super_block.fs_unique_id.val1), + &(statfs->f_fsid), + sizeof(cusfs_super_block.fs_unique_id.val1)); + + } + + rc = cusfs_get_free_table_stats(cufs,&num_free_blocks,&num_avail_free_blocks,0); + + if (!rc) { + + statfs->f_bfree = num_free_blocks; + statfs->f_bavail = num_avail_free_blocks; + } + + + rc = cusfs_get_inode_stats(cufs,&num_inodes,&num_avail_inodes,0); + + + if (!rc) { + + statfs->f_files = num_inodes; + statfs->f_ffree = num_avail_inodes; + } + + + + cusfs_release_cufs(cufs,0); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + + +/* + * NAME: cusfs_statfs64 + * + * FUNCTION: Return statfs informatin on filesystem + * for the specified disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: NULL for failure, otherwise success. + * + * + */ + +int cusfs_statfs64(char *path, struct statfs64 *statfs) +{ + int rc = 0; + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_super_block_t cusfs_super_block; + uint64_t num_inodes; + uint64_t num_avail_inodes; + uint64_t num_free_blocks; + uint64_t num_avail_free_blocks; + char device_name[PATH_MAX]; + char *filename; + + if (device_name == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"device_name is null"); + errno = EINVAL; + return -1; + } + + /* + * Copy path, since we're + * going to modify it. + */ + + strcpy(device_name,path); + + filename = strchr(device_name,CFLSH_USFS_DISK_DELIMINATOR); + + if (filename) { + + filename[0] = '\0'; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + cufs = cusfs_get_cufs(device_name,0); + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"could not find filesystem for device_name = %s", + device_name); + + return -1; + } + + bzero(statfs,sizeof(*statfs)); + bzero(&cusfs_super_block,sizeof(cusfs_super_block)); + + rc = cusfs_query_superblock(cufs,&cusfs_super_block,0); + + if (!rc) { + + if (cusfs_super_block.start_marker != CLFSH_USFS_SB_SM) { + + /* + * Not valid super block for this host + */ + + CUSFS_TRACE_LOG_FILE(1,"Not valid super block start_marker = 0x%llx", + cusfs_super_block.start_marker); + + + cusfs_release_cufs(cufs,0); + errno = ENXIO; + return -1; + + } +#ifdef _AIX + statfs->f_version = cusfs_super_block.version; + statfs->f_name_max = PATH_MAX; +#endif + //??statfs->f_type + statfs->f_bsize = cusfs_super_block.fs_block_size; + statfs->f_blocks = cusfs_super_block.num_blocks; //?? should this be in fs_block_size? + bcopy(&(cusfs_super_block.fs_unique_id.val1), + &(statfs->f_fsid), + sizeof(cusfs_super_block.fs_unique_id.val1)); + + + } + + rc = cusfs_get_free_table_stats(cufs,&num_free_blocks,&num_avail_free_blocks,0); + + if (!rc) { + + statfs->f_bfree = num_free_blocks; + statfs->f_bavail = num_avail_free_blocks; + } + + + rc = cusfs_get_inode_stats(cufs,&num_inodes,&num_avail_inodes,0); + + + if (!rc) { + + statfs->f_files = num_inodes; + statfs->f_ffree = num_avail_inodes; + } + + + cusfs_release_cufs(cufs,0); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + + +/* + * NAME: cusfs_remove_fs + * + * FUNCTION: Remove a CAPI flash user space filesystem + * on a disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: NULL for failure, otherwise success. + * + * + */ + +int cusfs_remove_fs(char *device_name, int flags) +{ + int rc = 0; + cflsh_usfs_t *cufs = NULL; + + + CUSFS_TRACE_LOG_FILE(5,"device_name = %s", + device_name); + + if (device_name == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"device_name is null"); + + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + cufs = cusfs_find_fs_for_disk(device_name,0); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Did not find a filesystem for this disk %s", + device_name); + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + errno = EINVAL; + return -1; + + } + + + if (cusfs_remove_superblock(cufs,0)) { + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + +#if !defined(_MASTER_LOCK_CLIENT) && defined(_MASTER_LOCK) + + rc = cusfs_cleanup_sem(cufs->master_handle); + if ( rc == -1 ) + { + CUSFS_TRACE_LOG_FILE(1,"cleanup failed for file system for this disk %s", + device_name); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return rc; + } + +#endif + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + cusfs_release_cufs(cufs,0); + return rc; +} + + +/* + * NAME: cusfs_fsck + * + * FUNCTION: Run filesystem check and repair + * on a filesystem on the specified disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: NULL for failure, otherwise success. + * + * + */ + +int cusfs_fsck(char *device_name, int flags) +{ + int rc = 0; + cflsh_usfs_t *cufs = NULL; + + + CUSFS_TRACE_LOG_FILE(5,"device_name = %s", + device_name); + + if (device_name == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"device_name is null"); + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + cufs = cusfs_find_fs_for_disk(device_name,0); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Did not find a filesystem for this disk %s", + device_name); + + errno = EINVAL; + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return rc; + + } + + //?? + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + cusfs_release_cufs(cufs,0); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + + + +/* + * NAME: cusfs_mkdir + * + * FUNCTION: Make a directory on in CAPI flash user space filesystem + * on a disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_mkdir(char *path,mode_t mode_flags) +{ + int rc = 0; + cflsh_usfs_t *cufs = NULL; + char device_name[PATH_MAX]; + char filename2[PATH_MAX]; + char *filename; + char *parent_filename; + char *directory_name; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_data_obj_t *parent_dir; + uid_t uid; + gid_t gid; + + + + + CUSFS_TRACE_LOG_FILE(5,"pathxs = %s", + path); + if (path == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"path is null"); + errno = EINVAL; + return -1; + } + + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + file = cusfs_open_disk_filename(path,0); + + if (file) { + + errno = EEXIST; + + CUSFS_TRACE_LOG_FILE(1,"filename already exists %s",path); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + + } + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + /* + * File does not exist yet + * Need to determine if the file system + * exists. + */ + + strcpy(device_name,path); + + + filename = strchr(device_name,CFLSH_USFS_DISK_DELIMINATOR); + + if (filename == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Invalid disk_pathname %s", + device_name); + + errno = EINVAL; + return rc; + } + + filename[0] = '\0'; + filename++; + + cufs = cusfs_find_fs_for_disk(device_name,0); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Did not find a filesystem for this disk %s", + device_name); + + errno = EINVAL; + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return rc; + } + + + /* + * Get parent filename of this file + */ + + strcpy(filename2,filename); + + directory_name = strrchr(filename2,'/'); + + directory_name[0] = '\0'; + + directory_name++; + + if (!strcmp(filename2,"")) { + + parent_filename = "/"; + } else { + parent_filename = filename2; + } + + + if (directory_name == NULL) { + + errno = EINVAL; + return -1; + } + + parent_dir = cusfs_lookup_data_obj(cufs,parent_filename,0); + + if (parent_dir == NULL) { + + return -1; + } + + uid = getuid(); + + gid = getgid(); + + if (mode_flags == 0) { + /* + * If no mode_flags were specified, + * then use the mode flags of the parent directory + */ + + mode_flags = parent_dir->inode->mode_flags & (S_IRWXU|S_IRWXG|S_IRWXO); + + } + + CUSFS_LOCK(parent_dir->lock); + + file = cusfs_create_data_obj(cufs,directory_name,CFLSH_USFS_INODE_FILE_DIRECTORY, + parent_dir->inode->index,mode_flags,uid,gid,NULL,0); + + + if (file == NULL) { + + rc = errno; + } + + CUSFS_UNLOCK(parent_dir->lock); + + /* + * Put absolute path in + * file struct + */ + + strcpy(device_name,path); + + + filename = strchr(device_name,CFLSH_USFS_DISK_DELIMINATOR); + + if (filename) { + + strcpy(file->filename,filename); + } + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + if (rc) { + + errno = rc; + + rc = -1; + } + return rc; +} + +/* + * NAME: cusfs_rmdir + * + * FUNCTION: Remove a diectory from CAPI flash user space filesystem + * on a disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_rmdir(char *pathname) +{ + int rc = 0; + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_data_obj_t *file; + + + CUSFS_TRACE_LOG_FILE(5,"pathname = %s", + pathname); + if (pathname == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"pathname is null"); + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + file = cusfs_open_disk_filename(pathname,0); + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"could not find file for pathname = %s", + pathname); + errno = ENOENT; + return -1; + + } + + cufs = file->fs; + + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"could not find filesystem for pathname = %s", + pathname); + errno = EINVAL; + + return -1; + } + + if (file->inode->type != CFLSH_USFS_INODE_FILE_DIRECTORY) { + + CUSFS_TRACE_LOG_FILE(1,"pathname = %s is not a directory", + pathname); + errno = ENOTDIR; + + return -1; + + } + + rc = cusfs_free_data_obj(cufs,file,0); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + +/* + * NAME: CUSFS_GET_FILE_HASH + * + * FUNCTION: Find file from index in the hash table + * + * + * INPUTS: + * file index - file identifier + * + * RETURNS: + * NULL = No chunk found. + * pointer = chunk found. + * + */ + +inline cflsh_usfs_data_obj_t *CUSFS_GET_FILE_HASH(int file_index, int check_rdy) +{ + cflsh_usfs_data_obj_t *file = NULL; + cflsh_usfs_t *cufs; + + + file = cusfs_global.file_hash[file_index & (MAX_NUM_CUSFS_FILE_HASH - 1)]; + + + while (file) { + + if ( (ulong)file & CUSFS_BAD_ADDR_MASK ) { + + CUSFS_TRACE_LOG_FILE(1,"Corrupted file address = 0x%llx, index = 0x%x", + (uint64_t)file, (file_index & (MAX_NUM_CUSFS_FILE_HASH - 1))); + + cusfs_global.num_bad_file_ids++; + file = NULL; + + break; + + } + + + CUSFS_TRACE_LOG_FILE(9,"file address = 0x%llx, index = 0x%x, name = %s", + (uint64_t)file, file->index, file->filename); + + if (file->index == file_index) { + + /* + * Found the specified chunk. Let's + * validate it. + */ + + if (CFLSH_EYECATCH_FILE(file)) { + /* + * Invalid chunk + */ + + CUSFS_TRACE_LOG_FILE(1,"Invalid file, index = %d", + file_index); + file = NULL; + } +#ifdef _NOT_YET + else if ((!(file->flags & CFLSH_FILE_RDY)) && + (check_rdy)) { + + + /* + * This chunk is not ready + */ + + CUSFS_TRACE_LOG_FILE(1,"chunk not ready, file_index = %d", + file_index); + file = NULL; + } +#endif + + break; + } + + file = file->fnext; + + + } + + + if (file) { + + cufs = file->fs; + + if (cufs) { + + + + if ( (ulong)cufs & CUSFS_BAD_ADDR_MASK ) { + + CUSFS_TRACE_LOG_FILE(1,"Corrupted cufs address = 0x%llx for file %s", + (uint64_t)cufs, file->filename); + + cusfs_global.num_bad_cusfs_ids++; + + errno = EFAULT; + + file = NULL; + + } + + } else { + errno = EINVAL; + + return NULL; + } + } + + + return file; + +} + + +/* + * NAME: cusfs_opendir64/cusfs_opendir + * + * FUNCTION: Opens a a directory + * on a disk. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: NULL on error, otherwise success. + * + * + */ +#ifdef _AIX +DIR64 *cusfs_opendir64(char *directory_name) +#else +DIR *cusfs_opendir(char *directory_name) +#endif /* !_AIX */ +{ + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_data_obj_t *file; +#ifdef _AIX + DIR64 *directory; +#else + cflsh_usfs_DIR_t *directory; +#endif /* _AIX */ + + + if (directory_name == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"directory_name = NULL"); + + errno = EINVAL; + return NULL; + } + + CUSFS_TRACE_LOG_FILE(5,"directory_name = %s", + directory_name); + + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + file = cusfs_open_disk_filename(directory_name,0); + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"directory not found for directory_name = %s", + directory_name); + + errno = ENOENT; + return NULL; + + } + + cufs = file->fs; + + + if (cufs == NULL) { + + errno = EINVAL; + + return NULL; + } + + if (file->inode->type != CFLSH_USFS_INODE_FILE_DIRECTORY) { + + errno = ENOTDIR; + + return NULL; + + } + + + directory = malloc(sizeof(*directory)); + + + if (directory == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc directory failed for directory_name = %s on disk %s with errno = %d ", + directory_name,cufs->device_name,errno); + return NULL; + } + + bzero(directory,sizeof(*directory)); + + + directory->dd_fd = file->index; + + directory->dd_blksize = cufs->fs_block_size; + + directory->dd_loc = 0; + + directory->dd_curoff = 0; + + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + cusfs_add_file_to_hash(file,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + +#ifdef _AIX + return directory; +#else + return (DIR *)directory; +#endif +} + +#ifdef _AIX +/* + * NAME: cusfs_opendir + * + * FUNCTION: Opens a a directory + * on a disk. For AIX, this routine use + * the opendir64 routine. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: NULL on error, otherwise success. + * + * + */ +DIR *cusfs_opendir(char *directory_name) +{ + DIR64 *directory64; + DIR *directory; + + directory64 = cusfs_opendir64(directory_name); + + if (directory64 == NULL) { + + return NULL; + } + + directory = malloc(sizeof(*directory)); + + + if (directory == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc directory failed for directory_name = %s with errno = %d ", + directory_name,errno); + return NULL; + } + + bzero(directory,sizeof(*directory)); + + + directory->dd_fd = directory64->dd_fd; + + directory->dd_blksize = directory64->dd_blksize; + + directory->dd_loc = 0; + + directory->dd_curoff = 0; + + free(directory64); + + return (directory); + +} +#endif + +/* + * NAME: cusfs_closedir64/cusfs_closedir + * + * FUNCTION: Closes a a directory + * on a disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int cusfs_closedir64(DIR64 *directory) +#else +int cusfs_closedir(DIR *directory_linux) +#endif /* !_AIX */ +{ + + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_data_obj_t *file; + int fd; +#ifndef _AIX + cflsh_usfs_DIR_t *directory = (cflsh_usfs_DIR_t *)directory_linux; +#endif + + + if (directory == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"directory_name = NULL"); + errno = EBADF; + return 0; + } + + fd = directory->dd_fd; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"directory not found on disk"); + + errno = EINVAL; + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + errno = EINVAL; + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + + } + + CUSFS_TRACE_LOG_FILE(9,"directory %s on disk %s", + file->filename,cufs->device_name); + + if (file->inode->type != CFLSH_USFS_INODE_FILE_DIRECTORY) { + + + CUSFS_TRACE_LOG_FILE(1,"file %s not directory on disk %s", + file->filename,cufs->device_name); + errno = ENOTDIR; + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + return -1; + + } + + cusfs_remove_file_from_hash(file,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + free (directory); + + CUSFS_TRACE_LOG_FILE(5,"rc = 0"); + + return 0; +} + +#ifdef _AIX +/* + * NAME: cusfs_closedir + * + * FUNCTION: Closes a a directory + * on a disk. On AIX this routine + * relies on cusfs_closedir64. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +int cusfs_closedir(DIR *directory) +{ + DIR64 *directory64; + int rc = 0; + + directory64 = malloc(sizeof(*directory64)); + + + if (directory64 == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc directory64 failed with errno = %d ", + errno); + return NULL; + } + + bzero(directory64,sizeof(*directory64)); + + + directory64->dd_fd = directory->dd_fd; + + directory64->dd_blksize = directory->dd_blksize; + + rc = cusfs_closedir64(directory64); + + free(directory64); + + return rc; +} +#endif /* _AIX */ + +/* + * NAME: cusfs_readdir64_r/cusfs_readdir_r + * + * FUNCTION: Returns a pointer to the next directory entry. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int cusfs_readdir64_r(DIR64 *directory, struct dirent64 *entry,struct dirent64 **result) +#else +int cusfs_readdir_r(DIR *directory_linux, struct dirent *entry,struct dirent **result) +#endif +{ + int rc = 0; + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_data_obj_t *file; + cflash_offset_t offset; +#ifdef _AIX + int namelen = MIN((CFLSH_USFS_DIR_MAX_NAME-1),_D_NAME_MAX); +#else + cflsh_usfs_DIR_t *directory = (cflsh_usfs_DIR_t *)directory_linux; + int namelen = MIN((CFLSH_USFS_DIR_MAX_NAME - 1),256); /* Linux hard codes this 256 */ + /* size in dirent.h */ +#endif + int fd; + cflsh_usfs_directory_entry_t disk_directory_entry; + + if (directory == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"directory = NULL on disk"); + + errno = EBADF; + return 0; + } + + fd = directory->dd_fd; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"directory not found on disk"); + + errno = EINVAL; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"file system not found on disk"); + errno = EINVAL; + return -1; + + } + + if (file->inode->type != CFLSH_USFS_INODE_FILE_DIRECTORY) { + + CUSFS_TRACE_LOG_FILE(1,"file %s is not directory on disk %s", + file->filename,cufs->device_name); + + errno = ENOTDIR; + + return -1; + + } + + + offset = directory->dd_loc; + + CUSFS_TRACE_LOG_FILE(1,"directory %s with offset 0x%llx on disk %s", + file->filename,offset,cufs->device_name); + + + + if (cusfs_get_next_directory_entry(cufs,file,&disk_directory_entry,&offset,0)) { + + + CUSFS_TRACE_LOG_FILE(1,"No more directory entries in directory %s on disk %s", + file->filename,cufs->device_name); + *result = NULL; + return 0; + } + + + + /* + * Fill in entry with the current directory entry item + */ + + + if (!(disk_directory_entry.flags & CFLSH_USFS_DIR_VAL)) { + + + CUSFS_TRACE_LOG_FILE(1,"Invalid directory entry in %s on disk %s", + file->filename,cufs->device_name); + *result = NULL; + return 0; + } + + entry->d_ino = disk_directory_entry.inode_number; + + + + directory->dd_loc = offset; + directory->dd_curoff = offset; + + + entry->d_reclen = sizeof(disk_directory_entry); +#ifdef _AIX + entry->d_namlen = namelen; + entry->d_offset = offset; +#else + entry->d_off = offset; +#endif + + + strncpy(entry->d_name,disk_directory_entry.filename,namelen); + + entry->d_name[namelen] = '\0'; + + + //?? update atime on directory entry + + + *result = entry; + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d, offset = 0x%llx, dir->dd_loc = 0x%llx", + rc, errno,offset,directory->dd_loc); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d"); + return rc; +} + +/* + * NAME: cusfs_readdir64/cufs_readdir + * + * FUNCTION: Returns a pointer to the next directory entry. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +struct dirent64 *cusfs_readdir64(DIR64 *directory) +#else +struct dirent *cusfs_readdir(DIR *directory) +#endif /* !AIX */ +{ + int rc = 0; +#ifdef _AIX + struct dirent64 entry; + struct dirent64 *result = &entry; + + rc = cusfs_readdir64_r(directory,&entry,&result); +#else + + struct dirent entry; + struct dirent *result = &entry; + + rc = cusfs_readdir_r(directory,&entry,&result); + +#endif + + if ((rc) || + (result == NULL)) { + + return NULL; + } + + result = malloc(sizeof(*result)); + + + if (result) { + + *result = entry; + + } +#ifdef _AIX + CUSFS_TRACE_LOG_FILE(5,"result = %p, name = %s, offset = 0x%llx", + result,entry.d_name,entry.d_offset); +#else + CUSFS_TRACE_LOG_FILE(5,"result = %p, name = %s, offset = 0x%llx", + result,entry.d_name,entry.d_off); +#endif + + return (result); +} + +#ifdef _AIX +/* + * NAME: cusfs_readdir + * + * FUNCTION: Returns a pointer to the next directory entry. + * On AIX this is done with cusfs + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +struct dirent *cusfs_readdir(DIR *directory) +{ + + DIR64 *directory64; + struct dirent64 *entry64; + struct dirent *entry = NULL; + int rc = 0; + + directory64 = malloc(sizeof(*directory64)); + + + if (directory64 == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc directory64 failed with errno = %d ", + errno); + return NULL; + } + + bzero(directory64,sizeof(*directory64)); + + + directory64->dd_fd = directory->dd_fd; + + directory64->dd_blksize = directory->dd_blksize; + + directory64->dd_loc = directory->dd_loc; + + directory64->dd_curoff = directory->dd_curoff; + + entry64 = cusfs_readdir64(directory64); + + if (entry64 == NULL) { + + free(directory64); + return entry; + } + + entry = malloc(sizeof(*entry)); + + + if (entry == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc entry failed with errno = %d ", + errno); + free(directory64); + return entry; + } + + bzero(entry,sizeof(*entry)); + + entry->d_reclen = entry64->d_reclen; + + entry->d_namlen = entry64->d_namlen; + entry->d_offset = entry64->d_offset; + + strncpy(entry->d_name,entry64->d_name,entry64->d_namlen); + + directory->dd_loc = directory64->dd_loc; + + directory->dd_curoff = directory64->dd_curoff; + + free(directory64); + + CUSFS_TRACE_LOG_FILE(5,"entry = %p, name = %s, errno = %d, offset = 0x%llx", + entry,entry->d_name,entry->d_offset); + return entry; +} + +#endif /* _AIX */ + +/* + * NAME: cusfs_telldir64/cusfs_telldir + * + * FUNCTION: Returns the current location in the specfied directory. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +offset_t cusfs_telldir64(DIR64 *directory) +#else +off_t cusfs_telldir(DIR *directory_linux) +#endif +{ +#ifdef _AIX + offset_t rc = 0; +#else + off_t rc = 0; +#endif + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_data_obj_t *file; + int fd; +#ifndef _AIX + cflsh_usfs_DIR_t *directory = (cflsh_usfs_DIR_t *)directory_linux; +#endif + + if (directory == NULL) { + + errno = EBADF; + return -1; + } + + fd = directory->dd_fd; + + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + errno = EINVAL; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (file->inode->type != CFLSH_USFS_INODE_FILE_DIRECTORY) { + + errno = ENOTDIR; + + + return -1; + + } + + + rc = directory->dd_curoff; + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + + +/* + * NAME: cusfs_seekdir64/cusfs_seekdir + * + * FUNCTION: Sets the position of the next readdir64_r operation. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +void cusfs_seekdir64(DIR64 *directory, offset_t location) +#else +void cusfs_seekdir(DIR *directory_linux, off_t location) +#endif +{ + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_data_obj_t *file; + int fd; +#ifndef _AIX + cflsh_usfs_DIR_t *directory = (cflsh_usfs_DIR_t *)directory_linux; +#endif + + if (directory == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"directory = NULL on disk"); + errno = EBADF; + return; + } + + + fd = directory->dd_fd; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"directory not found on disk"); + + errno = EINVAL; + return; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"file system not found on disk"); + + errno = EINVAL; + return; + + } + + if (file->inode->type != CFLSH_USFS_INODE_FILE_DIRECTORY) { + + CUSFS_TRACE_LOG_FILE(1,"file %s is not directory on disk %s", + file->filename,cufs->device_name); + + errno = ENOTDIR; + + + return; + + } + + + directory->dd_curoff = location; + directory->dd_loc = location; + + + + return; +} + + +/* + * NAME: cusfs_create_file + * + * FUNCTION: Create new file in filesystem + * + * + * INPUTS: + * NONE + * + * RETURNS: Null failure, otherwise success + * + * + */ + +cflsh_usfs_data_obj_t *cusfs_create_file(cflsh_usfs_t *cufs, char *full_pathname, + mode_t mode_flags, uid_t uid, gid_t gid, int flags) +{ + + char filename[PATH_MAX]; + char *parent_filename; + char *end_parent_filename; + cflsh_usfs_data_obj_t *file = NULL; + cflsh_usfs_data_obj_t *parent_dir; + + + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"file system is null"); + + errno = EINVAL; + return file; + } + + /* + * ?? Do we have to handle disk_name:pathname here + */ + + if (full_pathname == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"full_pathaname is null on disk %s", + cufs->device_name); + errno = EINVAL; + return file; + } + + + /* + * Get parent filename of this file + */ + + strcpy(filename,full_pathname); + + end_parent_filename = strrchr(filename,'/'); + + end_parent_filename[0] = '\0'; + + end_parent_filename++; + + if (!strcmp(filename,"")) { + + parent_filename = "/"; + } else { + parent_filename = filename; + } + + parent_dir = cusfs_lookup_data_obj(cufs,parent_filename,0); + + if (parent_dir == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"parent_directory not found for %s on disk %s", + parent_filename,cufs->device_name); + + return file; + } + + + CUSFS_LOCK(parent_dir->lock); + + file = cusfs_create_data_obj(cufs,end_parent_filename,CFLSH_USFS_INODE_FILE_REGULAR, + parent_dir->inode->index,mode_flags,uid,gid,NULL,0); + + CUSFS_UNLOCK(parent_dir->lock); + + if (file == NULL) { + + return NULL; + } + + + strcpy(file->filename,full_pathname); + + CUSFS_TRACE_LOG_FILE(5,"Created file = %s with inode num = 0x%llx of type %d on disk %s, ", + full_pathname,file->inode->index,file->inode->type, cufs->device_name); + + return file; +} + + +/* + * NAME: cusfs_create_link + * + * FUNCTION: create a hard/sym link + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_create_link(char *orig_path, char *path, + mode_t mode_flags, uid_t uid, gid_t gid, + int flags) +{ + int rc = 0; + cflsh_usfs_t *cufs; + char parent_filename[PATH_MAX]; + char *end_parent_filename; + cflsh_usfs_data_obj_t *link = NULL; + cflsh_usfs_data_obj_t *file = NULL; + cflsh_usfs_data_obj_t *parent_dir; + + + + /* + * ?? Do we have to handle disk_name:pathname here + */ + + if (orig_path == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"orig_path is null"); + + errno = EINVAL; + return -1; + } + + + + if (path == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"path is null"); + + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + cufs = cusfs_find_fs_from_path(orig_path,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"could not find filesystem for orig_path = %s", + orig_path); + + errno = EINVAL; + return -1; + } + + /* + * Get old parent filename of this file + */ + + strcpy(parent_filename,orig_path); + + end_parent_filename = strrchr(path,'/'); + + end_parent_filename[0] = '\0'; + + + end_parent_filename++; + + parent_dir = cusfs_lookup_data_obj(cufs,path,0); + + if (parent_dir == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"parent_directory not found for %s on disk %s", + path,cufs->device_name); + + errno = EINVAL; + return -1; + } + + + if (flags & CFLSH_USFS_LINK_SYM_FLAG) { + + /* + * This is a symbolic link + */ + + + CUSFS_LOCK(parent_dir->lock); + + link = cusfs_create_data_obj(cufs,end_parent_filename,CFLSH_USFS_INODE_FILE_SYMLINK, + parent_dir->inode->index,mode_flags,uid,gid, + orig_path, + 0); + + CUSFS_UNLOCK(parent_dir->lock); + + + + } else { + + /* + * This is a hard link + * + * + * Get original file this hard link will reference + */ + + + file = cusfs_lookup_data_obj(cufs,orig_path,0); + + if (file == NULL) { + + free(parent_dir); + + return -1; + } + + file->data_buf_size = cufs->fs_block_size; + + file->data_buf = malloc(file->data_buf_size); + + + if (file->data_buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to to malloc file->data_bu of size 0x%x for file = %s on disk %s", + file->data_buf_size,file->filename,cufs->device_name); + + + cusfs_free_file(file); + free(parent_dir); + + return -1; + + } + + if (file->inode->type == CFLSH_USFS_INODE_FILE_DIRECTORY) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed because tried to hard link a directory %s on disk %s", + orig_path,cufs->device_name); + cusfs_free_file(file); + free(parent_dir); + + return -1; + } + + CUSFS_LOCK(file->lock); + + + CUSFS_LOCK(parent_dir->lock); + + link = cusfs_create_data_obj(cufs,end_parent_filename,CFLSH_USFS_INODE_FILE_REGULAR, + parent_dir->inode->index,mode_flags,uid,gid, + NULL,0); + + CUSFS_UNLOCK(parent_dir->lock); + + link = cusfs_alloc_file(cufs,end_parent_filename,0); + + if (link == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"malloc of file %s for failed for disk %s", + end_parent_filename,cufs->device_name); + CUSFS_UNLOCK(file->lock); + cusfs_free_file(file); + free(parent_dir); + + return -1; + } + + + CUSFS_LOCK(parent_dir->lock); + if (cusfs_add_directory_entry(cufs,parent_dir->inode->index,file->inode,end_parent_filename,0)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to add directory entry to parent inode 0x%llx for file = %s on disk %s", + parent_dir->inode->index,end_parent_filename,cufs->device_name); + + CUSFS_UNLOCK(file->lock); + cusfs_free_file(file); + free(parent_dir); + cusfs_free_file(link); + + return -1; + } + + CUSFS_UNLOCK(parent_dir->lock); + + /* + * Increment count on original inode + */ + + file->inode->num_links++; + + cusfs_update_inode(cufs,file->inode,0); + + + CUSFS_UNLOCK(file->lock); + + + } + + CUSFS_TRACE_LOG_FILE(5,"Created link = %s on disk %s", + link->filename,cufs->device_name); + cusfs_free_file(link); + + cusfs_free_file(file); + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + +/* + * NAME: cusfs_remove_file + * + * FUNCTION: Remove a file from CAPI flash user space filesystem + * on a disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_remove_file(char *pathname, int flags) +{ + int rc = 0; + cflsh_usfs_t *cufs; + cflsh_usfs_data_obj_t *file; + + + /* + * ?? Do we have to handle disk_name:pathname here + */ + + if (pathname == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"pathname is null"); + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + cufs = cusfs_find_fs_from_path(pathname,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"could not find filesystem for pathname = %s", + pathname); + + errno = EINVAL; + return -1; + } + + if (!strcmp(pathname,"/")) { + + CUSFS_TRACE_LOG_FILE(1,"Trying to remove root filesystem / failed for disk %s", + cufs->device_name); + errno = EINVAL; + return -1; + } + + file = cusfs_lookup_data_obj(cufs,pathname,0); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"file not found for %s on disk %s", + pathname,cufs->device_name); + + return -1; + } + + + if (file->inode->type == CFLSH_USFS_INODE_FILE_DIRECTORY) { + + CUSFS_TRACE_LOG_FILE(1,"file %s is a directory on disk %s", + file->filename,cufs->device_name); + errno = EISDIR; + return -1; + + } + + rc = cusfs_free_data_obj(cufs,file,0); + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + +/* + * NAME: cusfs_list_files + * + * FUNCTION: List files from a directory + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_list_files(char *pathname, int flags) +{ + int rc = 0; + cflsh_usfs_t *cufs; + cflsh_usfs_data_obj_t *file; + + + /* + * ?? Do we have to handle disk_name:pathname here + */ + + if (pathname == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"pathname is null"); + + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + cufs = cusfs_find_fs_from_path(pathname,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"could not find filesystem for pathname = %s", + pathname); + + errno = EINVAL; + return -1; + } + + file = cusfs_lookup_data_obj(cufs,pathname,0); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"file not found for %s on disk %s", + pathname,cufs->device_name); + + errno = ENOENT; + return -1; + } + + if (file->inode->type != CFLSH_USFS_INODE_FILE_DIRECTORY) { + + CUSFS_TRACE_LOG_FILE(1,"file %s is not a directory on disk %s", + file->filename,cufs->device_name); + + errno = ENOTDIR; + + return -1; + } + + CUSFS_LOCK(file->lock); + rc = cusfs_list_files_directory(cufs,file->inode,file->filename,flags); + + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + +/* + * NAME: cusfs_rename + * + * FUNCTION: Rename a file/directory to another name + * in the same filesystem. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_rename(char *from_pathname, + char *to_pathname) +{ + int rc = 0; + cflsh_usfs_t *cufs; + cflsh_usfs_data_obj_t *file; + + + /* + * ?? Do we have to handle disk_name:pathname here + */ + + if (from_pathname == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"from_pathname is null"); + + errno = EINVAL; + return -1; + } + + if (to_pathname == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"to_pathname is null"); + + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = cusfs_open_disk_filename(from_pathname,0); + + cufs = file->fs; + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"could not find filesystem for pathname = %s", + from_pathname); + errno = EINVAL; + return -1; + } + + CUSFS_LOCK(file->lock); + rc = cusfs_mv_data_obj(cufs,file,to_pathname,0); + + CUSFS_UNLOCK(file->lock); + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + + + + +/* + * NAME: cusfs_open + * + * FUNCTION: Open a file + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_open(char *path, int flags, mode_t mode) +{ + int rc = -1; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs = NULL; + char device_name[PATH_MAX]; + char *filename; + uid_t uid; + gid_t gid; + +#ifdef _AIX + if (flags & (O_EFSON|O_EFSOFF|O_RAW)) { + + + /* + * We do not support encrypted filesystems + */ + + CUSFS_TRACE_LOG_FILE(1,"Unsupported flags = 0x%x",flags); + errno = EINVAL; + + return -1; + } +#endif /* _AIX */ + + CUSFS_TRACE_LOG_FILE(5,"path = %s, flags = %d, pid = %d", + path,flags,getpid()); + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + file = cusfs_open_disk_filename(path,0); + + if ((file == NULL) && + (flags & O_CREAT)) { + + + CUSFS_TRACE_LOG_FILE(9,"file %s does not exist, creating...", + path); + + /* + * File does not exist yet. + * Only create it if the 0_CREATE flag + * is specified. + * + * Need to first get file system + * object. + */ + + strcpy(device_name,path); + + + filename = strchr(device_name,CFLSH_USFS_DISK_DELIMINATOR); + + if (filename == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Invalid disk_pathname %s", + device_name); + + errno = EINVAL; + return rc; + } + + filename[0] = '\0'; + filename++; + + cufs = cusfs_find_fs_for_disk(device_name,0); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Did not find a filesystem for this disk %s", + device_name); + + errno = EINVAL; + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return rc; + } + + + /* + * If we get here, + * then the filesystem exits + */ + + uid = getuid(); + + + /* + * ?? TODO: If parent directory has S_ISGID set, then + * group id should be set to the parent directories. + * group id. + */ + + gid = getgid(); + + file = cusfs_create_file(cufs,filename,mode,uid,gid,0); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"failed to create file %s for this disk %s", + filename,device_name); + + rc = -1; + } else { + + + if (file->disk_lbas == NULL) { + + if (cusfs_get_all_disk_lbas_from_inode(cufs,file->inode,&(file->disk_lbas),&(file->num_disk_lbas),0)) { + + rc = -1; + + } else { + + rc = file->index; + } + + } else { + rc = file->index; + } + + } + + + + } else { + + cufs = file->fs; + + CUSFS_TRACE_LOG_FILE(9,"file %s already exists", + path); + + if (file->disk_lbas == NULL) { + + if (cusfs_get_all_disk_lbas_from_inode(cufs,file->inode,&(file->disk_lbas),&(file->num_disk_lbas),0)) { + + rc = -1; + + } else { + + rc = file->index; + } + + } else { + rc = file->index; + } + } + + if (flags & O_SYNC) { + file->flags |= CFLSH_USFS_SYNC_IO; + } + + if (flags & O_DIRECT) { + file->flags |= CFLSH_USFS_DIRECT_IO; + } + + cusfs_add_file_to_hash(file,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + + +/* + * NAME: cusfs_creat + * + * FUNCTION: Create a file + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_creat(char *path, mode_t mode) +{ + int rc = -1; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs = NULL; + char device_name[PATH_MAX]; + char *filename; + uid_t uid; + gid_t gid; + + + + + CUSFS_TRACE_LOG_FILE(5,"path = %s", + path); + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + file = cusfs_open_disk_filename(path,0); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(9,"file %s does not exist, creating...", + path); + + /* + * File does not exist yet + * Need to determine if the file system + * exists. + */ + + strcpy(device_name,path); + + + filename = strchr(device_name,CFLSH_USFS_DISK_DELIMINATOR); + + if (filename == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Invalid disk_pathname %s", + device_name); + + errno = EINVAL; + return rc; + } + + filename[0] = '\0'; + filename++; + + cufs = cusfs_find_fs_for_disk(device_name,0); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Did not find a filesystem for this disk %s", + device_name); + + errno = EINVAL; + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return rc; + } + + + /* + * If we get here, + * then the filesystem exits + */ + + uid = getuid(); + + gid = getgid(); + + file = cusfs_create_file(cufs,filename,mode,uid,gid,0); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"failed to create file %s for this disk %s", + filename,device_name); + + rc = -1; + } else { + + rc = file->index; + } + + + + } else { + + rc = file->index; + } + + cusfs_add_file_to_hash(file,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; + +} + + +/* + * NAME: cusfs_close + * + * FUNCTION: Close a file + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_close(int fd) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + + errno = EINVAL; + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + + } + + + CUSFS_LOCK(file->lock); + + if (cusfs_wait_for_aio_for_file(cufs,file,0)) { + + + CUSFS_TRACE_LOG_FILE(5,"wait failed for closed file = %s of type %d on disk %s, ", + file->filename,file->inode->type, cufs->device_name); + + CUSFS_UNLOCK(file->lock); + errno = EBUSY; + return -1; + } + + cusfs_flush_inode_timestamps(cufs,file); + + CUSFS_UNLOCK(file->lock); + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + cusfs_remove_file_from_hash(file,0); + + + CUSFS_TRACE_LOG_FILE(5,"closed file = %s of type %d on disk %s, ", + file->filename,file->inode->type, cufs->device_name); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + cusfs_display_stats(cufs,3); + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + + +/* + * NAME: cusfs_rdwr + * + * FUNCTION: Read/Write a file of the specified size. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +ssize_t _cusfs_rdwr(int fd, struct aiocb64 *aiocbp, void *buffer,size_t nbytes,int flags) +{ + ssize_t rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + size_t nblocks; + uint64_t cur_nblocks; + uint64_t offset; +#ifdef _REMOVE + int level; +#endif + int read_ahead_retry; + int aio_flags; + void *buf; + size_t buf_size; + size_t nblocks_in_bytes; + cflsh_usfs_disk_lba_e_t *disk_lbas; + uint64_t num_disk_lbas; + int i; +#ifdef _REMOVE + int tag; + uint64_t status; +#endif + size_t bytes_transferred = 0; + int copy_buf = FALSE; /* Whether we need to copy data Copy data for */ + /* the caller, when I/O is not block aligned. */ + + + + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + + if ((aiocbp) && (buffer == NULL)) { + + buffer = (void *)aiocbp->aio_buf; + } + + if (buffer == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Buffer is NULL for fd = %d",fd); + errno = EINVAL; + return -1; + } + + if (aiocbp) { + + offset = aiocbp->aio_offset; + } else { + + offset = file->offset; + } + + + if (nbytes == 0x0LL) { + + return 0; + } + + + CUSFS_TRACE_LOG_FILE(5,"%s%s file = %s of type %d, nbytes = 0x%llx, on disk %s, ", + (aiocbp ? "Async ":""), + ((flags & CFLSH_USFS_RDWR_WRITE) ? "Write":" Read"), + file->filename,file->inode->type, nbytes, + cufs->device_name); + + + CUSFS_LOCK(cufs->lock); + CUSFS_LOCK(file->lock); + + + if ((offset + nbytes) > file->inode->file_size) { + + if (!(flags & CFLSH_USFS_RDWR_WRITE)) { + + /* + * Do not allow read beyond end of file. + */ + + if (file->inode->file_size > offset) { + + nbytes = file->inode->file_size - offset; + + } else { + + + CUSFS_TRACE_LOG_FILE(1,"read beyond end of disk %s for file %", + cufs->device_name,file->filename); + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + return -1; + + } + + } else { + + /* + * This is a write beyond the current end of the file. + * Add more blocks if necessary to increase file size. + */ + + nblocks = (offset + nbytes)/cufs->fs_block_size; + + if ((offset + nbytes) % cufs->fs_block_size) { + + nblocks++; + + } + +#ifdef _REMOVE + if (cusfs_num_data_blocks_in_inode(cufs,file->inode,&cur_nblocks,&level,0)) { + + + + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + return -1; + } + +#endif /* REMOVE */ + + cur_nblocks = file->num_disk_lbas; + + if (cur_nblocks < nblocks) { + + /* + * We currently do not have enough blocks. + * So allocate them now. + */ + + nblocks -= cur_nblocks; + + CUSFS_TRACE_LOG_FILE(9,"For file = %s on disk %s need to add 0x%llx blocks ", + file->filename,cufs->device_name,nblocks); + + if (cusfs_add_fs_blocks_to_inode_ptrs(cufs,file->inode,nblocks,0)) { + + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + return -1; + } + + free(file->disk_lbas); + + if (cusfs_get_all_disk_lbas_from_inode(cufs,file->inode,&(file->disk_lbas),&(file->num_disk_lbas),0)) { + + /* + * This failure makes file unusable. + * ?? TODO: Do we need something more here + */ + + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + return -1; + } + } + + /* + * Update Inode with new file + * size. + */ + + file->inode->file_size = offset + nbytes; + + cusfs_update_inode(cufs,file->inode,0); + + } + } + + + if (cusfs_get_disk_lbas_for_transfer(cufs, file, offset, + nbytes, &disk_lbas,&num_disk_lbas,0)) { + + CUSFS_TRACE_LOG_FILE(1,"For file = %s on disk %s failed to get inode pointers offset = 0x%llx", + file->filename,cufs->device_name,offset); + + + + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + return -1; + + } + + +#ifdef _REMOVE + if (cusfs_get_disk_lbas_from_inode_ptr_for_transfer(cufs, file->inode, offset, + nbytes, &disk_lbas,&num_disk_lbas,0)) { + + CUSFS_TRACE_LOG_FILE(1,"For file = %s on disk %s failed to get inode pointers offset = 0x%llx", + file->filename,cufs->device_name,offset); + + + + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + return -1; + + } +#endif /* _REMOVE */ + + + +//?? buf_size = nbytes; + + + bytes_transferred = 0; + + for (i = 0; i < num_disk_lbas; i++) { + + nblocks = MIN(disk_lbas[i].num_lbas,cufs->max_xfer_size); + + + while (nblocks) { + + + nblocks_in_bytes = nblocks * cufs->disk_block_size; + + buf_size = nblocks_in_bytes; + + CUSFS_TRACE_LOG_FILE(9,"nblocks = 0x%llx buf_size = 0x%llx, i = 0x%x", + nblocks,buf_size,i); + + if ((offset % cufs->disk_block_size) || + (((offset + nbytes) % cufs->disk_block_size) && + (offset/cufs->disk_block_size == (offset + nbytes)/cufs->disk_block_size))) { + + /* + * If the start of the transfer is not aligned on a disk block + * boundary, then we need to use our internal copy buffer. + * + * Similarly if the end of the transfer is not disk block aligned, but it + * in the current nblocks where are doing I/O then we need to + * use our internal copy buffer. + */ + + + if (offset % cufs->disk_block_size) { + + /* + * This request does not start on + * disk block boundary and it is + * in the nblocks being transferred now. + */ + buf_size -= (offset % cufs->disk_block_size); + + } + + if (((offset + nbytes) % cufs->disk_block_size) && + (offset/cufs->disk_block_size == (offset + nbytes)/cufs->disk_block_size)) { + + /* + * This request does not end on + * disk block boundary and it is in the + * nblocks being transferred now. + */ + + buf_size -= (nblocks_in_bytes - ((offset + nbytes) % cufs->disk_block_size)); + + } + copy_buf = TRUE; + + buf = file->data_buf; + } else { + + copy_buf = FALSE; + buf = (buffer + bytes_transferred); + } + + CUSFS_TRACE_LOG_FILE(9,"nblocks = 0x%llx buf_size = 0x%llx, i = 0x%x, copy_buf = %d", + nblocks,buf_size,i,copy_buf); + + aio_flags = 0; + if (flags & CFLSH_USFS_RDWR_WRITE) { + + if (copy_buf) { + + /* + * For this case we need to do a read/modify/write + * First read in the full data blocks + */ + + rc = cblk_read(cufs->chunk_id,buf,disk_lbas[i].start_lba,nblocks,0); + + if (rc < 0) { + + + free(disk_lbas); + + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + return -1; + } + + if (rc < nblocks) { + + free(disk_lbas); + + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + + if (aiocbp) { + + errno = EINVAL; + return -1; + } else { + return bytes_transferred; + } + + } + + + bcopy((buffer + bytes_transferred),buf,buf_size); + } + + if (((cufs->flags & (CFLSH_USFS_DIRECT_IO|CFLSH_USFS_SYNC_IO)) || + (cufs->async_pool_num_cmds == 0)) && + (aiocbp == NULL)) { + /* + * We are not allowed to do async I/O for this write. + * So issue a synchronous write. + */ + + rc = cblk_write(cufs->chunk_id,buf,disk_lbas[i].start_lba,nblocks,0); + } else { + /* + * Issue asynchronous write + */ + + if (!copy_buf && aiocbp) { + aio_flags = CFLSH_USFS_RDWR_DIRECT; + } + + aio_flags |= CFLSH_USFS_RDWR_WRITE; + + rc = cusfs_issue_aio(cufs,file,disk_lbas[i].start_lba,aiocbp,buf,buf_size,aio_flags); + + + } + } else { + + + if ((file->seek_ahead.flags & CFLSH_USFS_D_RH_ACTIVE) && + (file->seek_ahead.start_offset <= offset) && + ((file->seek_ahead.nblocks * cufs->disk_block_size + file->seek_ahead.start_offset) >= (offset + buf_size))) { + + /* + * If we issued an async read ahead for these lbas, then + * check for completion of that async read now. + */ + + CUSFS_TRACE_LOG_FILE(9,"read ahead found. start_lba = 0x%llx,nblocks = 0x%llx", + file->seek_ahead.start_lba,file->seek_ahead.nblocks); + + read_ahead_retry = 0; + + while (read_ahead_retry++ < CFLSH_USFS_WAIT_READ_AHEAD_RETRY) { + if ((file->seek_ahead.cblk_status.status == CBLK_ARW_STATUS_SUCCESS) || + (file->seek_ahead.cblk_status.status == CBLK_ARW_STATUS_FAIL) || + (file->seek_ahead.cblk_status.status == CBLK_ARW_STATUS_INVALID)) { + + /* + * Read ahead completed. + */ + + + file->seek_ahead.flags &= ~CFLSH_USFS_D_RH_ACTIVE; + + if (file->seek_ahead.cblk_status.status == CBLK_ARW_STATUS_FAIL) { + + + rc = -1; + errno = file->seek_ahead.cblk_status.fail_errno; + } else if (file->seek_ahead.cblk_status.status == CBLK_ARW_STATUS_INVALID) { + + rc = -1; + errno = EINVAL; + } else { + + + + if (file->seek_ahead.cblk_status.blocks_transferred) { + + /* + * ?? This code is not handling the case where the transfer returned + * success, but had a resid. + */ + + rc = buf_size/cufs->disk_block_size; + + } + /* + * Set copy buff to allow the code below + * copy out the data. + */ + copy_buf = TRUE; + + buf = file->seek_ahead.local_buffer + (offset - file->seek_ahead.start_offset); + + + } + + break; + } else { + + /* + * We need to wait for completion + */ + + usleep(CFLSH_USFS_WAIT_READ_AHEAD); + } + + + } + + if (read_ahead_retry >= CFLSH_USFS_WAIT_READ_AHEAD_RETRY) { + + rc = -1; + errno = ETIMEDOUT; + } + + + CUSFS_TRACE_LOG_FILE(9,"read ahead found. rc = 0x%llx, errno = %d, read_ahead_retry = %d", + rc, errno, read_ahead_retry); + + } else { + + if (aiocbp) { + /* + * Issue asynchronous read + */ + if (!copy_buf) { + aio_flags = CFLSH_USFS_RDWR_DIRECT; + } + rc = cusfs_issue_aio(cufs,file,disk_lbas[i].start_lba,aiocbp,buf,buf_size,aio_flags); + + } else { + + /* + * Issue synchronous read + */ + rc = cblk_read(cufs->chunk_id,buf,disk_lbas[i].start_lba, + nblocks,0); +#ifdef _REMOVE + rc = cblk_aread(cufs->chunk_id,buf,disk_lbas[i].start_lba, + nblocks,&tag,NULL,0); + + CUSFS_TRACE_LOG_FILE(9,"rc = 0x%llx,nblocks = 0x%llx ", + rc,nblocks); + if (rc == 0) { + CUSFS_TRACE_LOG_FILE(9,"rc = 0x%llx,nblocks = 0x%llx ", + rc,nblocks); + rc = cblk_aresult(cufs->chunk_id,&tag,&status,CBLK_ARESULT_BLOCKING); + + CUSFS_TRACE_LOG_FILE(9,"rc = 0x%llx,nblocks = 0x%llx errno = %d", + rc,nblocks,errno); + + } +#endif + + } + } + + } + + + if (rc < 0) { + + + free(disk_lbas); + + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + return -1; + } + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"%s%s of file = %s of type %d on disk %s failed to disk at lba 0x%llx, ", + (aiocbp ? "Async ":""), + ((flags & CFLSH_USFS_RDWR_WRITE) ? "Write":" Read"), + file->filename,file->inode->type, cufs->device_name,disk_lbas[i].start_lba); + + + /* + * Return the total number of bytes read. + */ + + if (!(flags & CFLSH_USFS_RDWR_WRITE)) { + if (copy_buf) { + + + nblocks_in_bytes = rc * cufs->disk_block_size; + + buf_size = nblocks_in_bytes; + + + if (offset % nblocks_in_bytes) { + + /* + * This request does not start on + * disk block boundary and it is + * in the nblocks being transferred now. + */ + buf_size -= (offset % nblocks_in_bytes); + + } + + if (((offset + nbytes) % nblocks_in_bytes) && + (offset/nblocks_in_bytes == (offset + nbytes)/nblocks_in_bytes)) { + + /* + * This request does not end on + * disk block boundary and it is in the + * nblocks being transferred now. + */ + + buf_size -= (nblocks_in_bytes - ((offset + nbytes) % nblocks_in_bytes)); + + } + + bcopy(buf,(buffer + bytes_transferred),buf_size); + } + } + + + bytes_transferred += rc * cufs->disk_block_size; + + offset += rc * cufs->disk_block_size; + + free(disk_lbas); + + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + if (aiocbp) { + return 0; + } else { + return bytes_transferred; + } + } + + if (!(flags & CFLSH_USFS_RDWR_WRITE)) { + if (copy_buf) { + + bcopy(buf,(buffer + bytes_transferred),buf_size); + } + } + + + offset += buf_size; + + bytes_transferred += buf_size; + + if (nblocks > cufs->max_xfer_size) { + + + CUSFS_TRACE_LOG_FILE(1,"nblocks too large = 0x%llx buf_size = 0x%llx, nbytes = 0x%llx", + nblocks,buf_size,nbytes); + free(disk_lbas); + + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + +#ifndef ESYSERROR + errno = ELIBBAD; +#else + errno = ESYSERROR; +#endif + + return -1; + + } else { + nblocks = MIN((disk_lbas[i].num_lbas - nblocks),cufs->max_xfer_size); + } + + if (bytes_transferred >= nbytes) { + + + CUSFS_TRACE_LOG_FILE(5,"nblocks = 0x%llx buf_size = 0x%llx, nbytes = 0x%llx", + nblocks,buf_size,nbytes); + + if (bytes_transferred) { + + rc = bytes_transferred; + + } else { + + errno = EINVAL; + rc = -1; + } + + break; + } + + } /* while */ + + + } /* for */ + + + free(disk_lbas); + + CUSFS_TRACE_LOG_FILE(5,"%s%s success of file = %s of bytes 0x%llx, offset 0x%llx on disk %s, ", + (aiocbp ? "Async ":""), + ((flags & CFLSH_USFS_RDWR_WRITE) ? "Write":" Read"), + file->filename,bytes_transferred, offset,cufs->device_name); + + if (aiocbp == NULL) { + + file->offset = offset; + } + + + + if (flags & CFLSH_USFS_RDWR_WRITE) { + + file->inode_updates.flags |= CFLSH_USFS_D_INODE_MTIME; +#if !defined(__64BIT__) && defined(_AIX) + file->inode_updates.mtime = cflsh_usfs_time64(NULL); +#else + file->inode_updates.mtime = time(NULL); +#endif /* not 32-bit AIX */ + } else { + + file->inode_updates.flags |= CFLSH_USFS_D_INODE_ATIME; +#if !defined(__64BIT__) && defined(_AIX) + file->inode_updates.atime = cflsh_usfs_time64(NULL); +#else + file->inode_updates.atime = time(NULL); +#endif /* not 32-bit AIX */ + + } + + + CUSFS_UNLOCK(file->lock); + + CUSFS_UNLOCK(cufs->lock); + + if (aiocbp) { + return 0; + } else { + return bytes_transferred; + } + +} + + + +/* + * NAME: cusfs_read + * + * FUNCTION: Read a file of the specified size. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +ssize_t cusfs_read(int fd, void *buffer,size_t nbytes) +{ + + return (_cusfs_rdwr(fd,NULL,buffer,nbytes,0)); +} + +/* + * NAME: cusfs_write + * + * FUNCTION: Write a file of the specified size. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +ssize_t cusfs_write(int fd, void *buffer,size_t nbytes) +{ + + return (_cusfs_rdwr(fd,NULL,buffer,nbytes, CFLSH_USFS_RDWR_WRITE)); +} + + +/* + * NAME: cusfs_fsync + * + * FUNCTION: Write changes in a file to permanent storage + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_fsync(int fd) +{ + + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + + errno = EINVAL; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + + + CUSFS_TRACE_LOG_FILE(5,"fsync file = %s of type %d on disk %s, ", + file->filename,file->inode->type, cufs->device_name); + + CUSFS_LOCK(file->lock); + + + cusfs_flush_inode_timestamps(cufs,file); + + if (cusfs_wait_for_aio_for_file(cufs,file,CFLSH_USFS_ONLY_ASYNC_CMPLT)) { + + rc = -1; + } + + CUSFS_UNLOCK(file->lock); + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + +/* + * NAME: cusfs_lseek + * + * FUNCTION: Seek to a specified offset in the file + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +off_t cusfs_lseek(int fd, off_t Offset, int Whence) +{ + off_t rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + +#if !defined(__64BIT__) && defined(_AIX) + + CUSFS_TRACE_LOG_FILE(5,"lseek file = %s Offset = 0x%x of type %d on disk %s, ", + file->filename,Offset,file->inode->type, cufs->device_name); +#else + CUSFS_TRACE_LOG_FILE(5,"lseek file = %s Offset = 0x%llx of type %d on disk %s, ", + file->filename,Offset,file->inode->type, cufs->device_name); +#endif /* 64-bit */ + + CUSFS_LOCK(file->lock); + + switch (Whence) { + + case SEEK_SET: + if (Offset > file->inode->file_size) { + + errno = EINVAL; + CUSFS_UNLOCK(file->lock); + return -1; + } + file->offset = Offset; + break; + case SEEK_CUR: + if ((file->offset + Offset) > file->inode->file_size) { + + errno = EINVAL; + CUSFS_UNLOCK(file->lock); + return -1; + } + file->offset += Offset; + break; + case SEEK_END: + file->offset = file->inode->file_size; + break; + default: + errno = EINVAL; + return -1; + } +#if !defined(__64BIT__) && defined(_AIX) + rc = file->offset & 0xffffffff; +#endif /* AIX and 32-bit */ + + /* + * Issue read ahead at this offset + */ + cusfs_seek_read_ahead(cufs,file,file->offset,0); + + CUSFS_UNLOCK(file->lock); + +#if !defined(__64BIT__) && defined(_AIX) + + CUSFS_TRACE_LOG_FILE(5,"lseek file = %s rc = 0x%x of type %d on disk %s, ", + file->filename,rc,file->inode->type, cufs->device_name); +#else + CUSFS_TRACE_LOG_FILE(5,"lseek file = %s rc = 0x%llx of type %d on disk %s, ", + file->filename,rc,file->inode->type, cufs->device_name); +#endif /* 64-bit */ + return rc; +} + +#ifdef _AIX +/* + * NAME: cusfs_llseek + * + * FUNCTION: Seek to a specified offset in the file + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +offset_t cusfs_llseek(int fd, offset_t Offset ,int Whence) +{ + offset_t rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EBADF; + return -1; + + } + + + CUSFS_TRACE_LOG_FILE(5,"lseek file = %s Offset = 0x%llx of type %d on disk %s, ", + file->filename,Offset,file->inode->type, cufs->device_name); + + CUSFS_LOCK(file->lock); + + switch (Whence) { + + case SEEK_SET: + if (Offset > file->inode->file_size) { + + errno = EINVAL; + CUSFS_UNLOCK(file->lock); + return -1; + } + file->offset = Offset; + break; + case SEEK_CUR: + if ((file->offset + Offset) > file->inode->file_size) { + + errno = EINVAL; + CUSFS_UNLOCK(file->lock); + return -1; + } + file->offset += Offset; + break; + case SEEK_END: + file->offset = file->inode->file_size; + break; + default: + errno = EINVAL; + return -1; + } + + rc = file->offset; + + /* + * Issue read ahead at this offset + */ + cusfs_seek_read_ahead(cufs,file,file->offset,0); + + CUSFS_UNLOCK(file->lock); + + CUSFS_TRACE_LOG_FILE(5,"lseek file = %s rc = 0x%llx of type %d on disk %s, ", + file->filename,rc,file->inode->type, cufs->device_name); + return rc; +} + + +#endif /* AIX */ + +/* + * NAME: cusfs_lseek64 + * + * FUNCTION: Seek to a specified offset in the file + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +off64_t cusfs_lseek64(int fd, off64_t Offset, int Whence) +{ + + off64_t rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EBADF; + return -1; + + } + + + CUSFS_TRACE_LOG_FILE(5,"lseek file = %s Offset = 0x%llx of type %d on disk %s, ", + file->filename,Offset,file->inode->type, cufs->device_name); + + CUSFS_LOCK(file->lock); + + switch (Whence) { + + case SEEK_SET: + if (Offset > file->inode->file_size) { + + errno = EINVAL; + CUSFS_UNLOCK(file->lock); + return -1; + } + file->offset = Offset; + break; + case SEEK_CUR: + if ((file->offset + Offset) > file->inode->file_size) { + + errno = EINVAL; + CUSFS_UNLOCK(file->lock); + return -1; + } + file->offset += Offset; + break; + case SEEK_END: + file->offset = file->inode->file_size; + break; + default: + errno = EINVAL; + return -1; + } + + rc = file->offset; + + /* + * Issue read ahead at this offset + */ + cusfs_seek_read_ahead(cufs,file,file->offset,0); + + CUSFS_UNLOCK(file->lock); + + CUSFS_TRACE_LOG_FILE(5,"lseek file = %s rc = 0x%llx of type %d on disk %s, ", + file->filename,rc,file->inode->type, cufs->device_name); + return rc; +} + + +/* + * NAME: cusfs_fstat + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_fstat(int fd,struct stat *Buffer) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + uint64_t num_data_blocks; + int level; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + + if (Buffer == NULL) { + + errno = EFAULT; + return -1; + + } + + bzero(Buffer,sizeof(*Buffer)); + + CUSFS_LOCK(file->lock); + + + cusfs_flush_inode_timestamps(cufs,file); + + Buffer->st_mode = file->inode->mode_flags; + + Buffer->st_uid = file->inode->uid; + + Buffer->st_gid = file->inode->gid; + + Buffer->st_size = file->inode->file_size; + + Buffer->st_atime = file->inode->atime; + + Buffer->st_ctime = file->inode->ctime; + + Buffer->st_mtime = file->inode->mtime; + + Buffer->st_nlink = file->inode->num_links; + + Buffer->st_blksize = cufs->fs_block_size; + + + if (cusfs_num_data_blocks_in_inode(cufs,file->inode,&num_data_blocks,&level,0) == 0) { + + + Buffer->st_blocks = num_data_blocks; + } + + Buffer->st_ino = file->inode->index; + + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + + + + +/* + * NAME: cusfs_fstat64 + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_fstat64(int fd,struct stat64 *Buffer) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + uint64_t num_data_blocks; + int level; + + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + + if (Buffer == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Stat buffer is null for fd = %d",fd); + errno = EFAULT; + return -1; + + } + + bzero(Buffer,sizeof(*Buffer)); + + CUSFS_LOCK(file->lock); + + + cusfs_flush_inode_timestamps(cufs,file); + + Buffer->st_mode = file->inode->mode_flags; + + Buffer->st_uid = file->inode->uid; + + Buffer->st_gid = file->inode->gid; + + Buffer->st_size = file->inode->file_size; + + Buffer->st_atime = file->inode->atime; + + Buffer->st_ctime = file->inode->ctime; + + Buffer->st_mtime = file->inode->mtime; + + Buffer->st_nlink = file->inode->num_links; + + Buffer->st_blksize = cufs->fs_block_size; + + + if (cusfs_num_data_blocks_in_inode(cufs,file->inode,&num_data_blocks,&level,0) == 0) { + + + Buffer->st_blocks = num_data_blocks; + } + + + Buffer->st_ino = file->inode->index; + + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + +#ifdef _AIX + +/* + * NAME: cusfs_fstat64x + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_fstat64x(int fd,struct stat64x *Buffer) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + uint64_t num_data_blocks; + int level; + + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + + if (Buffer == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Stat buffer is null for fd = %d",fd); + errno = EFAULT; + return -1; + + } + + bzero(Buffer,sizeof(*Buffer)); + + CUSFS_LOCK(file->lock); + + + cusfs_flush_inode_timestamps(cufs,file); + + Buffer->st_mode = file->inode->mode_flags; + + Buffer->st_uid = file->inode->uid; + + Buffer->st_gid = file->inode->gid; + + Buffer->st_size = file->inode->file_size; + + Buffer->st_atime = file->inode->atime; + + Buffer->st_ctime = file->inode->ctime; + + Buffer->st_mtime = file->inode->mtime; + + Buffer->st_nlink = file->inode->num_links; + + Buffer->st_blksize = cufs->fs_block_size; + + + if (cusfs_num_data_blocks_in_inode(cufs,file->inode,&num_data_blocks,&level,0) == 0) { + + + Buffer->st_blocks = num_data_blocks; + } + + + Buffer->st_ino = file->inode->index; + + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + +#endif /* _AIX */ + + +/* + * NAME: cusfs_stat64 + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_stat64(char *path, struct stat64 *Buffer) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + uint64_t num_data_blocks; + int level; + + + CUSFS_TRACE_LOG_FILE(5,"path = %s, pid = %d", + path,getpid()); + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + file = cusfs_open_disk_filename(path,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for path = %s",path); + errno = ENOENT; + + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for path = %s",path); + errno = EINVAL; + return -1; + + } + + if (Buffer == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Stat buffer is null for path = %s",path); + errno = EFAULT; + return -1; + + } + + bzero(Buffer,sizeof(*Buffer)); + + CUSFS_LOCK(file->lock); + + cusfs_flush_inode_timestamps(cufs,file); + + Buffer->st_mode = file->inode->mode_flags; + + Buffer->st_uid = file->inode->uid; + + Buffer->st_gid = file->inode->gid; + + Buffer->st_size = file->inode->file_size; + + Buffer->st_atime = file->inode->atime; + + Buffer->st_ctime = file->inode->ctime; + + Buffer->st_mtime = file->inode->mtime; + + Buffer->st_nlink = file->inode->num_links; + + Buffer->st_blksize = cufs->fs_block_size; + + + if (cusfs_num_data_blocks_in_inode(cufs,file->inode,&num_data_blocks,&level,0) == 0) { + + + Buffer->st_blocks = num_data_blocks; + } + + + Buffer->st_ino = file->inode->index; + + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + +#ifdef _AIX + +/* + * NAME: cusfs_stat64x + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_stat64x(char *path, struct stat64x *Buffer) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + uint64_t num_data_blocks; + int level; + + + CUSFS_TRACE_LOG_FILE(5,"path = %s", + path); + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + file = cusfs_open_disk_filename(path,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for path = %s",path); + errno = ENOENT; + + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for path = %s",path); + errno = EINVAL; + return -1; + + } + + if (Buffer == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Stat buffer is null for path = %s",path); + errno = EFAULT; + return -1; + + } + + bzero(Buffer,sizeof(*Buffer)); + + CUSFS_LOCK(file->lock); + + cusfs_flush_inode_timestamps(cufs,file); + + Buffer->st_mode = file->inode->mode_flags; + + Buffer->st_uid = file->inode->uid; + + Buffer->st_gid = file->inode->gid; + + Buffer->st_size = file->inode->file_size; + + Buffer->st_atime = file->inode->atime; + + Buffer->st_ctime = file->inode->ctime; + + Buffer->st_mtime = file->inode->mtime; + + Buffer->st_nlink = file->inode->num_links; + + Buffer->st_blksize = cufs->fs_block_size; + + + if (cusfs_num_data_blocks_in_inode(cufs,file->inode,&num_data_blocks,&level,0) == 0) { + + + Buffer->st_blocks = num_data_blocks; + } + + + Buffer->st_ino = file->inode->index; + + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + + +#endif /* _AIX */ + +/* + * NAME: cusfs_stat + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ +int cusfs_stat(char *path, struct stat *Buffer) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + uint64_t num_data_blocks; + int level; + + + CUSFS_TRACE_LOG_FILE(5,"path = %s", + path); + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + file = cusfs_open_disk_filename(path,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for path = %s",path); + errno = ENOENT; + + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for path = %s",path); + errno = EINVAL; + return -1; + + } + + if (Buffer == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Stat buffer is null for path = %s",path); + errno = EFAULT; + return -1; + + } + + bzero(Buffer,sizeof(*Buffer)); + + CUSFS_LOCK(file->lock); + + cusfs_flush_inode_timestamps(cufs,file); + + Buffer->st_mode = file->inode->mode_flags; + + Buffer->st_uid = file->inode->uid; + + Buffer->st_gid = file->inode->gid; + + Buffer->st_size = file->inode->file_size; + + Buffer->st_atime = file->inode->atime; + + Buffer->st_ctime = file->inode->ctime; + + Buffer->st_mtime = file->inode->mtime; + + Buffer->st_nlink = file->inode->num_links; + + Buffer->st_blksize = cufs->fs_block_size; + + + if (cusfs_num_data_blocks_in_inode(cufs,file->inode,&num_data_blocks,&level,0) == 0) { + + + Buffer->st_blocks = num_data_blocks; + } + + + Buffer->st_ino = file->inode->index; + + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + + +/* + * NAME: cusfs_lstat64 + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ +int cusfs_lstat64(char *path, struct stat64 *Buffer) +{ + //?? Add real support for lstat + + return(cusfs_stat64(path,Buffer)); +} + +#ifdef _AIX +/* + * NAME: cusfs_lstat64x + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ +int cusfs_lstat64x(char *path, struct stat64x *Buffer) +{ + //?? Add real support for lstat + + return(cusfs_stat64x(path,Buffer)); +} + +#endif /* _AIX */ + +/* + * NAME: cusfs_lstat + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_lstat(char *path, struct stat *Buffer) +{ + //?? Add real support for lstat + + + return(cusfs_stat(path,Buffer)); +} + + +/* + * NAME: cusfs_readlink + * + * FUNCTION: Validate access to file + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ +int cusfs_readlink(const char *path, char *buffer, size_t buffer_size) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + + + if (buffer == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"buffer is null for path = %s",path); + errno = EFAULT; + return -1; + + } + + + CUSFS_TRACE_LOG_FILE(5,"path = %s", + path); + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + file = cusfs_open_disk_filename((char *)path,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for path = %s",path); + errno = ENOENT; + + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for path = %s",path); + errno = EINVAL; + return -1; + + } + + if (file->inode->type != CFLSH_USFS_INODE_FILE_SYMLINK) { + + + CUSFS_TRACE_LOG_FILE(1,"path = %s is not symlink",path); + errno = EINVAL; + return -1; + } + + if (strlen(file->inode->sym_link_path) > buffer_size) { + + CUSFS_TRACE_LOG_FILE(1,"symlink %d is larger than buffer size %d for path = %s is not symlink", + strlen(file->inode->sym_link_path),buffer_size,path); + errno = ERANGE; + return -1; + } + + + bzero(buffer,buffer_size); + + + strcpy(buffer,file->inode->sym_link_path); + + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + + +/* + * NAME: cusfs_access + * + * FUNCTION: Validate access to file + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_access(char *path, int mode) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + char device_name[PATH_MAX]; + char *filename; + + + CUSFS_TRACE_LOG_FILE(5,"path = %s, mode = 0x%x", + path,mode); + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + file = cusfs_open_disk_filename(path,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(9,"file % does not exist, creating...", + path); + + /* + * File does not exist yet + * Need to determine if the file system + * exists. + */ + + strcpy(device_name,path); + + + filename = strchr(device_name,CFLSH_USFS_DISK_DELIMINATOR); + + if (filename == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Invalid disk_pathname %s", + device_name); + + errno = EINVAL; + return rc; + } + + filename[0] = '\0'; + filename++; + + cufs = cusfs_find_fs_for_disk(device_name,0); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Did not find a filesystem for this disk %s", + device_name); + + errno = EINVAL; + + rc = -1; + } else { + + + /* + * Filesystem exists, but file does not + */ + + errno = ENOENT; + rc = -1; + } + + } else { + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for path = %s",path); + errno = EINVAL; + return -1; + + } + + if ((mode & R_OK) && + !(file->inode->mode_flags & (S_IRUSR|S_IRGRP|S_IROTH))) { + + CUSFS_TRACE_LOG_FILE(5,"path %s does not have read access",path); + rc = -1; + errno = EACCES; + } else if ((mode & W_OK) && + !(file->inode->mode_flags & (S_IWUSR|S_IWGRP|S_IWOTH))) { + + CUSFS_TRACE_LOG_FILE(5,"path %s does not have write access",path); + rc = -1; + errno = EROFS; + } else if ((mode & X_OK) && + !(file->inode->mode_flags & (S_IXUSR|S_IXGRP|S_IXOTH))) { + + CUSFS_TRACE_LOG_FILE(5,"path %s does not have execute access",path); + rc = -1; + errno = EACCES; + } + + } + + + CUSFS_TRACE_LOG_FILE(5,"path = %s, rc = %d, errno = %d", + path,rc, errno); + + + return rc; + +} + + +/* + * NAME: cusfs_fchmod + * + * FUNCTION: Change mode of file + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_fchmod(int fd,mode_t mode) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + + CUSFS_LOCK(file->lock); + + file->inode->mode_flags = (file->inode->mode_flags & S_IFMT) | mode; + + cusfs_update_inode(cufs,file->inode,0); + + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + +/* + * NAME: cusfs_fchown + * + * FUNCTION: Change owner/group + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_fchown(int fd, uid_t uid, gid_t gid) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + + CUSFS_LOCK(file->lock); + + if (uid != -1) { + file->inode->uid = uid; + } + + + if (gid != -1) { + file->inode->gid = gid; + } + + cusfs_update_inode(cufs,file->inode,0); + + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + +/* + * NAME: cusfs_ftruncate + * + * FUNCTION: Change the length of a regular file + * to the length specified by the caller. + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_ftruncate(int fd, off_t length) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + uint64_t num_data_blocks; + int level; + + uint64_t length_in_blocks; + uint64_t delta_blocks; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + if (length <= 0) { + + return 0; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + + if (file->inode->type != CFLSH_USFS_INODE_FILE_REGULAR) { + + CUSFS_TRACE_LOG_FILE(1,"%s Not regular file, type = %d", + file->filename,file->inode->type); + errno = EINVAL; + + return -1; + } + + length_in_blocks = length/cufs->fs_block_size; + + if (length % cufs->fs_block_size) { + length_in_blocks++; + } + + CUSFS_LOCK(file->lock); + + + if (cusfs_num_data_blocks_in_inode(cufs,file->inode,&num_data_blocks,&level,0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + + } + + + if (num_data_blocks > length_in_blocks) { + + /* + * Remove data blocks from file + */ + + delta_blocks = num_data_blocks - length_in_blocks; + if (cusfs_remove_fs_blocks_from_inode_ptrs(cufs,file->inode,delta_blocks,0)) { + + + CUSFS_UNLOCK(file->lock); + return -1; + } + + free(file->disk_lbas); + + if (cusfs_get_all_disk_lbas_from_inode(cufs,file->inode,&(file->disk_lbas),&(file->num_disk_lbas),0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + } + + + } else if (num_data_blocks < length_in_blocks) { + + /* + * Add data blocks to a file + */ + + delta_blocks = length_in_blocks - num_data_blocks; + + if (cusfs_add_fs_blocks_to_inode_ptrs(cufs,file->inode,delta_blocks,0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + } + + free(file->disk_lbas); + + if (cusfs_get_all_disk_lbas_from_inode(cufs,file->inode,&(file->disk_lbas),&(file->num_disk_lbas),0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + } + } + + + file->inode->file_size = length; + cusfs_update_inode(cufs,file->inode,0); + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + + +/* + * NAME: cusfs_ftruncate64 + * + * FUNCTION: Change the length of a regular file + * to the length specified by the caller. + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_ftruncate64(int fd, off64_t length) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + uint64_t num_data_blocks; + int level; + + uint64_t length_in_blocks; + uint64_t delta_blocks; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + if (length <= 0) { + + return 0; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + + if (file->inode->type != CFLSH_USFS_INODE_FILE_REGULAR) { + + CUSFS_TRACE_LOG_FILE(1,"%s Not regular file, type = %d", + file->filename,file->inode->type); + errno = EINVAL; + + return -1; + } + + length_in_blocks = length/cufs->fs_block_size; + + if (length % cufs->fs_block_size) { + length_in_blocks++; + } + + CUSFS_LOCK(file->lock); + + + if (cusfs_num_data_blocks_in_inode(cufs,file->inode,&num_data_blocks,&level,0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + + } + + + if (num_data_blocks > length_in_blocks) { + + /* + * Remove data blocks from file + */ + + delta_blocks = num_data_blocks - length_in_blocks; + if (cusfs_remove_fs_blocks_from_inode_ptrs(cufs,file->inode,delta_blocks,0)) { + + + CUSFS_UNLOCK(file->lock); + return -1; + } + + free(file->disk_lbas); + + if (cusfs_get_all_disk_lbas_from_inode(cufs,file->inode,&(file->disk_lbas),&(file->num_disk_lbas),0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + } + + } else if (num_data_blocks < length_in_blocks) { + + /* + * Add data blocks to a file + */ + + delta_blocks = length_in_blocks - num_data_blocks; + + if (cusfs_add_fs_blocks_to_inode_ptrs(cufs,file->inode,delta_blocks,0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + } + + free(file->disk_lbas); + + if (cusfs_get_all_disk_lbas_from_inode(cufs,file->inode,&(file->disk_lbas),&(file->num_disk_lbas),0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + } + } + + + file->inode->file_size = length; + cusfs_update_inode(cufs,file->inode,0); + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + + +/* + * NAME: cusfs_posix_fadvise + * + * FUNCTION: Provides advisory information about a file. + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_posix_fadvise(int fd, off_t offset, off_t length, int advise) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + uint64_t num_data_blocks; + int level; + + uint64_t length_in_blocks; + uint64_t delta_blocks; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + if (length <= 0) { + + return 0; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + + if (file->inode->type != CFLSH_USFS_INODE_FILE_REGULAR) { + + CUSFS_TRACE_LOG_FILE(1,"%s Not regular file, type = %d", + file->filename,file->inode->type); + errno = EINVAL; + + return -1; + } + + length_in_blocks = (offset + length)/cufs->fs_block_size; + + if (length % cufs->fs_block_size) { + length_in_blocks++; + } + + CUSFS_LOCK(file->lock); + + + if (cusfs_num_data_blocks_in_inode(cufs,file->inode,&num_data_blocks,&level,0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + + } + + /* + * Currently the only thing we use advise for is + * as a suggestion of how large the file + * might need to be. + */ + + if (num_data_blocks < length_in_blocks) { + + /* + * Add data blocks to a file + */ + + delta_blocks = length_in_blocks - num_data_blocks; + + if (cusfs_add_fs_blocks_to_inode_ptrs(cufs,file->inode,delta_blocks,0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + } + + free(file->disk_lbas); + + if (cusfs_get_all_disk_lbas_from_inode(cufs,file->inode,&(file->disk_lbas),&(file->num_disk_lbas),0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + } + } + + + file->inode->file_size = length; + cusfs_update_inode(cufs,file->inode,0); + CUSFS_UNLOCK(file->lock); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d, length = 0x%llx, advice = 0x%x", + rc, errno,length,advise); + + return rc; +} + + +#ifndef _AIX + +/* + * NAME: cusfs_posix_fadvise64 + * + * FUNCTION: Provides advisory information about a file. + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int cusfs_posix_fadvise64(int fd, off64_t offset, off64_t length, int advise) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs; + uint64_t num_data_blocks; + int level; + + uint64_t length_in_blocks; + uint64_t delta_blocks; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + if (length <= 0) { + + return 0; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for fd = %d",fd); + errno = EINVAL; + return -1; + + } + + if (file->inode->type != CFLSH_USFS_INODE_FILE_REGULAR) { + + CUSFS_TRACE_LOG_FILE(1,"%s Not regular file, type = %d", + file->filename,file->inode->type); + errno = EINVAL; + + return -1; + } + + length_in_blocks = (offset + length)/cufs->fs_block_size; + + if (length % cufs->fs_block_size) { + length_in_blocks++; + } + + CUSFS_LOCK(file->lock); + + + if (cusfs_num_data_blocks_in_inode(cufs,file->inode,&num_data_blocks,&level,0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + + } + + /* + * Currently the only thing we use advise for is + * as a suggestion of how large the file + * might need to be. + */ + + + if (num_data_blocks < length_in_blocks) { + + /* + * Add data blocks to a file + */ + + delta_blocks = length_in_blocks - num_data_blocks; + + if (cusfs_add_fs_blocks_to_inode_ptrs(cufs,file->inode,delta_blocks,0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + } + + free(file->disk_lbas); + + if (cusfs_get_all_disk_lbas_from_inode(cufs,file->inode,&(file->disk_lbas),&(file->num_disk_lbas),0)) { + + CUSFS_UNLOCK(file->lock); + return -1; + } + } + + + file->inode->file_size = length; + cusfs_update_inode(cufs,file->inode,0); + CUSFS_UNLOCK(file->lock); + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d, length = 0x%llx, advice = 0x%x", + rc, errno,length,advise); + + return rc; +} + +#endif /* !_AIX */ + +/* + * NAME: cusfs_link + * + * FUNCTION: link a file + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_link(char *orig_path, char *path) +{ + int rc = 0; + cflsh_usfs_data_obj_t *link; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs = NULL; + char device_name[PATH_MAX]; + char *filename; + uid_t uid; + gid_t gid; + mode_t mode; + + + CUSFS_TRACE_LOG_FILE(5,"orig_path = %s, path", + orig_path, path); + + if (orig_path == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Orig_path is null"); + errno = EINVAL; + return -1; + } + + if (path == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"path is null"); + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + + file = cusfs_open_disk_filename(orig_path,0); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Could not find file for orig_path = %s", + file->filename); + errno = EINVAL; + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + link = cusfs_open_disk_filename(path,0); + + if (link) { + + CUSFS_TRACE_LOG_FILE(1,"Filename %s for link already exists on disk %ss", + path,device_name); + errno = EINVAL; + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + + /* + * File does not exist yet + * Need to determine if the file system + * exists. + */ + + strcpy(device_name,path); + + + filename = strchr(device_name,CFLSH_USFS_DISK_DELIMINATOR); + + if (filename == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Invalid disk_pathname %s", + device_name); + + errno = EINVAL; + return 01; + } + + filename[0] = '\0'; + filename++; + + cufs = cusfs_find_fs_for_disk(device_name,0); + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Did not find a filesystem for this disk %s", + device_name); + + errno = EINVAL; + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + if (cufs != file->fs) { + + CUSFS_TRACE_LOG_FILE(1,"new file on disk %s in a different filesystem then original file on disk %s", + cufs->device_name,file->fs->device_name); + + errno = EINVAL; + + free(cufs); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return -1; + } + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + uid = getuid(); + + gid = getgid(); + + + mode = S_IRWXU | S_IRWXG | S_IRWXO; + + rc = cusfs_create_link(orig_path,path,mode,uid,gid,0); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + + return rc; +} + +/* + * NAME: cusfs_unlink + * + * FUNCTION: Unlink a file + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_unlink(char *path) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs = NULL; + + + CUSFS_TRACE_LOG_FILE(5,"path = %s", + path); + + if (path == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"path is null"); + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + + file = cusfs_open_disk_filename(path,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for %s",path); + errno = ENOENT; + + + return -1; + } + + + cufs = file->fs; + + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + + rc = cusfs_free_data_obj(cufs,file,0); + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + +/* + * NAME: cusfs_utime + * + * FUNCTION: set acces and modification time of a file + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_utime(char *path,struct utimbuf *times) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs = NULL; + + CUSFS_TRACE_LOG_FILE(5,"path = %s", + path); + if (path == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"path is null"); + errno = EINVAL; + return -1; + } + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + + file = cusfs_open_disk_filename(path,0); + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for %s",path); + errno = ENOENT; + + return -1; + } + + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found for %s",path); + errno = EINVAL; + return -1; + + } + + + + file->inode->mtime = times->modtime; + file->inode->atime = times->actime; + + + + rc = cusfs_update_inode(cufs,file->inode,0); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + +#ifdef _NOT_YET +/* + * NAME: cusfs_futimens + * + * FUNCTION: set access and modification time of a file + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_futimens(int fd,struct timespec times[2]) +{ + int rc = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_t *cufs = NULL; + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + file = CUSFS_GET_FILE_HASH(fd,0); + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + if (file == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No file found for fd = %d",fd); + errno = EBADF; + return -1; + } + + cufs = file->fs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No filesystem found"); + errno = EINVAL; + return -1; + + } + + + + file->inode->mtime = times[0].tv_sec; + file->inode->atime = times[1].tv_sec; + + + + rc = cusfs_update_inode(cufs,file->inode,0); + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d,", + rc, errno); + return rc; +} +#endif + + + +/* + * NAME: cusfs_aio_read64 + * + * FUNCTION: Read a file of the specified size asynchronously + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_read64(struct aiocb64 *aiocbp) +{ + + /* + * Set initial state of AIOCB + */ + + CUSFS_RET_AIOCB(aiocbp,-1,EINPROGRESS); +#ifdef _AIX + aiocbp->aio_word1 = 0; + if (aiocbp->aio_handle == NULL) { + + aiocbp->aio_handle = (aio_handle_t)aiocbp; + } +#else + aiocbp->__glibc_reserved[0] = 0; + if (aiocbp->__next_prio == NULL) { + aiocbp->__next_prio = (struct aiocb *)aiocbp; + } +#endif + return (_cusfs_rdwr(aiocbp->aio_fildes,aiocbp,(void *)aiocbp->aio_buf,aiocbp->aio_nbytes,0)); +} + +/* + * NAME: cusfs_aio_read + * + * FUNCTION: Read a file of the specified size asynchronously + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_read(struct aiocb *aiocbp) +{ + struct aiocb64 aiocbp64; + + + /* + * Set initial state of AIOCB + */ + + CUSFS_RET_AIOCB(aiocbp,-1,EINPROGRESS); + + + aiocbp64.aio_fildes = aiocbp->aio_fildes; + aiocbp64.aio_buf = aiocbp->aio_buf; + aiocbp64.aio_nbytes = aiocbp->aio_nbytes; + aiocbp64.aio_offset = aiocbp->aio_offset; + aiocbp64.aio_sigevent = aiocbp->aio_sigevent; +#ifdef _AIX + if (aiocbp->aio_handle == NULL) { + + aiocbp64.aio_handle = aiocbp; + } else { + aiocbp64.aio_handle = aiocbp->aio_handle; + } + aiocbp64.aio_word1 = CUSFS_AIOCB32_FLG; +#else + if (aiocbp->__next_prio == NULL) { + + aiocbp64.__next_prio = aiocbp; + } else { + aiocbp64.__next_prio = aiocbp->__next_prio; + } + + aiocbp64.__glibc_reserved[0] = 0; +#endif + + return (cusfs_aio_read64(&aiocbp64)); +} + +/* + * NAME: cusfs_aio_write64 + * + * FUNCTION: Write a file of the specified size asynchronously + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_write64(struct aiocb64 *aiocbp) +{ + + /* + * Set initial state of AIOCB + */ + + CUSFS_RET_AIOCB(aiocbp,-1,EINPROGRESS); +#ifdef _AIX + aiocbp->aio_word1 = 0; + if (aiocbp->aio_handle == NULL) { + + aiocbp->aio_handle = (aio_handle_t)aiocbp; + } +#else + aiocbp->__glibc_reserved[0] = 0; + if (aiocbp->__next_prio == NULL) { + aiocbp->__next_prio = (struct aiocb *)aiocbp; + } +#endif + return (_cusfs_rdwr(aiocbp->aio_fildes,aiocbp,(void *)aiocbp->aio_buf,aiocbp->aio_nbytes,CFLSH_USFS_RDWR_WRITE)); +} + +/* + * NAME: cusfs_aio_write + * + * FUNCTION: Write a file of the specified size asynchronously + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_write(struct aiocb *aiocbp) +{ + struct aiocb64 aiocbp64; + + + /* + * Set initial state of AIOCB + */ + + CUSFS_RET_AIOCB(aiocbp,-1,EINPROGRESS); + + aiocbp64.aio_fildes = aiocbp->aio_fildes; + aiocbp64.aio_buf = aiocbp->aio_buf; + aiocbp64.aio_nbytes = aiocbp->aio_nbytes; + aiocbp64.aio_offset = aiocbp->aio_offset; + aiocbp64.aio_sigevent = aiocbp->aio_sigevent; +#ifdef _AIX + if (aiocbp->aio_handle == NULL) { + + aiocbp64.aio_handle = aiocbp; + } else { + aiocbp64.aio_handle = aiocbp->aio_handle; + } + aiocbp64.aio_word1 = CUSFS_AIOCB32_FLG; +#else + if (aiocbp->__next_prio == NULL) { + + aiocbp64.__next_prio = aiocbp; + } else { + aiocbp64.__next_prio = aiocbp->__next_prio; + } + + aiocbp64.__glibc_reserved[0] = 0; +#endif + + + return (cusfs_aio_write64(&aiocbp64)); +} + +/* + * NAME: cusfs_aio_error64 + * + * FUNCTION: Get error status for the specified async I/O + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_error64(struct aiocb64 *aiocbp) +{ + + int rc = 0; + struct aiocb64 *aiocbp2; + +#ifdef _AIX + if (aiocbp->aio_handle == NULL) { + + aiocbp2 = aiocbp; + } else { + aiocbp2 = (struct aiocb64 *)aiocbp->aio_handle; + } + rc = aiocbp2->aio_errno; +#else + if (aiocbp->__next_prio == NULL) { + + aiocbp2 = aiocbp; + } else { + aiocbp2 = (struct aiocb64 *)aiocbp->__next_prio; + } + + rc = aiocbp2->__error_code; +#endif + + + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + +/* + * NAME: cusfs_aio_error + * + * FUNCTION: Get error status for the specified async I/O + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_error(struct aiocb *aiocbp) +{ + + int rc = 0; + struct aiocb *aiocbp2; + +#ifdef _AIX + if (aiocbp->aio_handle == NULL) { + + aiocbp2 = aiocbp; + } else { + aiocbp2 = aiocbp->aio_handle; + } + rc = aiocbp2->aio_errno; +#else + if (aiocbp->__next_prio == NULL) { + + aiocbp2 = aiocbp; + } else { + aiocbp2 = aiocbp->__next_prio; + } + + rc = aiocbp2->__error_code; +#endif + + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + +/* + * NAME: cusfs_aio_return64 + * + * FUNCTION: Return completion status for an aiocb + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_return64(struct aiocb64 *aiocbp) +{ + int rc = 0; + struct aiocb64 *aiocbp2; + +#ifdef _AIX + if (aiocbp->aio_handle == NULL) { + + aiocbp2 = aiocbp; + } else { + aiocbp2 = (struct aiocb64 *)aiocbp->aio_handle; + } + errno = aiocbp2->aio_errno; + rc = aiocbp->aio_return; +#else + if (aiocbp->__next_prio == NULL) { + + aiocbp2 = aiocbp; + } else { + aiocbp2 = (struct aiocb64 *)aiocbp->__next_prio; + } + + errno = aiocbp2->__error_code; + rc = aiocbp->__return_value; +#endif + + + + if (rc == 0) { + + CUSFS_TRACE_LOG_FILE(1,"rc = %d, errno = %d", + rc, errno); + } + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return rc; +} + +/* + * NAME: cusfs_aio_return + * + * FUNCTION: Return completion status for an aiocb + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_return(struct aiocb *aiocbp) +{ + int rc = 0; + struct aiocb *aiocbp2; + +#ifdef _AIX + if (aiocbp->aio_handle == NULL) { + + aiocbp2 = aiocbp; + } else { + aiocbp2 = aiocbp->aio_handle; + } + errno = aiocbp2->aio_errno; + rc = aiocbp->aio_return; +#else + if (aiocbp->__next_prio == NULL) { + + aiocbp2 = aiocbp; + } else { + aiocbp2 = aiocbp->__next_prio; + } + + errno = aiocbp2->__error_code; + rc = aiocbp->__return_value; +#endif + + + + CUSFS_TRACE_LOG_FILE(5,"rc = %d, errno = %d", + rc, errno); + return (rc); +} + +/* + * NAME: cusfs_aio_fsync64 + * + * FUNCTION: Write changes in a file to permanent storage + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_fsync64(int op,struct aiocb64 *aiocbp) +{ + + return(cusfs_fsync(aiocbp->aio_fildes)); + +} + +/* + * NAME: cusfs_aio_fsync + * + * FUNCTION: Write changes in a file to permanent storage + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_fsync(int op,struct aiocb *aiocbp) +{ + + return(cusfs_fsync(aiocbp->aio_fildes)); + +} + + +/* + * NAME: cusfs_aio_cancel64 + * + * FUNCTION: Cancel I/O operations for an aiocb. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_cancel64(int fd, struct aiocb64 *aiocbp) +{ + + /* + * Since we have no way to cancel requests, + * we just want for them to be synced to the disk. + */ + return (cusfs_fsync(fd)); +} + + +/* + * NAME: cusfs_aio_cancel + * + * FUNCTION: Cancel I/O operations for an aiocb. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_cancel(int fd, struct aiocb *aiocbp) +{ + + /* + * Since we have no way to cancel requests, + * we just want for them to be synced to the disk. + */ + return (cusfs_fsync(fd)); +} + +/* + * NAME: cusfs_aio_suspend64 + * + * FUNCTION: Wait for at least one of the operations to complete + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_suspend64(const struct aiocb64 *const list[], int nent, + const struct timespec *timeout) +{ + int i; + int rc = 0; + int retry = 0; + int done = FALSE; + + CUSFS_TRACE_LOG_FILE(5,"nent = %d",nent); + + for (i= 0; i < nent; i++) { + + if (list[i] != NULL) { + + + while (retry < CFLSH_USFS_WAIT_AIO_RETRY) { + rc = cusfs_aio_error64((struct aiocb64 *)list[i]); + if (!rc) { + + done = TRUE; + break; + } + + if (rc < 0) { + + break; + } + + retry++; + usleep(CFLSH_USFS_WAIT_AIO_DELAY); + } + + if (done) { + + break; + } + } + } + + + CUSFS_TRACE_LOG_FILE(5,"nent = %d, rc = %d",nent,rc); + return rc; +} + + +/* + * NAME: cusfs_aio_suspend + * + * FUNCTION: Wait for at least one of the operations to complete + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_aio_suspend(const struct aiocb *const list[], int nent, + const struct timespec *timeout) +{ + int i; + int rc = 0; + int retry = 0; + int done = FALSE; + + CUSFS_TRACE_LOG_FILE(5,"nent = %d",nent); + for (i= 0; i < nent; i++) { + + if (list[i] != NULL) { + + while (retry < CFLSH_USFS_WAIT_AIO_RETRY) { + rc = cusfs_aio_error((struct aiocb *)list[i]); + if (!rc) { + + done = TRUE; + break; + } + retry++; + usleep(CFLSH_USFS_WAIT_AIO_DELAY); + } + + if (done) { + + break; + } + } + } + + + + CUSFS_TRACE_LOG_FILE(5,"nent = %d, rc = %d",nent,rc); + + return rc; +} + diff --git a/src/usfs/cflsh_usfs_client.c b/src/usfs/cflsh_usfs_client.c new file mode 100644 index 00000000..b792c508 --- /dev/null +++ b/src/usfs/cflsh_usfs_client.c @@ -0,0 +1,948 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/master/mclient.c $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ + + +#define CFLSH_USFS_FILENUM 0x0500 + +#include "cflsh_usfs_internal.h" +#include "cflsh_usfs_protos.h" + + +/* Two cflsh_usfs_hndl_priv_t instances must not share any data, including + the fixed caller supplied data that is identical for all duplicated + handles. Sharing implies locking which the design prefers to avoid. + Different master handle can be run in parallel in different client threads + w/o locks or serialization. + + One cflsh_usfs_hndl_priv_t corresponds to one client connection to the server. + Client code does not care/enforce how many connections can be made. + It is for server to enforce. + + A connection is bound to a during registration. Multiple + connections (master handles) can be bound to the same process. + + This implementation does not prevent a child using a cflsh_usfs_hndl inherited + from its parent. Doing so is considered a programming error and the + results are undefined. +*/ +typedef struct +{ + int conn_fd; + uint8_t tag; + + /* save pid of registrant for limited special handling of child after + a fork */ + pid_t pid; + + /* save off caller supplied parms for mc_hdup etc */ +// ctx_hndl_t ctx_hndl; + char master_dev_path[MC_PATHLEN]; + +} cflsh_usfs_hndl_priv_t; + +#ifdef _MASTER_LOCK +/* + * NAME:cusfs_key_gen() + * + * FUNCTION: + * cusfs_key_gen() will generate key with help + * The key will be unique to each disk, resources. + * RETURN: + * + */ + +key_t cusfs_key_gen( char * file_name , cflsh_usfs_master_lock_t cufs_lock ) +{ + key_t key_value; + + key_value = ftok(file_name, cufs_lock ); + + if ( (key_t)-1 == key_value ) + { + CUSFS_TRACE_LOG_FILE(1, "%s: ftok() failed, errno %d\n", file_name,errno); + return (-1); + } + + return key_value; +} + +/* + * NAME:cusfs_create_sem() + * + * FUNCTION: + * cusfs_create_sem() will create semphore if it does not exist + * Or get the semaphore for existing one. + * On successful case, it will return semid to its caller. + * RETURN: + * + */ + +int cusfs_create_sem( key_t cufs_sem_key ) +{ + int rc; + + int cufs_sem_id; + + cufs_sem_id = semget( cufs_sem_key , 1 , S_IRUSR | + S_IWUSR | S_IRGRP | S_IWGRP | IPC_CREAT | IPC_EXCL); + if ( -1 == cufs_sem_id ) + { + /* It is already created by other process; + So Just need to get semid now */ + + cufs_sem_id = semget ( cufs_sem_key , 1 , S_IRUSR | + S_IWUSR | S_IRGRP | S_IWGRP ); + /* if failed to get semid ; then inform caller */ + if ( -1 == cufs_sem_id ) + return -1; + } + + else + { + /* Initialize the semaphore ; + Process created the sem for First time*/ + + rc = semctl(cufs_sem_id,0,SETVAL,1); + if(rc == -1) + { + return -1 ; + } + } + + return cufs_sem_id; +} + +/* + * NAME:cusfs_rm_sem() + * + * FUNCTION: + * cusfs_rm_sem() will delete semphore. + * On successful case, it will return semid to its caller. + * RETURN: + * + */ + +int cusfs_rm_sem( int cufs_sem_id ) +{ + int rc = 0; + + rc = semctl( cufs_sem_id, 1, IPC_RMID ); + if(rc == -1) + { + //TRACE_1 + return -1 ; + } + + return rc; +} + +/* + * NAME:cusfs_cleanup_sem() + * + * FUNCTION: + * cusfs_cleanup_sem() will cleanup all the semphore. + * On successful case, it will return 0 else -1. + * RETURN: + * + */ + +int cusfs_cleanup_sem ( cflsh_usfs_master_hndl_t cflsh_usfs_hndl ) +{ + int rc = 0; + cusfs_serv_sem_t *p_disk = ( cusfs_serv_sem_t *) cflsh_usfs_hndl; + + rc = cusfs_rm_sem( p_disk->fs_sem.cusfs_fs_semid ); + if ( rc == -1 ) + { + goto x_err; + } + rc = cusfs_rm_sem( p_disk->frtb_sem.cusfs_frtb_semid ); + if ( rc == -1 ) + { + goto x_err; + } + rc = cusfs_rm_sem( p_disk->intb_sem.cusfs_intb_semid ); + if ( rc == -1 ) + { + goto x_err; + } + rc = cusfs_rm_sem( p_disk->jrn_sem.cusfs_jrn_semid ); + if ( rc == -1 ) + { + goto x_err; + } + +x_err: + return rc; /* Need to improve the return code later */ +} + +#endif + +#ifdef _MASTER_LOCK_CLIENT +/* + * Procedure: uint64_t gen_rand + * + * Description: Generate random number + * + * + * Parameters: + * + * + * Return: + */ +static uint64_t gen_rand(cflsh_usfs_hndl_priv_t *p_cflsh_usfs_hndl_priv) +{ + uint64_t rand; + uint64_t p_low; + + asm volatile ( "mfspr %0, 268" : "=r"(rand) : ); // time base + + // add in low 32 bits of malloc'ed address to top 32 bits of TB + p_low = (((uint64_t)p_cflsh_usfs_hndl_priv) & 0xFFFFFFFF); + rand ^= (p_low << 32); + + return rand; +} + +/* + * Procedure: xfer_data + * + * Description: Perform a transfer operation for the given + * socket file descriptor. + * + * Parameters: + * fd: Socket File Descriptor + * op: Read or Write Operation + * buf: Buffer to either read from or write to + * exp_size: Size of data transfer + * + * Return: 0, if successful + * non-zero otherwise + */ +static int +xfer_data(int fd, int op, void *buf, ssize_t exp_size) +{ + int rc = 0; + ssize_t offset = 0; + ssize_t bytes_xfer = 0; + ssize_t target_size = exp_size; + struct iovec iov; + struct msghdr msg; + + while ( 1 ) + { + // Set up IO vector for IO operation. + memset(&msg, 0, sizeof(struct msghdr)); + iov.iov_base = buf + offset; + iov.iov_len = target_size; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + // Check to see if we are sending or receiving data + if ( op == XFER_OP_READ ) + { + bytes_xfer = recvmsg(fd, &msg, MSG_WAITALL); + } + else + { + bytes_xfer = sendmsg(fd, &msg, MSG_NOSIGNAL); + } + + if ( -1 == bytes_xfer ) + { + if ( EAGAIN == errno || EWOULDBLOCK == errno || EINTR == errno) + { + // just retry the whole request + continue; + } + else + { + // connection closed by the other end + rc = 1; + break; + } + } + else if ( 0 == bytes_xfer ) + { + // connection closed by the other end + rc = 1; + break; + } + else if ( bytes_xfer == target_size ) + { + // We have transfered all the bytes we wanted, we + // can stop now. + rc = 0; + break; + } + else + { + // less than target size - partial condition + // set up to transfer for the remainder of the request + offset += bytes_xfer; + target_size = (target_size - bytes_xfer); + } + } + + return rc; +} + + +/***************************************************************************** + * Procedure: blk_connect + * + * Description: Connect to the server entity + * + * Parameters: + * + * Return: 0 or greater is the file descriptor for the connection + * -1 is error + *****************************************************************************/ +static int +blk_connect(char *mdev_path) +{ + struct sockaddr_un svr_addr; + int conn_fd; + int rc; + int retry; + char *env_master_socket = getenv("CFLSH_USFS_MASTER_SOCKET"); + + + // Create a socket file descriptor + conn_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (conn_fd < 0) + { + CUSFS_TRACE_LOG_FILE(1,"blk_connect: socket failed: %d (%d)\n", + conn_fd, errno); + } + else + { + bzero(&svr_addr, sizeof(struct sockaddr_un)); + svr_addr.sun_family = AF_UNIX; + + if (env_master_socket) { + strcpy(svr_addr.sun_path, env_master_socket); + } else { + strcpy(svr_addr.sun_path, CFLSH_USFS_MASTER_SOCKET_DIR); + } + strcat(svr_addr.sun_path, mdev_path); + + // Connect to the server entity + for (retry = 0; retry < 3; retry++) { + // retry to handle ECONNREFUSED, there may be too many pending + // connections on the server (SOMAXCONN). + // + rc = connect(conn_fd, (struct sockaddr *)&svr_addr, sizeof(svr_addr)); + if (rc == 0) { + break; + } + usleep(100); + } + + if (rc) { + CUSFS_TRACE_LOG_FILE(1,"block_connect: Connect failed: %d (%d)\n", + rc, errno); + close(conn_fd); + conn_fd = -1; + } + } + + return conn_fd; +} + +/***************************************************************************** + * Procedure: blk_send_command + * + * Description: Send a command to the server entity + * + * Parameters: + * conn_fd: Socket File Descriptor + * cmd: Command to perform + * cmdblock: Command request block + * cmdresp: Command response block + * + * Return: 0, if successful + * non-zero otherwise + *****************************************************************************/ +static int +blk_send_command(int conn_fd, cflsh_usfs_master_req_t *p_mc_req, cflsh_usfs_master_resp_t *p_mc_resp) +{ + int rc = 0; + + // Send the command block + rc = xfer_data(conn_fd, XFER_OP_WRITE, (void *)p_mc_req, sizeof(*p_mc_req)); + if (rc) + { + CUSFS_TRACE_LOG_FILE(1,"blk_send_command: PID %d failed send: %d\n", + getpid(), rc); + } + else + { + // Wait for the response + rc = xfer_data(conn_fd, XFER_OP_READ, (void *)p_mc_resp, sizeof(*p_mc_resp)); + if (rc) + { + CUSFS_TRACE_LOG_FILE(1, "blk_send_command: PID %d failed read: %d\n", + getpid(), rc); + } + } + + return rc; +} + + +// backend function used by cflsh_usfs_master_register after making a connection +// to server. +// +static int +cflsh_usfs_master_register_back(cflsh_usfs_master_hndl_t cflsh_usfs_hndl, + int mode) + +{ + cflsh_usfs_master_req_t mc_req; + cflsh_usfs_master_resp_t mc_resp; + uint64_t challenge; + int rc; + + cflsh_usfs_hndl_priv_t *p_cflsh_usfs_hndl_priv = (cflsh_usfs_hndl_priv_t *) cflsh_usfs_hndl; + + challenge = gen_rand(p_cflsh_usfs_hndl_priv); + + + memset(&mc_req, '\0', sizeof(mc_req)); + memset(&mc_resp, '\0', sizeof(mc_resp)); + mc_req.header.size = sizeof(mc_req); + mc_resp.header.size = sizeof(mc_resp); + mc_req.header.tag = p_cflsh_usfs_hndl_priv->tag++; + + mc_req.header.command = CFLSH_USFS_CMD_MASTER_REG; + mc_req.reg.client_pid = getpid(); + mc_req.reg.client_fd = p_cflsh_usfs_hndl_priv->conn_fd; + mc_req.reg.mode = mode; + mc_req.reg.challenge = challenge; + + rc = blk_send_command(p_cflsh_usfs_hndl_priv->conn_fd, &mc_req, &mc_resp); + + + if (rc != 0) { + CUSFS_TRACE_LOG_FILE(1,"failed - transport error rc: %d, errno = %d\n", rc, errno); + errno = EIO; // transport error + rc = -1; + } + else if (mc_resp.header.tag != mc_req.header.tag) { + CUSFS_TRACE_LOG_FILE(1,"failed - tag mismatch: exp(%d), actual(%d)\n", mc_req.header.tag, mc_resp.header.tag); + errno = EIO; // this is a response for some other cmd + rc = -1; + } + else if (mc_resp.header.status != 0) { + CUSFS_TRACE_LOG_FILE(1,"failed - server errno: %d\n", mc_resp.header.status); + errno = mc_resp.header.status; + rc = -1; + } + else { + rc = 0; + } + + return rc; +} + +#endif /* _MASTER_LOCK_CLIENT */ +/* + * Procedure: cflsh_usfs_master_init + * + * Description: Init interfaces with master. + + * + * + * Return: 0, if successful + * non-zero otherwise + */ +int +cflsh_usfs_master_init() +{ + return 0; +} + +/* + * Procedure: cflsh_usfs_master_term + * + * Description: Terminate interfaces with master + * + * + * Return: 0, if successful + * non-zero otherwise + */ +int +cflsh_usfs_master_term() +{ + return 0; +} + +/* + * Procedure: cflsh_usfs_master_register + * + * Description: Register a client with the master + * + * + * Return: 0, if successful + * non-zero otherwise + */ +int +cflsh_usfs_master_register(char *master_dev_path, cflsh_usfs_master_hndl_t *p_cflsh_usfs_hndl) +{ +#ifdef _MASTER_LOCK +#ifdef _MASTER_LOCK_CLIENT + int conn_fd; + int rc; + cflsh_usfs_hndl_priv_t *p_cflsh_usfs_hndl_priv; + + conn_fd = blk_connect(master_dev_path); + if (conn_fd < 0) { + CUSFS_TRACE_LOG_FILE(1,"socket failed: %d (%d)\n", conn_fd, errno); + return -1; // errno set in blk_connect() + } + + p_cflsh_usfs_hndl_priv = (cflsh_usfs_hndl_priv_t *) malloc(sizeof(cflsh_usfs_hndl_priv_t)); + if (p_cflsh_usfs_hndl_priv == NULL) { + close(conn_fd); + CUSFS_TRACE_LOG_FILE(1,"cannot allocate client handle\n"); + errno = ENOMEM; + return -1; + } + + // init cflsh_usfs_hndl + memset(p_cflsh_usfs_hndl_priv, 0, sizeof(*p_cflsh_usfs_hndl_priv)); + p_cflsh_usfs_hndl_priv->conn_fd = conn_fd; + p_cflsh_usfs_hndl_priv->tag = 0; + p_cflsh_usfs_hndl_priv->pid = getpid(); + strncpy(p_cflsh_usfs_hndl_priv->master_dev_path, master_dev_path, + MC_PATHLEN - 1); + + + // initial MCREG of ctx_hndl (not a dup) + rc = cflsh_usfs_master_register_back(p_cflsh_usfs_hndl_priv, MCREG_INITIAL_REG); + + if (rc != 0) { + free(p_cflsh_usfs_hndl_priv); + close(conn_fd); + return rc; // errno is already set + } + + *p_cflsh_usfs_hndl = p_cflsh_usfs_hndl_priv; + + CUSFS_TRACE_LOG_FILE(9,"success - on client handle %p\n", + *p_cflsh_usfs_hndl); + return rc; + +#else /* _MASTER_LOCK_CLIENT */ + + int rc = 0; + + cusfs_serv_sem_t *p_disk =NULL ; + + if ( posix_memalign((void *)&(p_disk),4096, + sizeof( cusfs_serv_sem_t))) + { + CUSFS_TRACE_LOG_FILE(1,"cannot allocate p_disk handle \n"); + errno = ENOMEM; + return -1; + } + + strncpy(p_disk->master_dev_path, master_dev_path, MC_PATHLEN - 1); + + p_disk->fs_sem.cusfs_fs_key=cusfs_key_gen( p_disk->master_dev_path, \ + CFLSH_USFS_MASTER_FS_LOCK); + if( p_disk->fs_sem.cusfs_fs_key == -1 ) + { + CUSFS_TRACE_LOG_FILE(1,"cannot get fs key\n"); + rc =-1; + goto init_err; + } + + p_disk->frtb_sem.cusfs_frtb_key = cusfs_key_gen( p_disk->master_dev_path, \ + CFLSH_USFS_MASTER_FREE_TABLE_LOCK); + if ( p_disk->frtb_sem.cusfs_frtb_key == -1 ) + { + rc = -1; + CUSFS_TRACE_LOG_FILE(1,"cannot get free table key\n"); + goto init_err; + } + + p_disk->intb_sem.cusfs_intb_key = cusfs_key_gen( p_disk->master_dev_path, \ + CFLSH_USFS_MASTER_INODE_TABLE_LOCK ); + if ( p_disk->intb_sem.cusfs_intb_key == -1 ) + { + rc = -1; + CUSFS_TRACE_LOG_FILE(1,"cannot get intb key\n"); + goto init_err; + } + + p_disk->jrn_sem.cusfs_jrn_key = cusfs_key_gen( p_disk->master_dev_path, \ + CFLSH_USFS_MASTER_JOURNAL_LOCK ); + if ( p_disk->jrn_sem.cusfs_jrn_key == -1 ) + { + rc = -1; + CUSFS_TRACE_LOG_FILE(1,"cannot get jrn key\n"); + goto init_err; + } + + p_disk->fs_sem.cusfs_fs_semid = cusfs_create_sem( p_disk->fs_sem.cusfs_fs_key ) ; + if ( p_disk->fs_sem.cusfs_fs_semid == -1 ) + { + rc = -1; + goto init_err; + } + + p_disk->frtb_sem.cusfs_frtb_semid = cusfs_create_sem( p_disk->frtb_sem.cusfs_frtb_key ) ; + if ( p_disk->frtb_sem.cusfs_frtb_semid == -1 ) + { + rc = -1; + goto init_err; + } + + p_disk->intb_sem.cusfs_intb_semid = cusfs_create_sem( p_disk->intb_sem.cusfs_intb_key ) ; + if ( p_disk->intb_sem.cusfs_intb_semid == -1 ) + { + rc =-1; + goto init_err; + } + + p_disk->jrn_sem.cusfs_jrn_semid = cusfs_create_sem( p_disk->jrn_sem.cusfs_jrn_key ) ; + if( p_disk->jrn_sem.cusfs_jrn_semid == -1 ) + { + rc =-1; + goto init_err; + } +init_err: + *p_cflsh_usfs_hndl = p_disk; + return rc; + +#endif // END of MASTER_LOCK_CLIENT +#else + return 0; +#endif // END of MASTER_LOCK +} + + + +/* + * Procedure: cflsh_usfs_master_unregister + * + * Description: Unregister a client with the master + * + * + * Return: 0, if successful + * non-zero otherwise + */ +int cflsh_usfs_master_unregister(cflsh_usfs_master_hndl_t cflsh_usfs_hndl) +{ +#ifdef _MASTER_LOCK_CLIENT + cflsh_usfs_master_req_t mc_req; + cflsh_usfs_master_resp_t mc_resp; + int rc = 0; + cflsh_usfs_hndl_priv_t *p_cflsh_usfs_hndl_priv = (cflsh_usfs_hndl_priv_t *) cflsh_usfs_hndl; + + /* Unregister with server if this is called by parent i.e. the + original registrant */ + if (p_cflsh_usfs_hndl_priv-> pid == getpid()) { + memset(&mc_req, '\0', sizeof(mc_req)); + memset(&mc_resp, '\0', sizeof(mc_resp)); + mc_req.header.size = sizeof(mc_req); + mc_resp.header.size = sizeof(mc_resp); + mc_req.header.tag = p_cflsh_usfs_hndl_priv->tag++; + + mc_req.header.command = CFLSH_USFS_CMD_MASTER_UNREG; + + rc = blk_send_command(p_cflsh_usfs_hndl_priv->conn_fd, &mc_req, &mc_resp); + + if (rc != 0) { + CUSFS_TRACE_LOG_FILE(1,"failed - transport error rc: %d\n", rc); + errno = EIO; // transport error + rc = -1; + } + else if (mc_resp.header.tag != mc_req.header.tag) { + CUSFS_TRACE_LOG_FILE(1,"failed - tag mismatch: exp(%d), actual(%d)\n", + mc_req.header.tag, mc_resp.header.tag); + errno = EIO; // this is a response for some other cmd + rc = -1; + } + else if (mc_resp.header.status != 0) { + CUSFS_TRACE_LOG_FILE(1,"failed - server errno: %d\n", + mc_resp.header.status); + errno = mc_resp.header.status; + rc = -1; + } + else { + rc = 0; + + CUSFS_TRACE_LOG_FILE(9,"success - unregisterd client handle %p\n", cflsh_usfs_hndl); + + } + } + + close(p_cflsh_usfs_hndl_priv->conn_fd); + free(p_cflsh_usfs_hndl_priv); + return rc; +#else + return 0; +#endif /* !_MASTER_LOCK_CLIENT */ + +} + +/* + * Procedure: cflsh_usfs_master_lock + * + * Description: Request the specified shared lock. + * + * + * Return: 0, if successful + * non-zero otherwise + */ +int cflsh_usfs_master_lock(cflsh_usfs_master_hndl_t cflsh_usfs_hndl, + cflsh_usfs_master_lock_t lock, + uint64_t flags) +{ +#ifdef _MASTER_LOCK +#ifdef _MASTER_LOCK_CLIENT + cflsh_usfs_master_req_t mc_req; + cflsh_usfs_master_resp_t mc_resp; + int rc; + cflsh_usfs_hndl_priv_t *p_cflsh_usfs_hndl_priv = (cflsh_usfs_hndl_priv_t *) cflsh_usfs_hndl; + + memset(&mc_req, '\0', sizeof(mc_req)); + memset(&mc_resp, '\0', sizeof(mc_resp)); + mc_req.header.size = sizeof(mc_req); + mc_resp.header.size = sizeof(mc_resp); + mc_req.header.tag = p_cflsh_usfs_hndl_priv->tag++; + + mc_req.header.command = CFLSH_USFS_CMD_MASTER_LOCK; + mc_req.lock.lock = lock; + mc_req.lock.flags = flags; + + rc = blk_send_command(p_cflsh_usfs_hndl_priv->conn_fd, &mc_req, &mc_resp); + + if (rc != 0) { + CUSFS_TRACE_LOG_FILE(1,"failed - transport error rc: %d\n",rc); + errno = EIO; // transport error + rc = -1; + } + else if (mc_resp.header.tag != mc_req.header.tag) { + CUSFS_TRACE_LOG_FILE(1,"failed - tag mismatch: exp(%d), actual(%d)\n", + mc_req.header.tag, mc_resp.header.tag); + errno = EIO; // this is a response for some other cmd + rc = -1; + } + else if (mc_resp.header.status != 0) { + CUSFS_TRACE_LOG_FILE(1,"failed - server errno: %d\n", + mc_resp.header.status); + errno = mc_resp.header.status; + rc = -1; + } + else { + rc = 0; + + + CUSFS_TRACE_LOG_FILE(9,"success - locked %d\n", lock); + + } + + return rc; +#else + + int rc = 0; + int lock_semid = 0; + struct sembuf cflash_sembuf; + + cusfs_serv_sem_t *p_disk = ( cusfs_serv_sem_t *) cflsh_usfs_hndl; + + switch ( lock ) + { + case CFLSH_USFS_MASTER_FS_LOCK: + lock_semid = p_disk->fs_sem.cusfs_fs_semid; + break; + + case CFLSH_USFS_MASTER_FREE_TABLE_LOCK: + lock_semid = p_disk->frtb_sem.cusfs_frtb_semid; + break; + + case CFLSH_USFS_MASTER_INODE_TABLE_LOCK: + lock_semid = p_disk->intb_sem.cusfs_intb_semid; + break; + + case CFLSH_USFS_MASTER_JOURNAL_LOCK: + lock_semid = p_disk->jrn_sem.cusfs_jrn_semid; + break; + + default: + /* No lock sem registered */ + break; + } + + cflash_sembuf.sem_num = 0; + cflash_sembuf.sem_op = -1; + cflash_sembuf.sem_flg = IPC_NOWAIT; + + rc = semop(lock_semid, &cflash_sembuf, 1); + + if ( ( rc != 0) && ( errno == EINTR || errno == EAGAIN )) + { + while(1) + { + rc = semop(lock_semid, &cflash_sembuf, 1); + if(rc ==0) + { + break; + } + } + } + + if(rc ==0) + CUSFS_TRACE_LOG_FILE(9,"success - locked %d\n", lock); + + + return rc; +#endif /* !_MASTER_LOCK_CLIENT */ +#else + return 0; +#endif /* !_MASTER_LOCK */ +} + + +/* + * Procedure: cflsh_usfs_master_unlock + * + * Description: Request to release a shared lock + * + * + * Return: 0, if successful + * non-zero otherwise + */ +int +cflsh_usfs_master_unlock(cflsh_usfs_master_hndl_t cflsh_usfs_hndl, + cflsh_usfs_master_lock_t lock, + uint64_t flags) +{ +#ifdef _MASTER_LOCK +#ifdef _MASTER_LOCK_CLIENT + cflsh_usfs_master_req_t mc_req; + cflsh_usfs_master_resp_t mc_resp; + int rc; + cflsh_usfs_hndl_priv_t *p_cflsh_usfs_hndl_priv = (cflsh_usfs_hndl_priv_t *) cflsh_usfs_hndl; + + memset(&mc_req, '\0', sizeof(mc_req)); + memset(&mc_resp, '\0', sizeof(mc_resp)); + mc_req.header.size = sizeof(mc_req); + mc_resp.header.size = sizeof(mc_resp); + mc_req.header.tag = p_cflsh_usfs_hndl_priv->tag++; + + mc_req.header.command = CFLSH_USFS_CMD_MASTER_UNLOCK; + mc_req.lock.lock = lock; + mc_req.lock.flags = flags; + + rc = blk_send_command(p_cflsh_usfs_hndl_priv->conn_fd, &mc_req, &mc_resp); + + if (rc != 0) { + CUSFS_TRACE_LOG_FILE(1,"failed - transport error rc: %d\n", rc); + errno = EIO; // transport error + rc = -1; + } + else if (mc_resp.header.tag != mc_req.header.tag) { + CUSFS_TRACE_LOG_FILE(1,"failed - tag mismatch: exp(%d), actual(%d)\n", + mc_req.header.tag, mc_resp.header.tag); + errno = EIO; // this is a response for some other cmd + rc = -1; + } + else if (mc_resp.header.status != 0) { + CUSFS_TRACE_LOG_FILE(1,"failed - server errno: %d\n", + mc_resp.header.status); + errno = mc_resp.header.status; + rc = -1; + } + else { + rc = 0; + + CUSFS_TRACE_LOG_FILE(9,"success - unlocked %d\n", lock); + + } + + return rc; +#else + int rc = 0; + int lock_semid = 0; + struct sembuf cflash_sembuf; + + cusfs_serv_sem_t *p_disk = ( cusfs_serv_sem_t *) cflsh_usfs_hndl; + + switch ( lock ) + { + case CFLSH_USFS_MASTER_FS_LOCK: + lock_semid = p_disk->fs_sem.cusfs_fs_semid; + break; + + case CFLSH_USFS_MASTER_FREE_TABLE_LOCK: + lock_semid = p_disk->frtb_sem.cusfs_frtb_semid; + break; + + case CFLSH_USFS_MASTER_INODE_TABLE_LOCK: + lock_semid = p_disk->intb_sem.cusfs_intb_semid; + break; + + case CFLSH_USFS_MASTER_JOURNAL_LOCK: + lock_semid = p_disk->jrn_sem.cusfs_jrn_semid; + break; + + default: + /* No lock sem registered */ + break; + + } + cflash_sembuf.sem_num = 0; + cflash_sembuf.sem_op = 1; + cflash_sembuf.sem_flg = IPC_NOWAIT; + + rc = semop(lock_semid, &cflash_sembuf, 1); + + if ( ( rc != 0) && ( errno == EINTR || errno == EAGAIN )) + { + while(1) + { + rc = semop(lock_semid, &cflash_sembuf, 1); + if(rc ==0) + { + break; + } + } + } + + if(rc ==0) + CUSFS_TRACE_LOG_FILE(9,"success - unlocked %d\n", lock); + + return rc; + +#endif /* !_MASTER_LOCK_CLIENT */ +#else + return 0; /* !_MASTER_LOCK */ +#endif +} diff --git a/src/usfs/cflsh_usfs_client.h b/src/usfs/cflsh_usfs_client.h new file mode 100644 index 00000000..0dad3afb --- /dev/null +++ b/src/usfs/cflsh_usfs_client.h @@ -0,0 +1,359 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/include/mclient.h $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#ifndef _H_CFLASH_USFS_CLIENT +#define _H_CFLASH_USFS_CLIENT + + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + + +/*************************************************************************** + This file defines IPC messages to the master context deamon. + + The daemon listens on a unique per AFU UNIX socket. The + name of the socket can be derived from the disk as follows: + disk_device = /dev/sg4 + corresponding socket = CFLSH_USFS_MASTER_SOCKET_DIR/dev/sg4 + If CFLSH_USFS_MASTER_SOCKET_DIR is /tmp, the socket is /tmp/dev/sg4 + + A client wishing to do direct IPC with the master selects the + correct socket name to connect to based on the disk. +***************************************************************************/ + +#define CFLSH_USFS_MASTER_SOCKET_DIR "/opt/ibm/capikv/data" + +#define XFER_OP_READ 1 // Read from the Blk-MC Socket +#define XFER_OP_WRITE 2 // Write to the Blk-MC Socket + +/* Commad opcodes in IPC requests */ +#define CFLSH_USFS_CMD_MASTER_REG 1 // Register an filesystem + context handle with master +#define CFLSH_USFS_CMD_MASTER_UNREG 2 // Unregister a context handle with master + +#define CFLSH_USFS_CMD_MASTER_LOCK 3 // Acquire a shared lock for some filesystem critical resource + // (i.e. free block table, inode table, journal etc). +#define CFLSH_USFS_CMD_MASTER_UNLOCK 4 // Release a shared lock for some filesystem critical resource + + +typedef enum { + CFLSH_USFS_MASTER_FS_LOCK = 1, /* Filesystem lock */ + CFLSH_USFS_MASTER_FREE_TABLE_LOCK = 2, /* Free block table lock */ + CFLSH_USFS_MASTER_INODE_TABLE_LOCK = 3, /* Inode table lock */ + CFLSH_USFS_MASTER_JOURNAL_LOCK = 4, /* Journal lock */ + CFLSH_USFS_MASTER_LAST_ITEM = 5, /* Journal lock */ +} cflsh_usfs_master_lock_t; + + + +typedef struct cflsh_master_req_header +{ + int version; // 0 + int command; + int size; + int tag; // command tag to identify the active request +} cflsh_usfs_master_req_header_t; + + +typedef struct cflsh_usfs_master_resp_header +{ + int version; // 0 + int command; // same as command in the request + int size; + int tag; // same as tag in request + int status; // 0: success, otherwise set to a errno value +} cflsh_usfs_master_resp_header_t; + +typedef struct master_req +{ + cflsh_usfs_master_req_header_t header; + + union { + // The client sends a MASTER_REG with the challenge field set to something + // the server can use to validate that the client is the true owner of + // the AFU context it wants to register. The specifics of this protocol + // is left to the implementation. + // + // Since the server remembers the disk (implicitly tied to connection) and + // the registered ctx_hndl, these are not needed to be sent again in + // subsequent IPC messages. + // + // MASTER_REG must be the first command on a new connection. If the registation + // fails, the MCREG can be retried on the same connection any number of + // times. Once successful, subsequent MCREGs are failed until after a + // MCUNREG. + // + struct { + pid_t client_pid; + int client_fd; + int mode; +#define MCREG_INITIAL_REG 0x1 // fresh registration +#define MCREG_DUP_REG 0x0 // dup +// ctx_hndl_t ctx_hndl; + uint64_t challenge; + } reg; + + // After MCUNREG, the client can send a new MCREG on the existing + // connection or close it. + // + struct { + } unreg; + + struct { + cflsh_usfs_master_lock_t lock; + uint64_t flags; + } lock; + + + }; +} cflsh_usfs_master_req_t; + +typedef struct mc_resp { + cflsh_usfs_master_resp_header_t header; + + union { + struct { + } reg; + + struct { + } unreg; + + struct { + int foo; +// res_hndl_t res_hndl; + } open; + + struct { + } close; + + struct { + } dup; + + }; + +} cflsh_usfs_master_resp_t; + +/**************************************************************************** + * Each user process needs two interfaces to use the AFU: + * + * 1. a host transport MMIO map to issue commands directly to the AFU. + * The client opens the regular AFU device, registers its process + * context with the AFU driver (CXL_IOCTL_START_WORK ioctl) and + * then mmaps a 64KB host transport that has registers to issue + * commands and setup responses. + * + * 2. an interface to the master context daemon via a cflsh_usfs_master_hndl_t. + * This is used to create & size resource handles which are equivalent + * to virtual disks. The resource handle is written into the command + * block (IOARCB) sent to the AFU using interface #1. + * + * This header file describes the second interface. See sislite.h for the + * first interface. + * + * User must create interface #1 first. Parameters returned by #1 are + * required to establish interface #2. + * + * All routines described here return 0 on success and -1 on failure. + * errno is set on failure. + ***************************************************************************/ + +/* Max pathlen - e.g. for AFU device path */ +#define MC_PATHLEN 64 + +/* Permission Flags */ +#define MC_RDONLY 0x1u +#define MC_WRONLY 0x2u +#define MC_RDWR 0x3u + +/* cflsh_usfs_master_hndl_t is the client end point for communication to the + * master. After a fork(), the child must call mc_unregister + * on any inherited mc handles. It is a programming error to pass + * an inherited mc handle to any other API in the child. + */ +typedef void* cflsh_usfs_master_hndl_t; + + +/* mc_init & mc_term are called once per process */ +int cflsh_usfs_master_init(); +int cflsh_usfs_master_term(); + +/* master lock initilization data */ + +#ifdef _MASTER_LOCK + +typedef struct +{ + key_t cusfs_fs_key; + int cusfs_fs_semid ; +} cusfs_fs_sem_t ; + +typedef struct +{ + key_t cusfs_frtb_key; + int cusfs_frtb_semid ; +} cusfs_frtb_sem_t ; + +typedef struct +{ + key_t cusfs_intb_key; + int cusfs_intb_semid ; +} cusfs_intb_sem_t ; + + +typedef struct +{ + key_t cusfs_jrn_key; + int cusfs_jrn_semid ; +} cusfs_jrn_sem_t ; + +typedef struct cusfs_serv_sem_s +{ + char master_dev_path[MC_PATHLEN]; + char *name; + cusfs_fs_sem_t fs_sem ; + cusfs_frtb_sem_t frtb_sem; + cusfs_intb_sem_t intb_sem; + cusfs_jrn_sem_t jrn_sem; + + /* client IPC */ + +} __attribute__ ((aligned (0x1000))) cusfs_serv_sem_t; + +key_t cusfs_key_gen( char * file_name , cflsh_usfs_master_lock_t cufs_lock); + +int cusfs_create_sem( key_t cufs_sem_key ); + +int cusfs_rm_sem( int cufs_sem_id ); + +int cusfs_cleanup_sem ( cflsh_usfs_master_hndl_t cflsh_usfs_hndl ); + +#endif + +/* + * master_register outputs a client handle (cflsh_usfs_master_hndl_t) when successful. + * The handle is bound to the specified AFU and context handle. + * Subsequent API calls using the handle apply to that context + * and that AFU. The specified context handle must be owned by the + * requesting user process. + * + * The cflsh_usfs_master_hndl_t is a client endpoint for communication with the master + * context. It can be used to set up resources only for the single context + * on a single AFU bound to the mc handle. + * + * Users opening multiple AFU contexts must create multiple mc handles by + * registering each context. + * + * Client is responsible for serialization when the same mc handle is + * shared by multiple threads. + * + * Different threads can work on different mc handles in parallel w/o + * any serialization. + * + * It is possible to mc_register the same (AFU + ctx_hndl) multiple times + * creating multiple mc endpoints (cflsh_usfs_master_hndl) bound to the same context. + * However, to accomplish this, the mc_hdup function must be used to + * duplicate an existing cflsh_usfs_master_hndl. Calling mc_register more than once with + * the same parameters will cancel all but the most recent registation. + * + * Inputs: + * master_dev_path - e.g. /dev/cxl/afu0.0m + * The path must be to the master AFU device even if the + * user opened the corresponding regular device. + * + * Output: + * p_cflsh_usfs_master_hndl - The output cflsh_usfs_master_hndl_t is written using this pointer. + * The cflsh_usfs_master_hndl_t is passed in all subsequent calls. + * + */ +int cflsh_usfs_master_register(char *master_dev_path, cflsh_usfs_master_hndl_t *p_cflsh_usfs_master_hndl); + + +/* master_unregister invalidates the mc handle. All resource + * handles under the input mc handle are released if this cflsh_usfs_master_hndl is + * the only reference to those resources. + * + * Note that after a dup (see below), two contexts can be referencing + * the same resource handles. In that case, unregistering one context + * must not release resource handles since they are still referenced + * by the other context. + * + * Inputs: + * cflsh_usfs_master_hndl - a cflsh_usfs_master_hndl that specifies a (context + AFU) + * to unregister + * + * Output: + * none + */ +int cflsh_usfs_master_unregister(cflsh_usfs_master_hndl_t cflsh_usfs_master_hndl); + + +/* mc_open creates a zero sized virtual LBA space or a virtual disk + * on the AFU and context bound to the mc handle. + * + * Inputs: + * cflsh_usfs_master_hndl - mc handle that specifies a (context + AFU) + * + * flags - permission flags (see #defines) + * + * Output: + * p_res_hndl - A resource handle allocated by master is written using + * this pointer. The handle is valid only in the scope of + * the AFU context bound to the input cflsh_usfs_master_hndl or any + * contexts dup'ed to it. User must not mix up resource + * handles from different contexts. Resource handle A for + * AFU context 1 and handle B for AFU context 2 can have the + * same internal representation, but they refer to two + * completely different virtual LBA spaces. + */ +int cflsh_usfs_master_lock(cflsh_usfs_master_hndl_t cflsh_usfs_master_hndl, cflsh_usfs_master_lock_t lock, + uint64_t flags); + + +/* mc_close deallocates any resources (LBAs) allocated to a virtual + * disk if this context is the last reference to those resources. + * + * Inputs: + * cflsh_usfs_master_hndl - client handle that specifies a (context + AFU) + * res_hndl - resource handle identifying the virtual disk + * to close + * + * Output: + * none + */ +int cflsh_usfs_master_unlock(cflsh_usfs_master_hndl_t cflsh_usfs_master_hndl, cflsh_usfs_master_lock_t lock, + uint64_t flags); + + +#endif /* ifndef _H_CFLASH_USFS_CLIENT */ + diff --git a/src/usfs/cflsh_usfs_disk.c b/src/usfs/cflsh_usfs_disk.c new file mode 100644 index 00000000..8f827cf0 --- /dev/null +++ b/src/usfs/cflsh_usfs_disk.c @@ -0,0 +1,4155 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* bos720 src/bos/usr/ccs/lib/libcflsh_block/cflash_block.c 1.8 */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#ifdef _AIX +static char sccsid[] = "%Z%%M% %I% %W% %G% %U%"; +#endif + + + +/* + * COMPONENT_NAME: (sysxcflashusfs) CAPI Flash user space filesystem library + * + * FUNCTIONS: + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +#define CFLSH_USFS_FILENUM 0x0200 + +#include "cflsh_usfs_internal.h" +#include "cflsh_usfs_protos.h" +#include + +/* + * NAME: cusfs_generate_fs_unique_id + * + * FUNCTION: Get the disk block number for the filesystem block number. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 failure, Otherwise success + * + * + */ + +cflsh_usfs_fs_id_t cusfs_generate_fs_unique_id(void) +{ + cflsh_usfs_fs_id_t fs_serial_num; + struct utsname uname_arg; + int rc; + uint *word_ptr; + uint seed = 0; + + bzero(&fs_serial_num, sizeof(fs_serial_num)); + + + /* + * The first 64-bits will contain machine id + */ + + + rc = uname(&uname_arg); + + if (rc < 0) { + + return fs_serial_num; + } + + + CUSFS_TRACE_LOG_FILE(9,"uname: machined id = %s", + uname_arg.machine); + + + sscanf(uname_arg.machine,"%16"PRIX64"",&fs_serial_num.val1); + + if (fs_serial_num.val1 == 0LL) { + + /* + * If serial number does not decode as number, + * then try to use the ASCII hex values instead. + */ + + word_ptr = (uint *)&uname_arg.machine; + + fs_serial_num.val1 = *word_ptr; + } + + + /* + * The second 64-bits will contain the time in seconds, + * since epoch. + */ +#if !defined(__64BIT__) && defined(_AIX) + fs_serial_num.val2 = (uint64_t)cflsh_usfs_time64(NULL); +#else + fs_serial_num.val2 = (uint64_t)time(NULL); +#endif /* not 32-bit AIX */ + + /* + * The third 64-bit will contain a random number + */ + + word_ptr = (uint *)&fs_serial_num.val3; + + *word_ptr = rand_r(&seed); + + seed = *word_ptr + 5; + + word_ptr++; + + + + *word_ptr = rand_r(&seed); + + /* + * The fourth 64-bit will be zero for now. + */ + + CUSFS_TRACE_LOG_FILE(9,"fs_serial num = 0x%llx 0x%llx 0x%llx 0x%llx", + fs_serial_num.val1,fs_serial_num.val2,fs_serial_num.val3,fs_serial_num.val4); + + return fs_serial_num; +} + + +/* + * NAME: cusfs_get_disk_block_no_from_fs_block_no + * + * FUNCTION: Get the disk block number for the filesystem block number. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_get_disk_block_no_from_fs_block_no(cflsh_usfs_t *cufs, uint64_t fs_block_no, uint64_t *disk_block_no) +{ + int rc = 0; + + + + if (fs_block_no < cufs->superblock.fs_start_data_block) { + + CUSFS_TRACE_LOG_FILE(1,"Invalid fs_block_no = 0x%llx fs_start_data_lba = 0x%llx for disk %s", + fs_block_no,cufs->superblock.fs_start_data_block, cufs->device_name); + + errno = ERANGE; + return -1; + + } + + + + *disk_block_no = (fs_block_no * cufs->fs_block_size)/cufs->disk_block_size; + + + if (*disk_block_no < cufs->superblock.start_data_blocks) { + + CUSFS_TRACE_LOG_FILE(1,"Invalid disk_block_no = 0x%llx start_data_lba = 0x%llx for disk %s", + *disk_block_no,cufs->superblock.start_data_blocks,cufs->device_name); + + errno = ERANGE; + return -1; + } + + + if (*disk_block_no > cufs->num_blocks) { + + CUSFS_TRACE_LOG_FILE(1,"Invalid disk_block_no = 0x%llx last disk lba = 0x%llx for disk %s", + *disk_block_no,cufs->num_blocks,cufs->device_name); + + errno = ERANGE; + return -1; + } + + + + return rc; +} + + +/* + * NAME: cusfs_get_free_block_table_size + * + * FUNCTION: Get size of free block table in blocks + * + * + * INPUTS: + * NONE + * + * RETURNS: size of free block table in blocks + * + * + */ + +uint64_t cusfs_get_free_block_table_size(cflsh_usfs_t *cufs, + uint64_t num_blocks, + int flags) +{ + uint64_t num_disk_blocks_free_blk_tbl = 0; + + + /* + * If the end of the disk is not a full fs_block_size + * then we ignore that last sectors up to fs_block_size + * + * Since each bit in the free block table represents a block of + * size fs_block_size. Lets first determine the + * size in bytes this free block table. The free block data + * table will represent all blocks on the disk include the metadata + * We'll then mark the metadata blocks as in use. + */ + + + + + + /* + * Determine how many disk blocks (of cufs->disk_block_size) are needed to represent + * this free table. Each bit in the free table corresponds to one + * filesystem block size (cufs->fs_block_size). Thus num_inode_data_blocks is + * a preliminary value for the number of bits in the free block table. + * + */ + + + num_disk_blocks_free_blk_tbl = cufs->num_inode_data_blocks/(8*cufs->disk_block_size); + + if (cufs->num_inode_data_blocks % (8*cufs->disk_block_size)) { + + num_disk_blocks_free_blk_tbl++; + } + + + + return num_disk_blocks_free_blk_tbl; +} + + + +/* + * NAME: cusfs_create_superblock + * + * FUNCTION: Create superblock for this filesystem + * + * + * NOTE: This routine assumes the caller + * has the cusfs_global.global_lock. + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_create_superblock(cflsh_usfs_t *cufs, int flags) +{ + cflsh_usfs_super_block_t *cusfs_super_block; + int rc; + char timebuf[TIMELEN+1]; + char ctime_string[TIMELEN+1]; + char wtime_string[TIMELEN+1]; + char mtime_string[TIMELEN+1]; + char ftime_string[TIMELEN+1]; + + + + + cusfs_super_block = &cufs->superblock; + + bzero(cusfs_super_block, sizeof(*cusfs_super_block)); + + + cusfs_super_block->start_marker = CLFSH_USFS_SB_SM; + + cusfs_super_block->end_marker = CLFSH_USFS_SB_EM; + + + cusfs_super_block->fs_unique_id = cusfs_generate_fs_unique_id(); + +#ifdef _NOT_YET + if (cusfs_super_block->fs_unique_id.val1 == 0LL) { + + CUSFS_TRACE_LOG_FILE(1,"Invalid FS unique_id"); + return -1; + } +#endif + + cusfs_super_block->fs_block_size = cufs->fs_block_size; + + cusfs_super_block->disk_block_size = cufs->disk_block_size; + + cusfs_super_block->os_type = cusfs_global.os_type; + + cusfs_super_block->num_blocks = cufs->num_blocks; + + if (cusfs_super_block->num_blocks < + (CFLSH_USFS_FREE_BLOCK_TABLE_LBA + 2*cufs->fs_block_size)) { + + CUSFS_TRACE_LOG_FILE(1,"Disk is too small for fileystem num_blocks = 0x%llx needds to be 0xllx", + cusfs_super_block->num_blocks, + (CFLSH_USFS_FREE_BLOCK_TABLE_LBA + 2*cufs->fs_block_size)); + return -1; + + } + + cusfs_super_block->journal_start_lba = CFLSH_USFS_JOURNAL_START_TABLE_LBA; + + cusfs_super_block->journal_table_size = CFLSH_USFS_JOURNAL_TABLE_SIZE; + + cusfs_super_block->free_block_table_stats_lba = CFLSH_USFS_FBLK_TBL_STATS_OFFSET; + + cusfs_super_block->free_block_table_start_lba = CFLSH_USFS_FREE_BLOCK_TABLE_LBA; + + cusfs_super_block->free_block_table_size = cusfs_get_free_block_table_size(cufs, + cufs->num_blocks, + 0); + + cusfs_super_block->inode_table_stats_lba = CFLSH_USFS_INODE_TBL_STATS_OFFSET; + cusfs_super_block->inode_table_start_lba = CFLSH_USFS_FREE_BLOCK_TABLE_LBA + + cusfs_super_block->free_block_table_size; + + + cusfs_super_block->inode_table_size = (cufs->num_inode_data_blocks * sizeof(cflsh_usfs_inode_t))/cufs->disk_block_size; + + if ((cufs->num_inode_data_blocks * sizeof(cflsh_usfs_inode_t)) % cufs->disk_block_size) { + + cusfs_super_block->inode_table_size++; + } + + cusfs_super_block->num_inodes_per_disk_block = cufs->disk_block_size/sizeof(cflsh_usfs_inode_t); + + + cusfs_super_block->start_data_blocks = (cusfs_super_block->inode_table_start_lba + cusfs_super_block->inode_table_size + cufs->fs_block_size) & ~(cufs->fs_block_size - 1); + + cusfs_super_block->fs_start_data_block = (cusfs_super_block->start_data_blocks * cufs->disk_block_size)/cufs->fs_block_size; + + /* + * Set create and write time now + */ +#if !defined(__64BIT__) && defined(_AIX) + cusfs_super_block->create_time = cflsh_usfs_time64(NULL); + cusfs_super_block->write_time = cflsh_usfs_time64(NULL); +#else + cusfs_super_block->create_time = time(NULL); + cusfs_super_block->write_time = time(NULL); +#endif /* not 32-bit AIX */ + + bzero(cufs->buf,cufs->disk_block_size); + + rc = cblk_read(cufs->chunk_id,cufs->buf,CFLSH_USFS_PRIMARY_SB_OFFSET,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of superblock at lba = 0x%llx failed with errno = %d", + CFLSH_USFS_PRIMARY_SB_OFFSET,errno); + return -1; + } + + + cusfs_super_block = cufs->buf; + + if (cusfs_super_block->start_marker == CLFSH_USFS_SB_SM) { + + + CUSFS_TRACE_LOG_FILE(5,"superblock at lba = 0x%llx already exists!!", + CFLSH_USFS_PRIMARY_SB_OFFSET,errno); + + CUSFS_TRACE_LOG_FILE(5,"version = 0x%x, status = 0x%x. os_type = 0x%x flags = 0x%x", + cusfs_super_block->version,cusfs_super_block->status, + cusfs_global.os_type,cusfs_super_block->flags); + + CUSFS_TRACE_LOG_FILE(5,"disk_block_size = 0x%x, fs_block_size = 0x%x. num_blocks = 0x%llx", + cusfs_super_block->disk_block_size,cusfs_super_block->fs_block_size, + cusfs_super_block->num_blocks); + + + CUSFS_TRACE_LOG_FILE(5,"inode_table_start_lba = 0x%llx, inode_size = 0x%llx", + cusfs_super_block->inode_table_start_lba, + cusfs_super_block->inode_table_size); + + CUSFS_TRACE_LOG_FILE(5,"free_block_table_start = 0x%llx, free_block_table_size = 0x%llx", + cusfs_super_block->free_block_table_start_lba, + cusfs_super_block->free_block_table_size); + + + CUSFS_TRACE_LOG_FILE(5,"journal_start_lba = 0x%llx, journal_table_size = 0x%llx", + cusfs_super_block->journal_start_lba, + cusfs_super_block->journal_table_size); + + + /* + * ctime_r adds a newline character at the end. So strip + * that off for all times here to make traces presentable. + */ +#if !defined(__64BIT__) && defined(_AIX) + + sprintf(ctime_string,"%s",ctime64_r(&(cusfs_super_block->create_time),timebuf)); + + ctime_string[strlen(ctime_string) - 1] = '\0'; + + sprintf(wtime_string,"%s",ctime64_r(&(cusfs_super_block->write_time),timebuf)); + + wtime_string[strlen(wtime_string) - 1] = '\0'; + + sprintf(mtime_string,"%s",ctime64_r(&(cusfs_super_block->mount_time),timebuf)); + + mtime_string[strlen(mtime_string) - 1] = '\0'; + + sprintf(ftime_string,"%s",ctime64_r(&(cusfs_super_block->fsck_time),timebuf)); + + ftime_string[strlen(ftime_string) - 1] = '\0'; +#else + + sprintf(ctime_string,"%s",ctime_r(&(cusfs_super_block->create_time),timebuf)); + + ctime_string[strlen(ctime_string) - 1] = '\0'; + + sprintf(wtime_string,"%s",ctime_r(&(cusfs_super_block->write_time),timebuf)); + + wtime_string[strlen(wtime_string) - 1] = '\0'; + + sprintf(mtime_string,"%s",ctime_r(&(cusfs_super_block->mount_time),timebuf)); + + mtime_string[strlen(mtime_string) - 1] = '\0'; + + sprintf(ftime_string,"%s",ctime_r(&(cusfs_super_block->fsck_time),timebuf)); + + ftime_string[strlen(ftime_string) - 1] = '\0'; +#endif + + + + CUSFS_TRACE_LOG_FILE(5,"Creation time %s, Last write time %s, mount time %s, fsck_time %s", + ctime_string,wtime_string,mtime_string,ftime_string); + + + if (!(flags & CFLSH_USFS_FORCE_SB_CREATE)) { + errno = EEXIST; + return -1; + } + + } + + if (CFLASH_REV64(cusfs_super_block->start_marker) == CLFSH_USFS_SB_SM) { + + + CUSFS_TRACE_LOG_FILE(5,"superblock at lba = 0x%llx already exists buf for different endian host", + CFLSH_USFS_PRIMARY_SB_OFFSET,errno); + + CUSFS_TRACE_LOG_FILE(5,"version = 0x%x, status = 0x%x. os_type = 0x%x flags = 0x%x", + CFLASH_REV32(cusfs_super_block->version), + CFLASH_REV32(cusfs_super_block->status), + CFLASH_REV32(cusfs_global.os_type), + CFLASH_REV32(cusfs_super_block->flags)); + + CUSFS_TRACE_LOG_FILE(5,"disk_block_size = 0x%x, fs_block_size = 0x%x. num_blocks = 0x%llx", + CFLASH_REV32(cusfs_super_block->disk_block_size), + CFLASH_REV64(cusfs_super_block->fs_block_size), + CFLASH_REV64(cusfs_super_block->num_blocks)); + + + CUSFS_TRACE_LOG_FILE(5,"inode_table_start_lba = 0x%llx, inode_size = 0x%llx", + CFLASH_REV64(cusfs_super_block->inode_table_start_lba), + CFLASH_REV64(cusfs_super_block->inode_table_size)); + + CUSFS_TRACE_LOG_FILE(5,"free_block_table_start = 0x%llx, free_block_table_size = 0x%llx", + CFLASH_REV64(cusfs_super_block->free_block_table_start_lba), + CFLASH_REV64(cusfs_super_block->free_block_table_size)); + + + CUSFS_TRACE_LOG_FILE(5,"journal_start_lba = 0x%llx, journal_table_size = 0x%llx", + CFLASH_REV64(cusfs_super_block->journal_start_lba), + CFLASH_REV64(cusfs_super_block->journal_table_size)); + + + + if (!(flags & CFLSH_USFS_FORCE_SB_CREATE)) { +#ifdef _AIX + errno = ECORRUPT; +#else + errno = EBADR; +#endif /* ! AIX */ + return -1; + } + + + } + + bzero(cufs->buf,cufs->disk_block_size); + + *cusfs_super_block = cufs->superblock; + + + rc = cblk_write(cufs->chunk_id,cufs->buf, CFLSH_USFS_PRIMARY_SB_OFFSET,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of superblock at lba = 0x%llx failed with errno = %d", + CFLSH_USFS_PRIMARY_SB_OFFSET,errno); + return -1; + } + + cufs->flags |= CFLSH_USFS_FLG_SB_VAL; + + CUSFS_TRACE_LOG_FILE(5,"Superblock successfully written to disk %s", + cufs->device_name); + + return 0; +} + + +/* + * NAME: cusfs_update_superblock + * + * FUNCTION: Update superblock for this filesystem + * + * + * + * NOTE: This routine assumes the caller + * has the cusfs_global.global_lock. + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_update_superblock(cflsh_usfs_t *cufs, int flags) +{ + cflsh_usfs_super_block_t *cusfs_super_block; + int rc; + + + bzero(cufs->buf,cufs->disk_block_size); + + rc = cblk_read(cufs->chunk_id,cufs->buf,CFLSH_USFS_PRIMARY_SB_OFFSET,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of superblock at lba = 0x%llx failed with errno = %d", + CFLSH_USFS_PRIMARY_SB_OFFSET,errno); + return -1; + } + + + + cusfs_super_block = &cufs->superblock; + + /* + * Update write time + */ + +#if !defined(__64BIT__) && defined(_AIX) + cusfs_super_block->write_time = cflsh_usfs_time64(NULL); +#else + cusfs_super_block->write_time = time(NULL); +#endif /* not 32-bit AIX */ + + + cusfs_super_block = cufs->buf; + + *cusfs_super_block = cufs->superblock; + + + rc = cblk_write(cufs->chunk_id,cufs->buf, CFLSH_USFS_PRIMARY_SB_OFFSET,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of superblock at lba = 0x%llx failed with errno = %d", + CFLSH_USFS_PRIMARY_SB_OFFSET,errno); + return -1; + } + + + CUSFS_TRACE_LOG_FILE(5,"Superblock successfully updated and written to disk %s", + cufs->device_name); + + return 0; +} + + +/* + * NAME: cusfs_remove_superblock + * + * FUNCTION: Remove superblock from this disk\ + * and thus remove the filesystem. + * + * + * NOTE: This routine assumes the caller + * has the cusfs_global.global_lock. + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_remove_superblock(cflsh_usfs_t *cufs, int flags) +{ + int rc; + + + bzero(cufs->buf,cufs->disk_block_size); + + + rc = cblk_write(cufs->chunk_id,cufs->buf, CFLSH_USFS_PRIMARY_SB_OFFSET,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of superblock at lba = 0x%llx failed with errno = %d", + CFLSH_USFS_PRIMARY_SB_OFFSET,errno); + return -1; + } + + + CUSFS_TRACE_LOG_FILE(5,"Superblock successfully removed and written to disk %s", + cufs->device_name); + + return 0; +} + + +/* + * NAME: cusfs_validate_superblock + * + * FUNCTION: Validate an existing superblock for this filesystem + * + * + * NOTE: This routine assumes the caller + * has the cusfs_global.global_lock. + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_validate_superblock(cflsh_usfs_t *cufs, int flags) +{ + cflsh_usfs_super_block_t *cusfs_super_block; + int rc; + char timebuf[TIMELEN+1]; + char ctime_string[TIMELEN+1]; + char wtime_string[TIMELEN+1]; + +#if !defined(__64BIT__) && defined(_AIX) + time64_t rev_ctime, rev_wtime; +#else + time_t rev_ctime, rev_wtime; +#endif + + bzero(cufs->buf,cufs->disk_block_size); + + rc = cblk_read(cufs->chunk_id,cufs->buf,CFLSH_USFS_PRIMARY_SB_OFFSET,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of superblock at lba = 0x%llx failed with errno = %d", + CFLSH_USFS_PRIMARY_SB_OFFSET,errno); + return -1; + } + + + cusfs_super_block = cufs->buf; + + + + CUSFS_TRACE_LOG_FILE(5,"version = 0x%x, status = 0x%x. os_type = 0x%x flags = 0x%x", + cusfs_super_block->version,cusfs_super_block->status, + cusfs_super_block->os_type,cusfs_super_block->flags); + + CUSFS_TRACE_LOG_FILE(5,"disk_block_size = 0x%x, fs_block_size = 0x%llx, num_blocks = 0x%llx", + cusfs_super_block->disk_block_size,cusfs_super_block->fs_block_size, + cusfs_super_block->num_blocks); + + + CUSFS_TRACE_LOG_FILE(5,"inode_table_start_lba = 0x%llx, inode_size = 0x%llx", + cusfs_super_block->inode_table_start_lba, + cusfs_super_block->inode_table_size); + + CUSFS_TRACE_LOG_FILE(5,"free_block_table_start = 0x%llx, free_block_table_size = 0x%llx", + cusfs_super_block->free_block_table_start_lba, + cusfs_super_block->free_block_table_size); + + + CUSFS_TRACE_LOG_FILE(5,"journal_start_lba = 0x%llx, journal_table_size = 0x%llx", + cusfs_super_block->journal_start_lba, + cusfs_super_block->journal_table_size); + + + CUSFS_TRACE_LOG_FILE(9,"fs_serial num = 0x%llx 0x%llx 0x%llx 0x%llx", + cusfs_super_block->fs_unique_id.val1, + cusfs_super_block->fs_unique_id.val2, + cusfs_super_block->fs_unique_id.val3, + cusfs_super_block->fs_unique_id.val4); + + //?? Need check for endianess maybe just look at cusfs_super_block->start_marker ? + + + if (cusfs_super_block->start_marker != CLFSH_USFS_SB_SM) { + + CUSFS_TRACE_LOG_FILE(1,"Not valid super block start_marker = 0x%llx", + cusfs_super_block->start_marker); + + CUSFS_TRACE_LOG_FILE(1,"expected tart_marker = 0x%llx", + CLFSH_USFS_SB_SM); + + errno = EINVAL; + if (CFLASH_REV64(cusfs_super_block->start_marker) == CLFSH_USFS_SB_SM) { + + CUSFS_TRACE_LOG_FILE(1,"Actually superblock is valid for different endianess host", + cusfs_super_block->start_marker); + + CUSFS_TRACE_LOG_FILE(1,"version = 0x%x, status = 0x%x, os_type = 0x%x flags = 0x%x", + CFLASH_REV32(cusfs_super_block->version), + CFLASH_REV32(cusfs_super_block->status), + CFLASH_REV32(cusfs_super_block->os_type), + CFLASH_REV32(cusfs_super_block->flags)); + + CUSFS_TRACE_LOG_FILE(5,"disk_block_size = 0x%x, fs_block_size = 0x%llx, num_blocks = 0x%llx", + CFLASH_REV32(cusfs_super_block->disk_block_size), + CFLASH_REV64(cusfs_super_block->fs_block_size), + CFLASH_REV64(cusfs_super_block->num_blocks)); + + + CUSFS_TRACE_LOG_FILE(5,"inode_table_start_lba = 0x%llx, inode_size = 0x%llx", + CFLASH_REV64(cusfs_super_block->inode_table_start_lba), + CFLASH_REV64(cusfs_super_block->inode_table_size)); + + CUSFS_TRACE_LOG_FILE(5,"free_block_table_start = 0x%llx, free_block_table_size = 0x%llx", + CFLASH_REV64(cusfs_super_block->free_block_table_start_lba), + CFLASH_REV64(cusfs_super_block->free_block_table_size)); + + + CUSFS_TRACE_LOG_FILE(5,"journal_start_lba = 0x%llx, journal_table_size = 0x%llx", + CFLASH_REV64(cusfs_super_block->journal_start_lba), + CFLASH_REV64(cusfs_super_block->journal_table_size)); + + + + rev_ctime = CFLASH_REV64(cusfs_super_block->create_time); + rev_wtime = CFLASH_REV64(cusfs_super_block->write_time); + + + + + /* + * ctime_r adds a newline character at the end. So strip + * that off for all times here to make traces presentable. + */ +#if !defined(__64BIT__) && defined(_AIX) + + sprintf(ctime_string,"%s",ctime64_r(&rev_ctime,timebuf)); + + ctime_string[strlen(ctime_string) - 1] = '\0'; + + sprintf(wtime_string,"%s",ctime64_r(&rev_wtime,timebuf)); + + wtime_string[strlen(wtime_string) - 1] = '\0'; +#else + sprintf(ctime_string,"%s",ctime_r(&rev_ctime,timebuf)); + + ctime_string[strlen(ctime_string) - 1] = '\0'; + + sprintf(wtime_string,"%s",ctime_r(&rev_wtime,timebuf)); + + wtime_string[strlen(wtime_string) - 1] = '\0'; +#endif + + CUSFS_TRACE_LOG_FILE(5,"Creation time %s, last write time %s", + ctime_string,wtime_string); + + +#ifdef _AIX + errno = ECORRUPT; +#else + errno = EBADR; +#endif /* ! AIX */ + + + } + + return -1; + + } + + + if (cusfs_super_block->end_marker != CLFSH_USFS_SB_EM) { + + CUSFS_TRACE_LOG_FILE(1,"Not valid super block end_marker = 0x%llx", + cusfs_super_block->end_marker); + + errno = EINVAL; + + return -1; + + } + + /* + * ctime_r adds a newline character at the end. So strip + * that off for all times here to make traces presentable. + */ + +#if !defined(__64BIT__) && defined(_AIX) + + sprintf(ctime_string,"%s",ctime64_r(&(cusfs_super_block->create_time),timebuf)); + + ctime_string[strlen(ctime_string) - 1] = '\0'; + + sprintf(wtime_string,"%s",ctime64_r(&(cusfs_super_block->write_time),timebuf)); + + wtime_string[strlen(wtime_string) - 1] = '\0'; + + + CUSFS_TRACE_LOG_FILE(5,"Creation time %s, Last write time %s, mount time %s, fsck_time %s", + ctime_string,wtime_string, + ctime64_r(&(cusfs_super_block->mount_time),timebuf), + ctime64_r(&(cusfs_super_block->fsck_time),timebuf)); +#else + + sprintf(ctime_string,"%s",ctime_r(&(cusfs_super_block->create_time),timebuf)); + + ctime_string[strlen(ctime_string) - 1] = '\0'; + + sprintf(wtime_string,"%s",ctime_r(&(cusfs_super_block->write_time),timebuf)); + + wtime_string[strlen(wtime_string) - 1] = '\0'; + + + CUSFS_TRACE_LOG_FILE(5,"Creation time %s, Last write time %s, mount time %s, fsck_time %s", + ctime_string,wtime_string, + ctime_r(&(cusfs_super_block->mount_time),timebuf), + ctime_r(&(cusfs_super_block->fsck_time),timebuf)); + +#endif + + if (cusfs_super_block->os_type != cusfs_global.os_type) { + + CUSFS_TRACE_LOG_FILE(1,"os_type differences: our os_type = 0x%x, but super block's os_type = 0x%x", + cusfs_global.os_type, + cusfs_super_block->os_type); + + errno = EINVAL; + + return -1; + + + } + + if (cusfs_super_block->disk_block_size != cufs->disk_block_size) { + + CUSFS_TRACE_LOG_FILE(1,"disk_block_size differences: our disk_block_size = 0x%x, but super block's disk_block_size = 0x%x", + cufs->disk_block_size, + cusfs_super_block->disk_block_size); + + errno = EINVAL; + + return -1; + + } + + + if (cusfs_super_block->fs_block_size != cufs->fs_block_size) { + + CUSFS_TRACE_LOG_FILE(1,"fs_block_size differences: our disk_block_size = 0x%x, but super block's fs_block_size = 0x%x", + cufs->fs_block_size, + cusfs_super_block->fs_block_size); + + errno = EINVAL; + + return -1; + + } + + if (cusfs_super_block->num_blocks != cufs->num_blocks) { + + CUSFS_TRACE_LOG_FILE(1,"num_blocks differences: our num_blocks = 0x%llx, but super block's num_blocks = 0x%llx", + cufs->num_blocks, + cusfs_super_block->num_blocks); + + errno = EINVAL; + + return -1; + + } + + cufs->superblock = *cusfs_super_block; + + return 0; +} + + + +/* + * NAME: cusfs_query_superblock + * + * FUNCTION: Query for a superblock on the specified disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_query_superblock(cflsh_usfs_t *cufs, cflsh_usfs_super_block_t *superblock,int flags) +{ + cflsh_usfs_super_block_t *cusfs_super_block; + int rc; + + bzero(cufs->buf,cufs->disk_block_size); + + rc = cblk_read(cufs->chunk_id,cufs->buf,CFLSH_USFS_PRIMARY_SB_OFFSET,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of superblock at lba = 0x%llx failed with errno = %d", + CFLSH_USFS_PRIMARY_SB_OFFSET,errno); + return -1; + } + + + cusfs_super_block = cufs->buf; + + *superblock = *cusfs_super_block; + + + + return 0; +} + + +/* + * NAME: cusfs_create_free_block_table + * + * FUNCTION: Create the block free table map for this disk + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_create_free_block_table(cflsh_usfs_t *cufs, int flags) +{ + uint64_t remaining_size; + cflash_offset_t offset; + uint64_t num_blocks_write; + uint64_t byte_offset_data_start; + uint64_t bit_offset_in_byte_data_start; + uint64_t current_byte_offset; + uint64_t end_meta_data_offset; + uint64_t num_used_free_blocks = 0; + cflsh_usfs_free_block_stats_t *block_stat; + char *bptr; + int buf_size; + uint64_t i; + int rc = 0; + + + + CUSFS_LOCK(cufs->free_table_lock); + + + /* + * Determine which fs block corresponds to the first data block + */ + + + + buf_size = cufs->buf_size; + + remaining_size = cufs->superblock.free_block_table_size * cufs->disk_block_size; + + offset = cufs->superblock.free_block_table_start_lba; + + if (buf_size % cufs->disk_block_size) { + + CUSFS_TRACE_LOG_FILE(1,"buf_size = 0x%x, is not a multiple of device block size", + buf_size,cufs->disk_block_size); + } + + num_blocks_write = MIN(buf_size/cufs->disk_block_size,cufs->max_xfer_size); + + + /* + * Since the free block table includes the meta data blocks at the + * beginning of the disk, determine the offset in the free block + * table where the "real" data blocks start. + */ + + + byte_offset_data_start = cufs->superblock.fs_start_data_block/CUSFS_BITS_PER_BYTE; + + bit_offset_in_byte_data_start = cufs->superblock.fs_start_data_block % CUSFS_BITS_PER_BYTE; + + current_byte_offset = 0; + + while (remaining_size > 0) { + + + bzero(cufs->buf,cufs->buf_size); + + if (remaining_size < buf_size) { + + + num_blocks_write = remaining_size/cufs->disk_block_size; + + + if (remaining_size % cufs->disk_block_size) { + + num_blocks_write++; + } + + buf_size = remaining_size; + } + + + + if ((current_byte_offset <= byte_offset_data_start) || + (((byte_offset_data_start + 1) == current_byte_offset) && + (bit_offset_in_byte_data_start))) { + + + /* + * Since the free block table also includes + * the filesystem meta data blocks at the + * beginning of the disk we need to mark + * those as in use. + */ + + + end_meta_data_offset = MIN((buf_size + current_byte_offset),byte_offset_data_start); + + + for (bptr = cufs->buf; bptr < (char *)(cufs->buf + end_meta_data_offset); bptr++) { + + *bptr = 0xff; + num_used_free_blocks += 8; + + + } + + if (end_meta_data_offset < (buf_size + current_byte_offset)) { + + for (i=0;i< bit_offset_in_byte_data_start; i++) { + + *bptr |= (0x01 << ((CUSFS_BITS_PER_BYTE - 1) - i)); + num_used_free_blocks++; + } + } + + + } + + + + rc = cblk_write(cufs->chunk_id,cufs->buf,offset,num_blocks_write,0); + + if (rc < num_blocks_write) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table at lba = 0x%llx failed with errno = %d", + offset,errno); + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + remaining_size -= buf_size; + + current_byte_offset += buf_size; + + + offset += num_blocks_write; + + + } /* while */ + + + /* + * Create free block table statistics area + */ + + bzero (cufs->buf,cufs->disk_block_size); + + block_stat = (cflsh_usfs_free_block_stats_t *) cufs->buf; + + block_stat->start_marker = CLFSH_USFS_FB_SM; + + block_stat->end_marker = CLFSH_USFS_FB_EM; + + block_stat->total_free_blocks = cufs->superblock.free_block_table_size * cufs->disk_block_size * 8; + + block_stat->available_free_blocks = block_stat->total_free_blocks - num_used_free_blocks; + + num_blocks_write = 1; + + offset = CFLSH_USFS_FBLK_TBL_STATS_OFFSET; + rc = cblk_write(cufs->chunk_id,cufs->buf,offset,num_blocks_write,0); + + if (rc < num_blocks_write) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + + cufs->flags |= CFLSH_USFS_FLG_FT_VAL; + + /* should we update superblock with this status ?? */ + + CUSFS_TRACE_LOG_FILE(5,"Free block table successfully written starting at 0%llx to disk %s", + cufs->superblock.free_block_table_start_lba,cufs->device_name); + + + CUSFS_UNLOCK(cufs->free_table_lock); + + return 0; +} + +/* + * NAME: cusfs_create_inode_table + * + * FUNCTION: Create the inode table for this disk + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_create_inode_table(cflsh_usfs_t *cufs, int flags) +{ + int rc = 0; + uint64_t remaining_size; + cflash_offset_t offset; + uint64_t num_blocks_write; + uint64_t inode_index = 0; + uint64_t total_num_inodes = 0; + int buf_size; + int i,j; + int num_inodes_per_disk_block; + cflsh_usfs_inode_t *inode; + cflsh_usfs_inode_table_stats_t *inode_stat; + + CUSFS_LOCK(cufs->inode_table_lock); + + + + buf_size = cufs->buf_size; + + offset = cufs->superblock.inode_table_start_lba; + + remaining_size = cufs->num_inode_data_blocks * cufs->disk_block_size; + + /* + * Ignore end of disk block if it can not fit an inode entry + */ + + num_inodes_per_disk_block = cufs->disk_block_size/sizeof(cflsh_usfs_inode_t); + + if (buf_size % cufs->disk_block_size) { + + CUSFS_TRACE_LOG_FILE(1,"buf_size = 0x%x, is not a multiple of device block size", + cufs->buf_size,cufs->disk_block_size); + } + + num_blocks_write = MIN(buf_size/cufs->disk_block_size,cufs->max_xfer_size); + + while (remaining_size > 0) { + + + bzero(cufs->buf,cufs->buf_size); + + if (remaining_size < buf_size) { + + + num_blocks_write = remaining_size/cufs->disk_block_size; + + + if (remaining_size % cufs->disk_block_size) { + + num_blocks_write++; + } + + buf_size = remaining_size; + } + + inode = cufs->buf; + + for (j=0; j < num_blocks_write; j++) { + + inode = cufs->buf + (j * cufs->disk_block_size); + + for (i=0; i < num_inodes_per_disk_block; i++) { + + inode[i].start_marker = CLFSH_USFS_INODE_SM; + + inode[i].index = inode_index++; + + inode[i].end_marker = CLFSH_USFS_INODE_EM; + + total_num_inodes++; + + } + } + + + rc = cblk_write(cufs->index,cufs->buf,offset,num_blocks_write,0); + + if (rc < num_blocks_write) { + + CUSFS_TRACE_LOG_FILE(1,"write of inode table at lba = 0x%llx failed with errno = %d", + offset,errno); + + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + remaining_size -= buf_size; + + + offset += num_blocks_write; + + + + + } /* while */ + + + /* + * Create inode table statistics area + */ + + bzero (cufs->buf,cufs->disk_block_size); + + inode_stat = (cflsh_usfs_inode_table_stats_t *) cufs->buf; + + inode_stat->start_marker = CLFSH_USFS_IT_SM; + + inode_stat->end_marker = CLFSH_USFS_IT_EM; + + inode_stat->total_inodes = total_num_inodes; + + inode_stat->available_inodes = total_num_inodes; + + num_blocks_write = 1; + + offset = CFLSH_USFS_INODE_TBL_STATS_OFFSET; + rc = cblk_write(cufs->chunk_id,cufs->buf,offset,num_blocks_write,0); + + if (rc < num_blocks_write) { + + CUSFS_TRACE_LOG_FILE(1,"write of inode table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + + + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + + cufs->flags |= CFLSH_USFS_FLG_IT_VAL; + + /* should we update superblock with this status ?? */ + + CUSFS_TRACE_LOG_FILE(5,"Inode table successfully written starting at 0%llx to disk %s", + cufs->superblock.inode_table_start_lba,cufs->device_name); + + + CUSFS_UNLOCK(cufs->inode_table_lock); + + return 0; +} + +/* + * NAME: cusfs_create_journal + * + * FUNCTION: Create the filesystem journal for this disk + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_create_journal(cflsh_usfs_t *cufs, int flags) +{ + int rc = 0; + uint64_t remaining_size; + cflash_offset_t offset; + uint64_t num_blocks_write; + int buf_size; + int need_header = TRUE; + cflsh_usfs_journal_hdr_t *journal_hdr; + + + CUSFS_LOCK(cufs->journal_lock); + + buf_size = cufs->buf_size; + + offset = cufs->superblock.journal_start_lba; + + remaining_size = cufs->superblock.journal_table_size * cufs->disk_block_size; + + if (cufs->buf_size % cufs->disk_block_size) { + + CUSFS_TRACE_LOG_FILE(1,"buf_size = 0x%x, is not a multiple of device block size", + buf_size,cufs->disk_block_size); + } + + num_blocks_write = MIN(buf_size/cufs->disk_block_size,cufs->max_xfer_size); + + while (remaining_size > 0) { + + + bzero(cufs->buf,cufs->buf_size); + + if (remaining_size < buf_size) { + + + num_blocks_write = remaining_size/cufs->disk_block_size; + + + if (remaining_size % cufs->disk_block_size) { + + num_blocks_write++; + } + + buf_size = remaining_size; + } + + + if (need_header) { + + journal_hdr = cufs->buf; + need_header = FALSE; + + journal_hdr->start_marker = CLFSH_USFS_JOURNAL_SM; + + /* + * To compute nuber of entries, we first need to get the number of bytes needed + * after the header. Since the header does contain the first entry we need to adjust this. + */ + + journal_hdr->num_entries = (remaining_size - sizeof(*journal_hdr) + sizeof(cflsh_usfs_directory_entry_t))/sizeof(cflsh_usfs_directory_entry_t); + cufs->superblock.num_journal_entries = journal_hdr->num_entries; + + } + + + rc = cblk_write(cufs->index,cufs->buf,offset,num_blocks_write,0); + + if (rc < num_blocks_write) { + + CUSFS_TRACE_LOG_FILE(1,"write of inode table at lba = 0x%llx failed with errno = %d", + offset,errno); + + CUSFS_UNLOCK(cufs->journal_lock); + + return -1; + } + + remaining_size -= buf_size; + + + offset += num_blocks_write; + + + + + } /* while */ + + + + cufs->flags |= CFLSH_USFS_FLG_IT_VAL; + + /* should we update superblock with this status ?? */ + + + CUSFS_TRACE_LOG_FILE(5,"Journal successfully written starting at 0%llx to disk %s", + cufs->superblock.journal_start_lba,cufs->device_name); + + + CUSFS_UNLOCK(cufs->journal_lock); + + return 0; +} + + + +/* + * NAME: cusfs_add_journal_event + * + * FUNCTION: Add a journal event to be written + * to the journal for this filesystem. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_add_journal_event(cflsh_usfs_t *cufs, + cflsh_journal_entry_type_t journal_type, + uint64_t block_no, uint64_t inode_num, + uint64_t old_file_size, uint64_t new_file_size, + uint64_t directory_inode_num, + int flags) +{ + int rc = 0; + cflsh_usfs_journal_ev_t *journal; + + journal = malloc(sizeof(*journal)); + + if (journal == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc journal entry with errno = %d for disk %s", + errno,cufs->device_name); + return -1; + } + + bzero(journal,sizeof(*journal)); + + journal->journal_entry.start_marker = CLFSH_USFS_JOURNAL_E_SM; + + journal->journal_entry.type = journal_type; + + switch (journal_type) { + + case CFLSH_USFS_JOURNAL_E_ADD_BLK: + case CFLSH_USFS_JOURNAL_E_FREE_BLK: + journal->journal_entry.blk.block_no = block_no; + break; + case CFLSH_USFS_JOURNAL_E_GET_INODE: + case CFLSH_USFS_JOURNAL_E_FREE_INODE: + journal->journal_entry.inode.inode_num = inode_num; + break; + case CFLSH_USFS_JOURNAL_E_ADD_INODE_DBLK: + case CFLSH_USFS_JOURNAL_E_FREE_INODE_DBLK: + journal->journal_entry.inode_dblk.inode_num = inode_num; + journal->journal_entry.inode_dblk.block_num = block_no; + break; + case CFLSH_USFS_JOURNAL_E_INODE_FSIZE: + journal->journal_entry.inode_fsize.old_file_size = old_file_size; + journal->journal_entry.inode_fsize.new_file_size = new_file_size; + break; + case CFLSH_USFS_JOURNAL_E_FREE_DIR_E: + journal->journal_entry.dir.directory_inode_num = directory_inode_num; + journal->journal_entry.dir.file_inode_num = inode_num; + break; + default: + CUSFS_TRACE_LOG_FILE(1,"Unknown journal type = %d for disk %s", + journal_type,cufs->device_name); + free(journal); + return -1; + + } + + + + CUSFS_LOCK(cufs->lock); + CUSFS_Q_NODE_TAIL(cufs->add_journal_ev_head,cufs->add_journal_ev_tail,journal, + prev,next); + + CUSFS_UNLOCK(cufs->lock); + + CUSFS_TRACE_LOG_FILE(5,"Journal event of type %d add for disk %s", + journal_type, cufs->device_name); + + + return rc; +} + + +/* + * NAME: cusfs_flush_journal_events + * + * FUNCTION: Add a journal event to be written + * to the journal for this filesystem. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ +int cusfs_flush_journal_events(cflsh_usfs_t *cufs, int flags) +{ + int rc = 0; + + if (cufs->add_journal_ev_head == NULL) { + + /* + * No events to flush + */ + return 0; + } + + return rc; +} + +/* + * NAME: cusfs_get_free_table_stats + * + * FUNCTION: Get get free table statistics + * + * + * INPUTS: + * + * + * RETURNS: 0 Success, Otherwise failure + * + * + */ + +int cusfs_get_free_table_stats(cflsh_usfs_t *cufs,uint64_t *total_blocks, uint64_t *available_blocks, + int flags) +{ + cflash_offset_t offset; + uint64_t num_blocks_read; + uint64_t rc = 0; + cflsh_usfs_free_block_stats_t *block_stat; + + + if (cufs == NULL) { + + + errno = EINVAL; + return -1; + } + + if (total_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + if (available_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + + CUSFS_LOCK(cufs->free_table_lock); + + + /* + * Get inode table statistics + */ + + offset = CFLSH_USFS_FBLK_TBL_STATS_OFFSET; + + num_blocks_read = 1; + + rc = cblk_read(cufs->chunk_id,cufs->inode_tbl_buf,offset,num_blocks_read,0); + + if (rc < num_blocks_read) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + block_stat = (cflsh_usfs_free_block_stats_t *) cufs->inode_tbl_buf; + + + if ((block_stat->start_marker != CLFSH_USFS_FB_SM) || + (block_stat->end_marker != CLFSH_USFS_FB_EM)) { + + CUSFS_TRACE_LOG_FILE(1,"corrupted free block table stats at lba = 0x%llx sm = 0x%llx em = 0x%llx", + offset,block_stat->start_marker,block_stat->end_marker); + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + + } + + *total_blocks = block_stat->total_free_blocks; + *available_blocks = block_stat->available_free_blocks; + + + + CUSFS_UNLOCK(cufs->free_table_lock); + return 0; +} + +/* + * NAME: cusfs_get_data_blocks + * + * FUNCTION: Get free contiguous data blocks and mark as allocated. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 failure, lba (in filesystem blocks) if successful. + * + * + */ + +uint64_t cusfs_get_data_blocks(cflsh_usfs_t *cufs, int *num_blocks,int flags) +{ + uint64_t lba = 0; + uint64_t remaining_size; + cflash_offset_t offset; + cflsh_usfs_free_block_stats_t *block_stat; + uint64_t num_blocks_read; + uint64_t current_byte_offset; + int buf_size; + uint64_t byte_offset; + int i,j; + char *byte; + uint64_t rc; + int num_blocks_found = 0; + int leave_loop = FALSE; + + if ((num_blocks == NULL) || + (*num_blocks == 0)) { + + return 0; + } + + + + if (cufs == NULL) { + + + errno = EINVAL; + return 0; + } + + + CUSFS_LOCK(cufs->free_table_lock); + + if (cflsh_usfs_master_lock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + + /* + * Read free block table from disk until we find a free bit (LBA). + */ + + + + buf_size = cufs->free_blk_tbl_buf_size; + + remaining_size = cufs->superblock.free_block_table_size * cufs->disk_block_size; + + offset = cufs->superblock.free_block_table_start_lba; + + if (buf_size % cufs->disk_block_size) { + + CUSFS_TRACE_LOG_FILE(1,"buf_size = 0x%x, is not a multiple of device block size", + buf_size,cufs->disk_block_size); + } + + num_blocks_read = MIN(buf_size/cufs->disk_block_size,cufs->max_xfer_size); + + + current_byte_offset = 0; + + while (remaining_size > 0) { + + + bzero(cufs->free_blk_tbl_buf,cufs->buf_size); + + if (remaining_size < buf_size) { + + + num_blocks_read = remaining_size/cufs->disk_block_size; + + + if (remaining_size % cufs->disk_block_size) { + + num_blocks_read++; + } + + buf_size = remaining_size; + } + + + + rc = cblk_read(cufs->chunk_id,cufs->free_blk_tbl_buf,offset,num_blocks_read,0); + + if (rc < num_blocks_read) { + + CUSFS_TRACE_LOG_FILE(1,"reaad of free block table at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->free_table_lock); + return 0; + } + + + byte = cufs->free_blk_tbl_buf; + + for (i = 0; i < buf_size; i++) { + + if (((*byte) & 0xff) != 0xff) { + + + if (!num_blocks_found) { + + byte_offset = current_byte_offset + (byte - (char *)cufs->free_blk_tbl_buf); + + + /* + * Determine LBA offset to the first bit of this + * byte. + */ + + + lba = byte_offset * CUSFS_BITS_PER_BYTE; + + CUSFS_TRACE_LOG_FILE(9,"Found byte that has freeblock at = 0x%llx and lba >= 0x%llx", + byte_offset,lba); + + } + + for (j=0;j< CUSFS_BITS_PER_BYTE; j++) { + + if (!( (*byte) & (0x01 << ((CUSFS_BITS_PER_BYTE - 1) - j)))) { + + + /* + * Mark block now as in use and write + * it back. + */ + + *byte |= 0x01 << ((CUSFS_BITS_PER_BYTE - 1) - j); + + num_blocks_found++; + if (*num_blocks == num_blocks_found) { + leave_loop = TRUE; + break; + } + } else if (num_blocks_found) { + + /* + * We are only returning contiguous blocks. + * So if we encounter a non-free block, then + * give up. + */ + leave_loop = TRUE; + + break; + } + + if (!num_blocks_found) { + lba++; + } + } /* inner for */ + + if (leave_loop) { + + break; + } + } else if (num_blocks_found) { + + /* + * We are only returning contiguous blocks. + * So if we encounter a non-free block, then + * give up. + */ + + leave_loop = TRUE; + break; + } + + byte++; + + } /* outer for loop */ + + + if (lba) { + + rc = cblk_write(cufs->chunk_id,cufs->free_blk_tbl_buf,offset, + num_blocks_read,0); + + if (rc < num_blocks_read) { + + + CUSFS_TRACE_LOG_FILE(1,"update write of free block table at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->free_table_lock); + return 0; + } + + if ((*num_blocks == num_blocks_found) || + (leave_loop)) { + break; + } + + } + + + remaining_size -= buf_size; + + current_byte_offset += buf_size; + + + offset += num_blocks_read; + + + } /* while */ + + CUSFS_TRACE_LOG_FILE(5,"Found data block = 0x%llx and num_blocks = 0x%x on device = %s", + lba,num_blocks_found,cufs->device_name); + + + *num_blocks = num_blocks_found; + /* + * Update free block table statistics + */ + + offset = CFLSH_USFS_FBLK_TBL_STATS_OFFSET; + + num_blocks_read = 1; + + rc = cblk_read(cufs->chunk_id,cufs->free_blk_tbl_buf,offset,num_blocks_read,0); + + if (rc < num_blocks_read) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + block_stat = (cflsh_usfs_free_block_stats_t *) cufs->free_blk_tbl_buf; + + + if ((block_stat->start_marker != CLFSH_USFS_FB_SM) || + (block_stat->end_marker != CLFSH_USFS_FB_EM)) { + + CUSFS_TRACE_LOG_FILE(1,"corrupted free block table stats at lba = 0x%llx sm = 0x%llx em = 0x%llx", + offset,block_stat->start_marker,block_stat->end_marker); + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + + } + + block_stat->available_free_blocks -= *num_blocks; + + rc = cblk_write(cufs->chunk_id,cufs->free_blk_tbl_buf,offset,num_blocks_read,0); + + if (rc < num_blocks_read) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + + + + /* + * ?? Add validation check that this lba is not the meta data + * at the beginning of the disk. + */ + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + + CUSFS_UNLOCK(cufs->free_table_lock); + + return lba; +} + + +/* + * NAME: cusfs_get_data_block_list + * + * FUNCTION: Get a list of data blocks and mark as allocated. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_get_data_blocks_list(cflsh_usfs_t *cufs, uint64_t *num_blocks, uint64_t **block_list, int flags) +{ + + uint64_t lba = 0; + uint64_t remaining_size; + cflash_offset_t offset; + cflsh_usfs_free_block_stats_t *block_stat; + uint64_t num_blocks_read; + uint64_t current_byte_offset; + int buf_size; + uint64_t byte_offset; + int i,j; + int update_write_needed; + char *byte; + void *blk_stat_buf; + uint64_t rc; + uint64_t num_blocks_found = 0; + int leave_loop = FALSE; + uint64_t *local_block_list; + + if ((num_blocks == NULL) || + (*num_blocks == 0)) { + + return -1; + } + + + + if (cufs == NULL) { + + + errno = EINVAL; + return -1; + } + + + blk_stat_buf = malloc(cufs->disk_block_size); + + if (blk_stat_buf == NULL) { + + return -1; + } + + CUSFS_TRACE_LOG_FILE(9,"Looking for 0x%llx num_blocks on device = %s", + *num_blocks,cufs->device_name); + + CUSFS_LOCK(cufs->free_table_lock); + + + if (cflsh_usfs_master_lock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + + /* + * Get free block table statistics + */ + + offset = CFLSH_USFS_FBLK_TBL_STATS_OFFSET; + + num_blocks_read = 1; + + rc = cblk_read(cufs->chunk_id,blk_stat_buf,offset,num_blocks_read,0); + + if (rc < num_blocks_read) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + block_stat = (cflsh_usfs_free_block_stats_t *) blk_stat_buf; + + + if ((block_stat->start_marker != CLFSH_USFS_FB_SM) || + (block_stat->end_marker != CLFSH_USFS_FB_EM)) { + + CUSFS_TRACE_LOG_FILE(1,"corrupted free block table stats at lba = 0x%llx sm = 0x%llx em = 0x%llx", + offset,block_stat->start_marker,block_stat->end_marker); + + free(blk_stat_buf); + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + + } + + + if (*num_blocks > block_stat->available_free_blocks) { + + CUSFS_TRACE_LOG_FILE(1,"Not enough free blocks available, available free blocks = 0x%llx number requested 0x%llx", + block_stat->available_free_blocks,*num_blocks); + + free(blk_stat_buf); + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->free_table_lock); + + errno = ENOMEM; + return -1; + + } + + local_block_list = malloc((sizeof(uint64_t) * (*num_blocks))); + + + if (local_block_list == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc block_list of size 0x%llx", + (sizeof(uint64_t) * (*num_blocks))); + + free(blk_stat_buf); + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->free_table_lock); + + errno = ENOMEM; + return -1; + } + + bzero(local_block_list,(sizeof(uint64_t) * (*num_blocks))); + + /* + * Read free block table from disk until we find a free bit (LBA). + */ + + + + buf_size = cufs->free_blk_tbl_buf_size; + + remaining_size = cufs->superblock.free_block_table_size * cufs->disk_block_size; + + offset = cufs->superblock.free_block_table_start_lba; + + if (buf_size % cufs->disk_block_size) { + + CUSFS_TRACE_LOG_FILE(1,"buf_size = 0x%x, is not a multiple of device block size", + buf_size,cufs->disk_block_size); + } + + num_blocks_read = MIN(buf_size/cufs->disk_block_size,cufs->max_xfer_size); + + + current_byte_offset = 0; + + while (remaining_size > 0) { + + + bzero(cufs->free_blk_tbl_buf,cufs->buf_size); + + if (remaining_size < buf_size) { + + + num_blocks_read = remaining_size/cufs->disk_block_size; + + + if (remaining_size % cufs->disk_block_size) { + + num_blocks_read++; + } + + buf_size = remaining_size; + } + + update_write_needed = FALSE; + + rc = cblk_read(cufs->chunk_id,cufs->free_blk_tbl_buf,offset,num_blocks_read,0); + + if (rc < num_blocks_read) { + + CUSFS_TRACE_LOG_FILE(1,"read of free block table at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + free(blk_stat_buf); + free(local_block_list); + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + + byte = cufs->free_blk_tbl_buf; + + for (i = 0; i < buf_size; i++) { + + if (((*byte) & 0xff) != 0xff) { + + + + + byte_offset = current_byte_offset + (byte - (char *)cufs->free_blk_tbl_buf); + + + /* + * Determine LBA offset to the first bit of this + * byte. + */ + + + lba = byte_offset * CUSFS_BITS_PER_BYTE; + + CUSFS_TRACE_LOG_FILE(9,"Found byte that has freeblock at = 0x%llx and lba >= 0x%llx", + byte_offset,lba); + + + for (j=0;j< CUSFS_BITS_PER_BYTE; j++) { + + if (!( (*byte) & (0x01 << ((CUSFS_BITS_PER_BYTE - 1) - j)))) { + + + /* + * Mark block now as in use and write + * it back. + */ + + *byte |= 0x01 << ((CUSFS_BITS_PER_BYTE - 1) - j); + + local_block_list[num_blocks_found++] = lba; + + CUSFS_TRACE_LOG_FILE(9,"Found lba 0x%llx", + lba); + + update_write_needed = TRUE; + if (*num_blocks == num_blocks_found) { + leave_loop = TRUE; + break; + } + } + + lba++; + + } /* inner for */ + + if (leave_loop) { + + break; + } + } + + byte++; + + } /* outer for loop */ + + + if (update_write_needed) { + + rc = cblk_write(cufs->chunk_id,cufs->free_blk_tbl_buf,offset, + num_blocks_read,0); + + if (rc < num_blocks_read) { + + + CUSFS_TRACE_LOG_FILE(1,"update write of free block table at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + free(blk_stat_buf); + free(local_block_list); + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + if ((*num_blocks == num_blocks_found) || + (leave_loop)) { + break; + } + + } + + + remaining_size -= buf_size; + + current_byte_offset += buf_size; + + + offset += num_blocks_read; + + + } /* while */ + + CUSFS_TRACE_LOG_FILE(5,"Found data block = 0x%llx and num_blocks = 0x%llx on device = %s", + lba,num_blocks_found,cufs->device_name); + + + + *num_blocks = num_blocks_found; + /* + * Update free block table statistics + */ + + offset = CFLSH_USFS_FBLK_TBL_STATS_OFFSET; + + num_blocks_read = 1; + + block_stat->available_free_blocks -= *num_blocks; + + rc = cblk_write(cufs->chunk_id,blk_stat_buf,offset,num_blocks_read,0); + + if (rc < num_blocks_read) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + + free(local_block_list); + rc = -1; + } else { + + rc = 0; + } + + *block_list = local_block_list; + + free(blk_stat_buf); + + + /* + * ?? Add validation check that this lba is not the meta data + * at the beginning of the disk. + */ + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + + CUSFS_UNLOCK(cufs->free_table_lock); + + return rc; +} + + +/* + * NAME: cusfs_release_data_blocks + * + * FUNCTION: Release contiguous data blocks and mark as unallocated. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise error + * + * + */ + +int cusfs_release_data_blocks(cflsh_usfs_t *cufs, uint64_t lba,int num_blocks,int flags) +{ + int rc = 0; + uint64_t lba_offset; + uint64_t byte_offset; + uint64_t byte_offset_in_block; + uint64_t bit_offset; + char *byte; + cflsh_usfs_free_block_stats_t *block_stat; + + + if (num_blocks == 0) { + + CUSFS_TRACE_LOG_FILE(1,"Num blocks = 0 for free_block = 0x%llx,for disk %s", + lba,cufs->device_name); + + errno = EINVAL; + return -1; + } + + + if (num_blocks > 1) { + + /* + * currently we don't support multiple blocks + */ + + errno = EINVAL; + return -1; + + } + + + if (cufs == NULL) { + + + errno = EINVAL; + return -1; + } + + /* + * Determine bit offset of the this LBA + * in the free block table. + */ + + byte_offset = lba/CUSFS_BITS_PER_BYTE; + + bit_offset = lba % CUSFS_BITS_PER_BYTE; + + lba_offset = byte_offset/cufs->disk_block_size; + + if (byte_offset % cufs->disk_block_size) { + + lba_offset++; + + byte_offset_in_block = byte_offset % cufs->disk_block_size; + } else { + byte_offset_in_block = 0; + } + + CUSFS_LOCK(cufs->free_table_lock); + + if (cflsh_usfs_master_lock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + /* + * Read disk block containing this LBA byte/bit + */ + + rc = cblk_read(cufs->chunk_id,cufs->free_blk_tbl_buf,lba_offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of free block table at lba = 0x%llx failed with errno = %d", + lba_offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + byte = cufs->free_blk_tbl_buf + byte_offset_in_block; + + *byte &= ~(0x01 << ((CUSFS_BITS_PER_BYTE - 1) - bit_offset)); + + /* + * Write updated disk block containing this LBA byte/bit now indicating + * it is free. + */ + + rc = cblk_write(cufs->chunk_id,cufs->free_blk_tbl_buf,lba_offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table at lba = 0x%llx failed with errno = %d", + lba_offset,errno); + + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_FREE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + + + CUSFS_TRACE_LOG_FILE(9,"lba = 0x%llx marked free ",lba); + + /* + * Update free block table statistics + */ + + lba_offset = CFLSH_USFS_FBLK_TBL_STATS_OFFSET; + + rc = cblk_read(cufs->chunk_id,cufs->free_blk_tbl_buf,lba_offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + lba_offset,errno); + // cflsh_usfs_master_unregister(??,??,??,??); + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + block_stat = (cflsh_usfs_free_block_stats_t *) cufs->free_blk_tbl_buf; + + + if ((block_stat->start_marker != CLFSH_USFS_FB_SM) || + (block_stat->end_marker != CLFSH_USFS_FB_EM)) { + + CUSFS_TRACE_LOG_FILE(1,"corrupted free block table stats at lba = 0x%llx sm = 0x%llx em = 0x%llx", + lba_offset,block_stat->start_marker,block_stat->end_marker); + // cflsh_usfs_master_unregister(??,??,??,??); + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + + } + + block_stat->available_free_blocks += num_blocks; + + rc = cblk_write(cufs->chunk_id,cufs->free_blk_tbl_buf,lba_offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + lba_offset,errno); + + // cflsh_usfs_master_unregister(??,??,??,??); + CUSFS_UNLOCK(cufs->free_table_lock); + return -1; + } + + + // cflsh_usfs_master_unregister(??,??,??,??); + CUSFS_UNLOCK(cufs->free_table_lock); + + return rc; +} + + +/* + * NAME: cusfs_add_directory_entry + * + * FUNCTION: Add a directory entry for the specified + * inode + * + * NOTES: Assume caller has directory data object's lock. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure + * + * + */ +int cusfs_add_directory_entry(cflsh_usfs_t *cufs,uint64_t directory_inode_num,cflsh_usfs_inode_t *inode, + char *filename,int flags) +{ + int rc = 0; + char *buf; + char *buf_new_fs_block; + int buf_size; + int old_directory_entry_offset; + cflsh_usfs_inode_t *parent_inode; + cflsh_usfs_directory_t *directory_hdr; + cflsh_usfs_directory_entry_t *directory_entry; + int added_entry; + cflsh_usfs_data_obj_t *directory; + + if (strlen(filename) >= CFLSH_USFS_DIR_MAX_NAME) { + + CUSFS_TRACE_LOG_FILE(1,"filename = %s too long %d bytes", + filename,strlen(filename)); + + errno = EINVAL; + return -1; + + } + + directory = cusfs_find_data_obj_from_inode(cufs,directory_inode_num,0); + + if (directory == NULL) { + + + parent_inode = cusfs_get_inode_from_index(cufs,directory_inode_num,0); + + if (parent_inode == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to get parent inode 0x%llx for file = %s of type %d on disk %s", + directory_inode_num,filename,inode->type, cufs->device_name); + + + return -1; + } + + + buf_size = parent_inode->file_size; + + buf = malloc(buf_size); + + if (buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc data buffer for parent inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + directory_inode_num,filename,inode->type, cufs->device_name, errno); + + + + return -1; + } + + + } else { + + + parent_inode = directory->inode; + + buf = directory->data_buf; + + buf_size = directory->data_buf_size; + } + + + + if (cusfs_read_data_object(cufs,parent_inode,buf,&buf_size,0)) { + + + + CUSFS_TRACE_LOG_FILE(1,"Failed to read parent inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + directory_inode_num,filename,inode->type, cufs->device_name, errno); + + if (directory == NULL) { + free(buf); + } + + return -1; + + } + + /* + * Add file entry to directory. + * + * NOTE: Since directory entries can span + * a filesystem block, we need to + * ensure that a potential directory entry is + * is contained in the allocated blocks for this + * filesystem. We chose to allow directory entries + * to span blocks, since the filename string can + * be very large in some OS (AIX allows 1024). We + * could have chosen to truncate that to 256, but + * decided to allow this option instead. + */ + + directory_hdr = (cflsh_usfs_directory_t *)buf; + + directory_entry = directory_hdr->entry; + + added_entry = FALSE; + + while (directory_entry < (cflsh_usfs_directory_entry_t *)((buf + buf_size) - sizeof(*directory_entry))) { + + if (!(directory_entry->flags & CFLSH_USFS_DIR_VAL)) { + + directory_entry->flags |= CFLSH_USFS_DIR_VAL; + directory_entry->inode_number = inode->index; + directory_entry->type = inode->type; + strcpy(directory_entry->filename,filename); + added_entry = TRUE; + break; + } + + directory_entry++; + } + + + if (!added_entry) { + + /* + * Need to add 1 data block to this directory. + * Then we reallocate the data buffer and + * add the directory where we left off in our + * search above. + */ + + CUSFS_TRACE_LOG_FILE(5,"Could not find unused directory entry for = %s of type %d on disk %s, errno = %d", + filename,inode->type, cufs->device_name, errno); + + + + if (cusfs_add_fs_blocks_to_inode_ptrs(cufs,parent_inode,1,0)) { + + + return -1; + } + + + CUSFS_TRACE_LOG_FILE(9,"Added block for directory entry for = %s of type %d on disk %s, errno = %d", + filename,inode->type, cufs->device_name, errno); + + + /* + * Save offset of the directory entry + * where we left off above. Since realloc + * can change the location of the memory referenced + * by buf, we will need this offset to return + * to where we left off. + */ + + old_directory_entry_offset = (char *)directory_entry - buf; + + + /* + * Update inode file size to reflect this + * this addition. Below we will update + * inode on the disk block. + */ + + parent_inode->file_size += cufs->fs_block_size; + + buf_size = parent_inode->file_size; + + buf = realloc(buf,buf_size); + + if (buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to realloc after directory expansion parent inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + directory_inode_num,filename,inode->type, cufs->device_name, errno); + if (directory == NULL) { + free(buf); + } + + return -1; + + } + + + /* + * Initialize newly allocated filesystem + * data block. + */ + + buf_new_fs_block = buf - cufs->fs_block_size; + + bzero(buf_new_fs_block,cufs->fs_block_size); + + directory_entry = (cflsh_usfs_directory_entry_t *)(buf + old_directory_entry_offset); + + + /* + * Add new directory entry where we left + * off. + */ + + directory_entry->flags |= CFLSH_USFS_DIR_VAL; + directory_entry->inode_number = inode->index; + directory_entry->type = inode->type; + strcpy(directory_entry->filename,filename); + + + } + + if (cusfs_write_data_object(cufs,parent_inode,buf,&buf_size,0)) { + + + + CUSFS_TRACE_LOG_FILE(1,"Failed to write parent inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + directory_inode_num,filename,inode->type, cufs->device_name, errno); + + + if (directory == NULL) { + free(buf); + } + + return -1; + + } + + /* + * Update directory modification + * time and maybe file_size if changed above. + */ + +#if !defined(__64BIT__) && defined(_AIX) + parent_inode->mtime = cflsh_usfs_time64(NULL); +#else + parent_inode->mtime = time(NULL); +#endif /* not 32-bit AIX */ + + cusfs_update_inode(cufs,parent_inode,0); + + if (directory == NULL) { + free(buf); + } + + CUSFS_TRACE_LOG_FILE(9,"filename = %s of type %d on disk %s, rc = %d", + filename,inode->type, cufs->device_name, rc); + + return rc; +} + + + +/* + * NAME: cusfs_find_inode_num_from_directory_by_name + * + * FUNCTION: Find inode number for the filename + * in the specified directory. + * + * + * INPUTS: + * NONE + * + * RETURNS: NULL - error, otherwise success + * + * + */ +cflsh_usfs_inode_t *cusfs_find_inode_num_from_directory_by_name(cflsh_usfs_t *cufs, + char *name,char *buf, + int buf_size,int flags) +{ + cflsh_usfs_inode_t *inode = NULL; + cflsh_usfs_directory_t *directory_hdr; + cflsh_usfs_directory_entry_t *directory_entry; + + + if (strlen(name) >= CFLSH_USFS_DIR_MAX_NAME) { + + CUSFS_TRACE_LOG_FILE(1,"filename = %s too long %d bytes", + name,strlen(name)); + + errno = EINVAL; + return NULL; + + } + + /* + * Remove file entry to directory + */ + + directory_hdr = (cflsh_usfs_directory_t *)buf; + + directory_entry = directory_hdr->entry; + + while (directory_entry < (cflsh_usfs_directory_entry_t *)((buf + buf_size) - sizeof(*directory_entry))) { + + if ((directory_entry->flags & CFLSH_USFS_DIR_VAL) && + !(strcmp(directory_entry->filename,name))) { + + CUSFS_TRACE_LOG_FILE(9,"Found directory entry to be requested with filename = %s of type 0x%x", + directory_entry->filename,directory_entry->type); + + inode = cusfs_get_inode_from_index(cufs,directory_entry->inode_number,0); + + + break; + } + + directory_entry++; + } + + return inode; +} + +/* + * NAME: cusfs_remove_directory_entry + * + * FUNCTION: Remove a directory entry for the specified + * inode + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +int cusfs_remove_directory_entry(cflsh_usfs_t *cufs,cflsh_usfs_inode_t *inode, + char *filename,int flags) +{ + int rc = 0; + char *buf; + int buf_size; + cflsh_usfs_directory_t *directory_hdr; + cflsh_usfs_directory_entry_t *directory_entry; + int remove_entry; + cflsh_usfs_data_obj_t *directory; + cflsh_usfs_inode_t *parent_inode = NULL; + + + if (strlen(filename) >= CFLSH_USFS_DIR_MAX_NAME) { + + CUSFS_TRACE_LOG_FILE(1,"filename = %s too long %d bytes", + filename,strlen(filename)); + + errno = EINVAL; + return -1; + + } + + directory = cusfs_find_data_obj_from_inode(cufs,inode->directory_inode,0); + + if (directory == NULL) { + + + parent_inode = cusfs_get_inode_from_index(cufs,inode->directory_inode,0); + + if (parent_inode == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to get parent inode 0x%llx for file = %s of type %d on disk %s", + inode->directory_inode,filename,inode->type, cufs->device_name); + + + return -1; + } + + + buf_size = parent_inode->file_size; + + buf = malloc(buf_size); + + if (buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc data buffer for parent inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + inode->directory_inode,filename,inode->type, cufs->device_name, errno); + + + + return -1; + } + + + } else { + + /* + * Only lock the directory structure + * if it is in use + */ + + CUSFS_LOCK(directory->lock); + parent_inode = directory->inode; + + buf = directory->data_buf; + + buf_size = directory->data_buf_size; + } + + + + if (cusfs_read_data_object(cufs,parent_inode,buf,&buf_size,0)) { + + + + CUSFS_TRACE_LOG_FILE(1,"Failed to read parent inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + inode->directory_inode,filename,inode->type, cufs->device_name, errno); + + if (directory == NULL) { + free(buf); + } else { + CUSFS_UNLOCK(directory->lock); + } + + return -1; + + } + + /* + * Remove file entry to directory + */ + + directory_hdr = (cflsh_usfs_directory_t *)buf; + + directory_entry = directory_hdr->entry; + + remove_entry = FALSE; + + while (directory_entry < (cflsh_usfs_directory_entry_t *)((buf + buf_size) - sizeof(*directory_entry))) { + + if ((directory_entry->flags & CFLSH_USFS_DIR_VAL) && + (directory_entry->inode_number == inode->index)) { + + //?? Add more validation of entry + + CUSFS_TRACE_LOG_FILE(9,"Found directory entry to be removed with filename = %s of type 0x%x", + directory_entry->filename,directory_entry->type); + + + bzero(directory_entry,sizeof(*directory_entry)); + remove_entry = TRUE; + break; + } + + directory_entry++; + } + + if (!remove_entry) { + + + CUSFS_TRACE_LOG_FILE(1,"Could not find directory entry to remove for = %s of type %d on disk %s, errno = %d", + filename,inode->type, cufs->device_name, errno); + + + if (directory == NULL) { + free(buf); + } else { + CUSFS_UNLOCK(directory->lock); + } + + return -1; + + } + + if (cusfs_write_data_object(cufs,parent_inode,buf,&buf_size,0)) { + + + + CUSFS_TRACE_LOG_FILE(1,"Failed to write parent inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + inode->directory_inode,filename,inode->type, cufs->device_name, errno); + + + + if (directory == NULL) { + free(buf); + } else { + CUSFS_UNLOCK(directory->lock); + } + + return -1; + + } + + /* + * Update directory modification + * time. + */ + +#if !defined(__64BIT__) && defined(_AIX) + parent_inode->mtime = cflsh_usfs_time64(NULL); +#else + parent_inode->mtime = time(NULL); +#endif /* not 32-bit AIX */ + + cusfs_update_inode(cufs,parent_inode,0); + + if (directory == NULL) { + free(buf); + } else { + CUSFS_UNLOCK(directory->lock); + } + + return rc; +} + + +/* + * NAME: cusfs_check_directory_empty + * + * FUNCTION: Remove a directory entry for the specified + * inode + * + * + * INPUTS: + * NONE + * + * RETURNS: 1 if directory is empty, 0 otherwise. + * + * + */ +int cusfs_check_directory_empty(cflsh_usfs_t *cufs,cflsh_usfs_inode_t *inode, + char *filename,int flags) +{ + char *buf; + int buf_size; + cflsh_usfs_directory_t *directory_hdr; + cflsh_usfs_directory_entry_t *directory_entry; + int found_entry = 0; + cflsh_usfs_data_obj_t *directory; + + + if (inode->type != CFLSH_USFS_INODE_FILE_DIRECTORY) { + + + CUSFS_TRACE_LOG_FILE(1,"This inode is not a directory %s of type %d on disk %s, errno = %d", + filename,inode->type, cufs->device_name, errno); + + + + return 0; + } + + + + directory = cusfs_find_data_obj_from_inode(cufs,inode->index,0); + + if (directory == NULL) { + + + buf_size = inode->file_size; + + buf = malloc(buf_size); + + if (buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc data buffer for parent inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + inode->index,filename,inode->type, cufs->device_name, errno); + + + + return 0; + } + + + } else { + + + + buf = directory->data_buf; + + buf_size = directory->data_buf_size; + } + + + + if (cusfs_read_data_object(cufs,inode,buf,&buf_size,0)) { + + + + CUSFS_TRACE_LOG_FILE(1,"Failed to read inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + inode->index,filename,inode->type, cufs->device_name, errno); + + if (directory == NULL) { + free(buf); + } + + return 0; + + } + + /* + * See if any entries exist in this directory + */ + + directory_hdr = (cflsh_usfs_directory_t *)buf; + + directory_entry = directory_hdr->entry; + + found_entry = FALSE; + + while (directory_entry < (cflsh_usfs_directory_entry_t *)((buf + buf_size) - sizeof(*directory_entry))) { + + if (directory_entry->flags & CFLSH_USFS_DIR_VAL) { + + + CUSFS_TRACE_LOG_FILE(9,"Found directory entry with filename = %s of type 0x%x", + directory_entry->filename,directory_entry->type); + + + found_entry = TRUE; + break; + } + + directory_entry++; + } + + if (directory == NULL) { + free(buf); + } + + return !found_entry; +} + + +/* + * NAME: cusfs_list_files_directory + * + * FUNCTION: List all files in a directory + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise error + * + * + */ +int cusfs_list_files_directory(cflsh_usfs_t *cufs,cflsh_usfs_inode_t *inode, + char *filename,int flags) +{ + int rc = 0; + char *buf; + int buf_size; + cflsh_usfs_directory_t *directory_hdr; + cflsh_usfs_directory_entry_t *directory_entry; + cflsh_usfs_data_obj_t *directory; + char time_string[TIMELEN+1]; + char timebuf[TIMELEN+1]; + + + if (inode->type != CFLSH_USFS_INODE_FILE_DIRECTORY) { + + + CUSFS_TRACE_LOG_FILE(1,"This inode is not a directory %s of type %d on disk %s, errno = %d", + filename,inode->type, cufs->device_name, errno); + + + errno = EINVAL; + return -1; + } + + + + directory = cusfs_find_data_obj_from_inode(cufs,inode->index,0); + + if (directory == NULL) { + + + buf_size = inode->file_size; + + buf = malloc(buf_size); + + if (buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc data buffer for parent inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + inode->index,filename,inode->type, cufs->device_name, errno); + + + + return -1; + } + + + } else { + + + + buf = directory->data_buf; + + buf_size = directory->data_buf_size; + } + + + + if (cusfs_read_data_object(cufs,inode,buf,&buf_size,0)) { + + + + CUSFS_TRACE_LOG_FILE(1,"Failed to read inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + inode->index,filename,inode->type, cufs->device_name, errno); + + if (directory == NULL) { + free(buf); + } + + return -1; + + } + + /* + * See if any entries exist in this directory + */ + + directory_hdr = (cflsh_usfs_directory_t *)buf; + + directory_entry = directory_hdr->entry; + + /* + * ?? TODO: This printing out of data needs to move to caller + * Instead we may need to provide data for ls to work + */ + + printf("mode\tuid\tgid\tmtime \tsize\tinode\tfilename\n"); + + while (directory_entry < (cflsh_usfs_directory_entry_t *)((buf + buf_size) - sizeof(*directory_entry))) { + + if (directory_entry->flags & CFLSH_USFS_DIR_VAL) { + + inode = cusfs_get_inode_from_index(cufs,directory_entry->inode_number,0); + + if (inode) { + +#if !defined(__64BIT__) && defined(_AIX) + sprintf(time_string,"%s",ctime64_r(&(inode->mtime),timebuf)); +#else + sprintf(time_string,"%s",ctime_r(&(inode->mtime),timebuf)); +#endif + /* + * Remove newline from time_string + */ + + time_string[strlen(time_string) - 1] = '\0'; + + printf("%o\t%d\t%d\t%s\t0x%"PRIX64"\t0x%"PRIX64"\t%s\n", + inode->mode_flags, inode->uid, inode->gid, + time_string, + inode->file_size,inode->index,directory_entry->filename); + + free(inode); + } else { + printf("0x%"PRIX64" %s\n",directory_entry->inode_number,directory_entry->filename); + } + } + + directory_entry++; + } + + if (directory == NULL) { + free(buf); + } + + return rc; +} + + + +/* + * NAME: cusfs_get_next_directory_entry + * + * FUNCTION: Get the next directory entry after file->offset. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise error + * + * + */ +int cusfs_get_next_directory_entry(cflsh_usfs_t *cufs,cflsh_usfs_data_obj_t *directory, + cflsh_usfs_directory_entry_t *return_dir_entry, cflash_offset_t *offset, + int flags) +{ + int rc = 0; + char *buf; + int buf_size; + int found_entry = FALSE; + cflash_offset_t next_rel_offset; + cflsh_usfs_directory_t *directory_hdr; + cflsh_usfs_directory_entry_t *directory_entry; + + + if (directory == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"No directory specified for device %s", + cufs->device_name); + + + errno = EINVAL; + return -1; + } + + CUSFS_TRACE_LOG_FILE(9,"directory %s with offset 0x%llx on disk %s", + directory->filename,offset,cufs->device_name); + + + if (directory->inode->type != CFLSH_USFS_INODE_FILE_DIRECTORY) { + + + CUSFS_TRACE_LOG_FILE(1,"This inode is not a directory %s of type %d on disk %s", + directory->filename,directory->inode->type, cufs->device_name); + + + errno = EINVAL; + return -1; + } + + + buf_size = directory->inode->file_size; + + if (*offset > directory->inode->file_size) { + + CUSFS_TRACE_LOG_FILE(1,"This offset 0x%llx is beyond the end of this directory %s on this disk", + *offset,directory->filename,cufs->device_name); + + errno = EINVAL; + return -1; + + } + + buf = malloc(buf_size); + + if (buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc data buffer for directory = %s on disk %s, errno = %d", + directory->filename,cufs->device_name, errno); + + return -1; + } + + + CUSFS_LOCK(directory->lock); + + if (cusfs_read_data_object(cufs,directory->inode,buf,&buf_size,0)) { + + + + CUSFS_TRACE_LOG_FILE(1,"Failed to read inode data 0x%llx for file = %s of type %d on disk %s, errno = %d", + directory->inode->index,directory->filename,directory->inode->type, + cufs->device_name); + + free(buf); + CUSFS_UNLOCK(directory->lock); + return -1; + + } + + directory_hdr = (cflsh_usfs_directory_t *)buf; + + if (directory_hdr->start_marker != CLFSH_USFS_DIRECTORY_SM) { + + + CUSFS_TRACE_LOG_FILE(1,"Invalid directory header, start_marker = 0x%llx for file = %s of type %d on disk %s, errno = %d", + directory_hdr->start_marker,directory->inode->index,directory->filename,directory->inode->type, + cufs->device_name); + + free(buf); + CUSFS_UNLOCK(directory->lock); + return -1; + } + + directory_entry = directory_hdr->entry; + + CUSFS_TRACE_LOG_FILE(7,"directory_entry %p, buf %p, buf_size = 0x%x, sizeof directory_entry = 0x%x", + directory_entry,buf,buf_size,sizeof(*directory_entry)); + + next_rel_offset = (cflash_offset_t)((char *)directory_entry - buf); + + while (directory_entry < (cflsh_usfs_directory_entry_t *)((buf + buf_size) - sizeof(*directory_entry))) { + + CUSFS_TRACE_LOG_FILE(9,"directory entry loop = %s with inode 0x%llx in directory %s at offset 0x%llx on disk %s", + directory_entry->filename,directory_entry->inode_number, directory->filename,*offset, + cufs->device_name); + + + /* + * Advance offset to next + * directory entry + */ + next_rel_offset += sizeof(*directory_entry); + + + if (directory_entry >= (cflsh_usfs_directory_entry_t *)(buf + *offset)) { + + if (directory_entry->flags & CFLSH_USFS_DIR_VAL) { + + *return_dir_entry = *directory_entry; + + found_entry = TRUE; + + CUSFS_TRACE_LOG_FILE(9,"directory entry found = %s with inode 0x%llx in directory %s on disk %s", + directory_entry->filename,directory_entry->inode_number, directory->filename, + cufs->device_name); + /* + * Advance offset to next + * directory entry + */ + *offset = next_rel_offset; + break; + } + } + + + directory_entry++; + } + + free(buf); + +#if !defined(__64BIT__) && defined(_AIX) + directory->inode->atime = cflsh_usfs_time64(NULL); +#else + directory->inode->atime = time(NULL); +#endif /* not 32-bit AIX */ + cusfs_update_inode(cufs,directory->inode,0); + CUSFS_UNLOCK(directory->lock); + + if (!found_entry) { + + CUSFS_TRACE_LOG_FILE(5,"No entries found in directory %s with offset 0x%llx on disk %s", + directory->filename,offset,cufs->device_name); + rc = -1; + } + + return rc; +} + + + + + +/* + * NAME: cusfs_clear_data_obj + * + * FUNCTION: Clear data object will all zeroes for + * the associated inode + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 succesa, otherwise failure + * + * + */ +int cusfs_clear_data_obj(cflsh_usfs_t *cufs,cflsh_usfs_inode_t *inode,int flags) +{ + int rc = 0; + char *buf; + int buf_size; + + + buf_size = inode->file_size; + + buf = malloc(buf_size); + + if (buf == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc inode data buffer for 0x%llx of type %d on disk %s, errno = %d", + inode->index,inode->type, cufs->device_name, errno); + + return -1; + } + + + + if (cusfs_read_data_object(cufs,inode,buf,&buf_size,0)) { + + + + CUSFS_TRACE_LOG_FILE(1,"Failed to read inode data 0x%llx of type %d on disk %s, errno = %d", + inode->index,inode->type, cufs->device_name, errno); + + free(buf); + + return -1; + + } + + /* + * ?? TODO: This code may need to iterate a block at a time + * to avoid large buffers in the future. + */ + + bzero(buf,buf_size); + + if (cusfs_write_data_object(cufs,inode,buf,&buf_size,0)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to write inode data 0x%llx of type %d on disk %s, errno = %d", + inode->index,inode->type, cufs->device_name, errno); + + + + rc = -1; + + } + + free(buf); + + return rc; +} + +/* + * NAME: cusfs_initialize_directory + * + * FUNCTION: Initialize a directory + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 succesa, otherwise failure + * + * + */ +int cusfs_initialize_directory(cflsh_usfs_t *cufs,cflsh_usfs_inode_t *inode,int flags) +{ + int rc = 0; + char *buf; + int buf_size; + cflsh_usfs_directory_t *directory_hdr; + + + if (inode->type != CFLSH_USFS_INODE_FILE_DIRECTORY) { + + return -1; + } + + buf_size = inode->file_size; + + buf = malloc(buf_size); + + if (buf == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc inode data buffer for 0x%llx of type %d on disk %s, errno = %d", + inode->index,inode->type, cufs->device_name, errno); + + return -1; + } + + + + if (cusfs_read_data_object(cufs,inode,buf,&buf_size,0)) { + + + + CUSFS_TRACE_LOG_FILE(1,"Failed to read inode data 0x%llx of type %d on disk %s, errno = %d", + inode->index,inode->type, cufs->device_name, errno); + + free(buf); + + return -1; + + } + + /* + * ?? TODO: This code may need to iterate a block at a time + * to avoid large buffers in the future. + */ + + bzero(buf,buf_size); + + directory_hdr = (cflsh_usfs_directory_t *)buf; + + directory_hdr->start_marker = CLFSH_USFS_DIRECTORY_SM; + + if (cusfs_write_data_object(cufs,inode,buf,&buf_size,0)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to write inode data 0x%llx of type %d on disk %s, errno = %d", + inode->index,inode->type, cufs->device_name, errno); + + + + rc = -1; + + } + + free(buf); + + return rc; +} + + +/* + * NAME: cusfs_create_data_obj + * + * FUNCTION: Creates a filesystem data object of + * of inode_type (file, directory, symlink etc). + * + * NOTES: Assumes caller has parent directory's lock. + * + * INPUTS: + * NONE + * + * RETURNS: Null failure, Otherwise success + * + * + */ + +cflsh_usfs_data_obj_t *cusfs_create_data_obj(cflsh_usfs_t *cufs, + char *filename, + cflsh_usfs_inode_file_type_t inode_type, + uint64_t parent_directory_inode_num, + mode_t mode_flags, uid_t uid, gid_t gid, + char *sym_link_name, + int flags) +{ + int num_blocks = 1; + uint64_t block_num = 0; + cflsh_usfs_data_obj_t *file; + cflsh_usfs_inode_t *inode; + uint64_t file_size; + int alloc_flags = 0; + + + + if (cufs == NULL) { + + + errno = EINVAL; + return NULL; + } + + + CUSFS_TRACE_LOG_FILE(9,"file = %s of type %d on disk %s", + filename,inode_type, cufs->device_name); + + + if (inode_type == CFLSH_USFS_INODE_FILE_REGULAR) { + alloc_flags = CFLSH_USFS_ALLOC_READ_AHEAD; + } + + file = cusfs_alloc_file(cufs,filename,alloc_flags); + + if (file == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"malloc of file %s of type %d for failed for disk %s", + filename,inode_type, cufs->device_name); + return NULL; + } + + if (inode_type != CFLSH_USFS_INODE_FILE_SYMLINK) { + + /* + * If this is a symlink then we store the filename in + * union sharing the same space as the inode_ptrs + */ + + block_num = cusfs_get_data_blocks(cufs,&num_blocks,0); + + if (num_blocks == 0) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block for file = %s of type %d on disk %s", + filename,inode_type, cufs->device_name); + + cusfs_free_file(file); + return NULL; + } + + } + + /* + * For directory inodes always provide a file + * size of the filesystem block size. + * For other inodes, set the file size to 0 + * even though cusfs_get_inode will allocate + * one filesystem block size. + */ + if (inode_type == CFLSH_USFS_INODE_FILE_DIRECTORY) { + + file_size = cufs->fs_block_size; + + } else { + + file_size = 0; + + } + + inode = cusfs_get_inode(cufs,parent_directory_inode_num,inode_type, + mode_flags,uid,gid,&block_num,num_blocks,file_size, + sym_link_name, + 0); + + if (inode == NULL) { + + if (inode_type != CFLSH_USFS_INODE_FILE_SYMLINK) { + cusfs_release_data_blocks(cufs,block_num,num_blocks,0); + } + + + CUSFS_TRACE_LOG_FILE(1,"Failed to get inode for file = %s of type %d on disk %s", + filename,inode_type, cufs->device_name); + cusfs_free_file(file); + + return NULL; + } + + file->inode = inode; + + CUSFS_TRACE_LOG_FILE(9,"inode->index = 0x%llx on disk %s", + inode->index, cufs->device_name); + + if (flags & CFLSH_USFS_ROOT_DIR) { + + file->flags |= CFLSH_USFS_ROOT_DIR; + + inode->flags |= CFLSH_USFS_INODE_ROOT_DIR; + + cusfs_update_inode(cufs,inode,0); + + } else { + + CUSFS_TRACE_LOG_FILE(9,"Adding inode->index = 0x%llx with filename %s", + inode->index, filename); + if (cusfs_add_directory_entry(cufs,parent_directory_inode_num,inode,filename,0)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to add directory entry to parent inode 0x%llx for file = %s of type %d on disk %s", + parent_directory_inode_num,filename,inode_type, cufs->device_name); + + + if (inode_type != CFLSH_USFS_INODE_FILE_SYMLINK) { + cusfs_release_inode(cufs,inode,0); + } + + cusfs_release_data_blocks(cufs,block_num,num_blocks,0); + cusfs_free_file(file); + + + return NULL; + } + + } + + if (inode_type == CFLSH_USFS_INODE_FILE_DIRECTORY) { + + /* + * For new directory, we need to bzero its data blocks + */ + + if (cusfs_initialize_directory(cufs,inode,0)) { + + if (inode_type != CFLSH_USFS_INODE_FILE_SYMLINK) { + cusfs_release_inode(cufs,inode,0); + } + + cusfs_release_data_blocks(cufs,block_num,num_blocks,0); + cusfs_free_file(file); + + + return NULL; + } + } + + file->data_buf_size = cufs->fs_block_size; + + file->data_buf = malloc(file->data_buf_size); + + + if (file->data_buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to to malloc file->data_bu of size 0x%x for file = %s of type %d on disk %s", + file->data_buf_size,filename,inode_type, cufs->device_name); + + + if (inode_type != CFLSH_USFS_INODE_FILE_SYMLINK) { + cusfs_release_inode(cufs,inode,0); + } + + cusfs_release_data_blocks(cufs,block_num,num_blocks,0); + cusfs_free_file(file); + + + return NULL; + + } + + if (file) { + + CUSFS_LOCK(cufs->lock); + CUSFS_Q_NODE_TAIL(cufs->filelist_head,cufs->filelist_tail,file,prev,next); + CUSFS_UNLOCK(cufs->lock); + + } + + CUSFS_TRACE_LOG_FILE(5,"Created file = %s of type %d on disk %s", + filename,inode_type, cufs->device_name); + + + return file; +} + + + + +/* + * NAME: cusfs_free_data_obj + * + * FUNCTION: Free a filesystem data object + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_free_data_obj(cflsh_usfs_t *cufs,cflsh_usfs_data_obj_t *file,int flags) +{ + int rc = 0; + + if (cufs == NULL) { + + + errno = EINVAL; + return -1; + } + + if (file == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Null file object passed for disk %s", + cufs->device_name); + return -1; + } + + if (file->inode == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Null file->inode for file %s object passed for disk %s", + file->filename,cufs->device_name); + return -1; + } + + CUSFS_LOCK(file->lock); + + if (file->inode->type == CFLSH_USFS_INODE_FILE_DIRECTORY) { + + + if (file->inode->flags & CFLSH_USFS_INODE_ROOT_DIR) { + + + CUSFS_TRACE_LOG_FILE(1,"Can not remove root directory file %s object passed for disk %s", + file->filename,cufs->device_name); + CUSFS_UNLOCK(file->lock); + return -1; + } + + + + if ((cusfs_check_directory_empty(cufs,file->inode,file->filename,0)) == 0) { + + + + CUSFS_TRACE_LOG_FILE(1,"directory file %s object is not empty for disk %s", + file->filename,cufs->device_name); + CUSFS_UNLOCK(file->lock); + return -1; + + } + + /* ??TODO2: Need a force flag to allow this directory to be deleted + and delete the hierarchy underneath. + */ + + } + + + // Remove from parent directory + + if (cusfs_remove_directory_entry(cufs,file->inode,file->filename,0)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to remove file %s of type %s from directory 0x%llx on disk %s", + file->filename,file->inode->type, file->inode->directory_inode,cufs->device_name); + + CUSFS_UNLOCK(file->lock); + return -1; + } + + + cusfs_release_inode_and_data_blocks(cufs,file->inode,0); + + CUSFS_UNLOCK(file->lock); + + CUSFS_LOCK(cufs->lock); + + CUSFS_DQ_NODE(cufs->filelist_head,cufs->filelist_tail,file,prev,next); + + CUSFS_UNLOCK(cufs->lock); + + + cusfs_free_file(file); + + + return rc; +} + + +/* + * NAME: cusfs_move_data_obj + * + * FUNCTION: Move a data object in a filesystem + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_mv_data_obj(cflsh_usfs_t *cufs,cflsh_usfs_data_obj_t *file,char *new_full_pathname,int flags) +{ + int rc = 0; + char full_path[PATH_MAX]; + char *new_filename; + cflsh_usfs_data_obj_t *new_parent_dir; + + + + if (cufs == NULL) { + + + errno = EINVAL; + return -1; + } + + if (file == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Null file object passed for disk %s", + cufs->device_name); + return -1; + } + + if (file->inode == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Null file->inode for file %s object passed for disk %s", + file->filename,cufs->device_name); + return -1; + } + + if (new_full_pathname == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Null new_full_pathname is null for file %s passed for disk %s", + file->filename,cufs->device_name); + return -1; + } + + + + strcpy(full_path,new_full_pathname); + + /* + * Get parent directory of new path + */ + + new_filename = strrchr(full_path,'/'); + + new_filename[0] = '\0'; + + new_filename++; + + + new_parent_dir = cusfs_lookup_data_obj(cufs,full_path,0); + + if (new_parent_dir == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to find new directory %s for disk %s", + new_parent_dir,file->filename,cufs->device_name); + return -1; + } + + + /* + * First add file to new location. Then remove it from + * old location. + */ + + CUSFS_LOCK(new_parent_dir->lock); + if (cusfs_add_directory_entry(cufs,new_parent_dir->inode->index,file->inode,new_filename,0)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to aadd file %s to new directory for disk %s", + file->filename,cufs->device_name); + CUSFS_UNLOCK(new_parent_dir->lock); + return -1; + } + + CUSFS_UNLOCK(new_parent_dir->lock); + + + if (cusfs_remove_directory_entry(cufs,file->inode,file->filename,0)) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to remove file %s to new directory for disk %s", + file->filename,cufs->device_name); + return -1; + } + + strcpy(file->filename,new_filename); + + + return rc; +} + +/* + * NAME: cusfs_find_root_inode + * + * FUNCTION: This code code finds and returns the inode + * of the root directory of this filesystem. + * + * + * INPUTS: + * + * + * NOTES: The root directory inode should be the first + * inode in the inode table. However this code + * allows it to be anywhere in this disk block + * of inodes. + * + * + * RETURNS: NULL failure, Otherwise success + * + * + */ + +cflsh_usfs_inode_t *cusfs_find_root_inode(cflsh_usfs_t *cufs, int flags) +{ + cflsh_usfs_inode_t *inode = NULL; + cflsh_usfs_inode_t *inode_ptr = NULL; + int buf_size; + int rc = 0; + + if (cufs == NULL) { + + + errno = EINVAL; + return inode; + } + + + CUSFS_LOCK(cufs->inode_table_lock); + + /* + * Only read the first disk block of the inode table for the root directoy. + */ + + buf_size = cufs->disk_block_size; + + rc = cblk_read(cufs->chunk_id,cufs->inode_tbl_buf,cufs->superblock.inode_table_start_lba,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"reaad of inode table at lba = 0x%llx failed with errno = %d", + cufs->superblock.inode_table_start_lba,errno); + CUSFS_UNLOCK(cufs->inode_table_lock); + return inode; + + } + + + inode_ptr = cufs->inode_tbl_buf; + + while (inode_ptr < (cflsh_usfs_inode_t *)(cufs->inode_tbl_buf + buf_size)) { + + if ((inode_ptr->flags & CFLSH_USFS_INODE_INUSE) && + (inode_ptr->flags & CFLSH_USFS_INODE_ROOT_DIR)) { + + + if ((inode_ptr->start_marker != CLFSH_USFS_INODE_SM) || + (inode_ptr->end_marker != CLFSH_USFS_INODE_EM)) { + + CUSFS_TRACE_LOG_FILE(1,"Invalid inode markers inode_ptr = %p offset = 0x%llx", + inode_ptr, (cufs->inode_tbl_buf + buf_size)); + + CUSFS_UNLOCK(cufs->inode_table_lock); + return inode; + } + + + inode = (cflsh_usfs_inode_t *) malloc(sizeof(cflsh_usfs_inode_t)); + + if (inode == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"malloc of inode failed with errno = %d", + errno); + + CUSFS_UNLOCK(cufs->inode_table_lock); + return inode; + + } + + + + *inode = *inode_ptr; + + break; + } + inode_ptr++; + } + + + /* + * ?? Add validation check that this lba is not the meta data + * at the beginning of the disk. + */ + + CUSFS_TRACE_LOG_FILE(5,"root inode found = 0x%p inode->index = 0x%llx on device = %s", + inode,inode->index,cufs->device_name); + + + + CUSFS_UNLOCK(cufs->inode_table_lock); + return inode; +} diff --git a/src/usfs/cflsh_usfs_disk.h b/src/usfs/cflsh_usfs_disk.h new file mode 100644 index 00000000..96919ad3 --- /dev/null +++ b/src/usfs/cflsh_usfs_disk.h @@ -0,0 +1,633 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* bos720 src/bos/usr/ccs/lib/libcflsh_block/cflsh_usfs_disk.h 1.6 */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/* %Z%%M% %I% %W% %G% %U% */ + + +/* + * COMPONENT_NAME: (sysxcflashusfs) CAPI Flash user space file system library + * + * FUNCTIONS: Data structures of the filesystem that reside on + * a CAPI flash disk. + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +#ifndef _H_CFLASH_USFS_DISK +#define _H_CFLASH_USFS_DISK +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _AIX +#include +#else +#include +#endif +#if !defined(_AIX) && !defined(_MACOSX) +#include +#include +#include /* For SYS_xxx definitions */ +#endif /* !_AIX && !_MACOSX */ +#include +#include +#ifdef _OS_INTERNAL +#include +#else +#include +#endif +#include +#include "cflsh_usfs_client.h" + +#ifndef TIMELEN +#define TIMELEN 26 /* Linux does have a define for the minium size of the a timebuf */ + /* However linux man pages say it is 26 */ +#endif + + +/************************************************************************/ +/* CAPI block library settings */ +/************************************************************************/ +#define CFLSH_USFS_DISK_MAX_CMDS 0x2000 /* Maxium number of requests */ + /* this process will queue to a */ + /* disk for this filesystem. */ + +/************************************************************************/ +/* Filesystem metadata offsets in 4K sectors */ +/* */ +/* */ +/* ----------------------- */ +/* sector 0 */ +/* */ +/* UNUSED */ +/* ----------------------- */ +/* sector 1 */ +/* */ +/* SUPER BLOCK */ +/* ----------------------- */ +/* sector 2 */ +/* */ +/* Free Block Table stats */ +/* ----------------------- */ +/* sector 3 */ +/* */ +/* Inode table stats */ +/* ----------------------- */ +/* sector 4-0xf */ +/* */ +/* UNUSED */ +/* ----------------------- */ +/* sector 0x10 - 0x30 */ +/* */ +/* JOURNAL (128K) */ +/* ----------------------- */ +/* sector 0x40 - m */ +/* */ +/* FREE BLOCK TABLE */ +/* ----------------------- */ +/* sector m - n */ +/* */ +/* INODE TABLE */ +/* ----------------------- */ +/* sector n - last lba */ +/* */ +/* DATA BLOCKs */ +/* ----------------------- */ +/* */ +/************************************************************************/ + + +#define CFLSH_USFS_PRIMARY_SB_OFFSET 1 /* Disk sector for primary */ + /* super block. */ +#define CFLSH_USFS_BACKUP_SB_OFFSET 100000 + +#define CFLSH_USFS_FBLK_TBL_STATS_OFFSET 2 /* Disk sector for free */ + /* block table statistics */ +#define CFLSH_USFS_INODE_TBL_STATS_OFFSET 3 /* Disk sector for Inode */ + /* table statistics */ + + + +#define CFLSH_USFS_JOURNAL_START_TABLE_LBA 0x10 + +#define CFLSH_USFS_JOURNAL_TABLE_SIZE 0x20 /* 128 K Log */ + +#define CFLSH_USFS_FREE_BLOCK_TABLE_LBA 0x40 + +/************************************************************************/ +/* Super block data structure for this filesystem. */ +/************************************************************************/ + +#define CFLSH_USFS_FORCE_SB_CREATE 0x0001 /* Force the creation of */ + /* of superblock even if one */ + /* already exist. */ + + +typedef enum { + CFLSH_OS_TYPE_INVAL = 0, /* Invalid OS type */ + CFLSH_OS_TYPE_AIX = 1, /* AIX OS */ + CFLSH_OS_TYPE_LINUX = 2, /* Linux OS */ + CFLSH_OS_TYPE_MACOSX = 3, /* Mac OSX */ +} cflsh_usfs_os_type_t; + + +#define CFLSH_USFS_BLOCK_SIZE_MIN 0x1000 /* Minimum data block size */ + /* allowed */ + +#define CFLSH_USFS_DEVICE_BLOCK_SIZE 0x1000 /* physical disk block size */ + +typedef struct cflsh_usfs_fs_id_s { + uint64_t val1; + uint64_t val2; + uint64_t val3; + uint64_t val4; +} cflsh_usfs_fs_id_t; + + +typedef struct cflsh_usfs_super_block_s { + uint64_t start_marker; +#define CLFSH_USFS_SB_SM __EYEC8('c','u','f','s','_','s','b','a') + uint32_t version; + uint32_t status; + uint64_t flags; +#define CFLSH_USFS_SB_MOUNTED 0x001 /* Filesystem is mounted */ +#define CFLSH_USFS_SB_FT_VAL 0x002 /* Free block table is written */ +#define CFLSH_USFS_SB_IT_VAL 0x004 /* Inode table is written */ +#define CFLSH_USFS_SB_JL_VAL 0x008 /* Journal is written */ +#define CFLSH_USFS_SB_ROOT_DIR 0x010 /* Root directory inode written */ + +/* ?? how to handle multiple mounts */ +/* ?? Need some indication of endianess of FS creator */ +/* ?? need filesystem identifier */ + + uint64_t fs_block_size; /* Block size referenced by inodes */ + /* This is the smallest */ + /* addressable data size in the */ + /* filesystem */ + uint64_t num_blocks; /* Size of file system in blocks */ + + + uint32_t disk_block_size; /* Block size used by disk */ + + uint64_t inode_table_stats_lba; /* start LBA of inode table */ + /* statistics. */ + uint64_t inode_table_start_lba; /* Start LBA of Inode table */ + uint64_t inode_table_size; /* Size in disk blocks of */ + /* node table */ + + uint64_t num_inodes; /* Number of inodes for this file- */ + /* system. */ + uint64_t num_inodes_per_disk_block;/* Number of inodes for this */ + /* filesystem. */ + uint64_t free_block_table_stats_lba;/* start LBA of free block table */ + /* statistics. */ + + uint64_t free_block_table_start_lba; /* start LBA of free block */ + /* table. */ + uint64_t free_block_table_size;/* Size in disk blocks of */ + /* Journal */ + + uint64_t journal_start_lba; /* start LBA of Journal */ + uint64_t journal_table_size; /* Size in disk blocks of */ + /* Journal */ + uint64_t num_journal_entries; /* Number of entries in the journal*/ + uint64_t start_data_blocks; /* Disk sector where data blocks */ + /* start */ + uint64_t fs_start_data_block; /* Filesystem block where data */ + /* starts. */ + cflsh_usfs_os_type_t os_type; /* OS type that created this */ + /* filesystem. */ + +#if !defined(__64BIT__) && defined(_AIX) + time64_t create_time; /* Creation time of fileystem */ + time64_t mount_time; /* Last time fileystem was mounted*/ + time64_t write_time; /* Last time fileystem was written*/ + time64_t fsck_time; /* Last time fileystem had fsck */ + /* run. */ +#else + time_t create_time; /* Creation time of fileystem */ + time_t mount_time; /* Last time fileystem was mounted*/ + time_t write_time; /* Last time fileystem was written*/ + time_t fsck_time; /* Last time fileystem had fsck */ + /* run. */ +#endif + + + /* ?? + + permissions, + access info + + type, + locking files per process or not. + + inode number of root directory + + */ + cflsh_usfs_fs_id_t fs_unique_id; /* File system unique id */ + + uint64_t end_marker; + +#define CLFSH_USFS_SB_EM __EYEC8('c','u','f','s','_','s','b','e') +} cflsh_usfs_super_block_t; + + +/************************************************************************/ +/* Free block table data structures for this filesystem. */ +/************************************************************************/ + +typedef struct cflsh_usfs_free_block_stats_s { + uint64_t start_marker; +#define CLFSH_USFS_FB_SM __EYEC8('c','u','f','s','_','f','b','a') + uint32_t version; + uint32_t status; + uint64_t flags; + uint64_t total_free_blocks; /* Total number of free blocks */ + /* in free block table. */ + uint64_t available_free_blocks; /* Available free blocks */ + /* in free block table. */ + + + uint64_t end_marker; + +#define CLFSH_USFS_FB_EM __EYEC8('c','u','f','s','_','f','b','e') +} cflsh_usfs_free_block_stats_t; + + +/************************************************************************/ +/* Inode data structures for this filesystem. */ +/************************************************************************/ + + + +typedef struct cflsh_usfs_inode_table_stats_s { + uint64_t start_marker; +#define CLFSH_USFS_IT_SM __EYEC8('c','u','f','s','_','i','t','a') + uint32_t version; + uint32_t status; + uint64_t flags; + uint64_t total_inodes; /* Total number of inodes in */ + /* inode table. */ + uint64_t available_inodes; /* Available inodes in inode */ + /* table. */ + + + uint64_t end_marker; + +#define CLFSH_USFS_IT_EM __EYEC8('c','u','f','s','_','i','t','e') +} cflsh_usfs_inode_table_stats_t; + + + + +#ifdef _REMOVE +typedef struct cflsh_usfs_inode_ptr_s { + uint64_t flags; + uint64_t start_lba; + uint64_t end_lba; +} cflsh_usfs_inode_ptr_t; +#endif /* _REMOVE */ + +typedef struct cflsh_usfs_inode_ptr_s { + uint64_t lba; +} cflsh_usfs_inode_ptr_t; + + +enum { + CFLSH_USFS_INODE_HDR_INVAL = 0, /* Invalid Inode header type */ + CFLSH_USFS_INODE_HDR_SINGLE = 1, /* Single indirect inode header */ + CFLSH_USFS_INODE_HDR_DOUBLE = 2, /* Double indirect inode header */ + CFLSH_USFS_INODE_HDR_TRIPLE = 3, /* Triple indirect inode header */ + +} cflash_usfs_inode_hdr_type_t; + +#ifdef _REMOVE + +typedef struct cflsh_usfs_inode_indirect_hdr_s { + uint64_t start_marker; +#define CLFSH_USFS_INODE_IND_SM __EYEC8('c','u','f','s','i','t','i','h') + uint64_t flags; + uint32_t version; + uint32_t status; + cflash_usfs_inode_hdr_type_t type; + uint32_t num_inode_ptrs; + cflsh_usfs_inode_ptr_t inode_ptr[1]; +} cflsh_usfs_inode_indirect_hdr_t; +#endif /* _REMOVE */ + +#define CFLSH_USFS_INODE_FLG_INDIRECT_ENTRIES 0x1 /* data blocks are */ + /* single indirect enteries */ +#define CFLSH_USFS_INODE_FLG_D_INDIRECT_ENTRIES 0x2/* data blocks are */ + /* double indirect enteries */ + +#define CFLSH_USFS_INODE_FLG_NEW_INDIRECT_BLK 0x1 + + +#define CFLSH_USFS_NUM_DIRECT_INODE_PTRS 12 +#define CFLSH_USFS_INDEX_SINGLE_INDIRECT 12 +#define CFLSH_USFS_INDEX_DOUBLE_INDIRECT 13 +#define CFLSH_USFS_INDEX_TRIPLE_INDIRECT 14 + + + +#define CFLSH_USFS_NUM_TOTAL_INODE_PTRS 15 + +typedef +enum { + CFLSH_USFS_INODE_FILE_INVAL = 0, /* Invalid Inode header type */ + CFLSH_USFS_INODE_FILE_DIRECTORY = 1, /* Directory */ + CFLSH_USFS_INODE_FILE_REGULAR = 2, /* Regular file */ + CFLSH_USFS_INODE_FILE_SYMLINK = 3, /* Symbolic link file */ + CFLSH_USFS_INODE_FILE_SPECIAL = 4, /* Special file */ +} cflsh_usfs_inode_file_type_t; + +#define CFLSH_USFS_SYM_LINK_MAX 256 + +typedef struct cflsh_usfs_inode_s { + uint64_t start_marker; +#define CLFSH_USFS_INODE_SM __EYEC8('c','u','f','s','_','i','t','a') + uint32_t version; + uint32_t status; + uint64_t flags; +#define CFLSH_USFS_INODE_INUSE 0x0001 /* Inode entry in use. */ +#define CFLSH_USFS_INODE_ROOT_DIR 0x0002 /* This is the root */ + /* directory of the */ + /* filesystem */ + + uint64_t file_size; /* File size in bytes */ + uint64_t index; + + mode_t mode_flags; /* 0777 bits used as r,w,x permissions */ + + uid_t uid; /* local uid */ + gid_t gid; /* local gid */ + + uint32_t reserved1; + uint32_t reserved2; + + cflsh_usfs_inode_file_type_t type; + + + //?? Add union so this space can be used for inlined data + // such a symlinks + union { + uint64_t ptrs[CFLSH_USFS_NUM_TOTAL_INODE_PTRS]; + char sym_link_path[CFLSH_USFS_SYM_LINK_MAX]; + }; + + + uint64_t directory_inode; /* Inode index of directory */ + uint64_t num_links; /* Number of links to this */ +#if !defined(__64BIT__) && defined(_AIX) + time64_t ctime; /* Inode change time - when */ + /* inode was last modified */ + time64_t mtime; /* Last time file content */ + /* was modified. */ + time64_t atime; /* Last time file was */ + /* accessed. */ +#else + time_t ctime; /* Inode change time - when */ + /* inode was last modified */ + time_t mtime; /* Last time file content */ + /* was modified. */ + time_t atime; /* Last time file was */ + /* accessed. */ +#endif + /* ?? + + timestamp for creating inode + data block for attributes. + */ + + /* + * Need to ensure inode is exactly 512 bytes in size. + * Larger sizes could be used in the future provided + * that they evenly fit in 4096. + * This allows a 4096 disk block to evenly fit inodes in + * it, without any of them spanning disk block boundaries. + * The inode processing code in this library assumes + * inodes do not span disk block boundaries. + */ + + char pad[144]; + + + uint64_t end_marker; +#define CLFSH_USFS_INODE_EM __EYEC8('c','u','f','s','_','i','t','e') +} cflsh_usfs_inode_t; + + + +/************************************************************************/ +/* free block table data structures for this filesystem. */ +/************************************************************************/ + +typedef struct cflsh_usfs_free_block_tble_hdr_s { + uint64_t start_marker; +#define CLFSH_USFS_FREE_TBL_SM __EYEC8('c','u','f','s','_','f','t','h') + uint64_t flags; +#define CFLSH_USFS_FBLK_TBL_COMPLETE 0x0001 /* Free table has been completed */ + /* This means it has been */ + /* fully populated on disk */ + uint32_t version; + uint32_t status; + uint64_t num_entries; + uint64_t entry[1]; +} cflsh_usfs_free_block_tble_hdr_t; + + +/************************************************************************/ +/* Journal data structures for this filesystem. */ +/************************************************************************/ + + +/* Journal entry types */ +typedef enum { + CFLSH_USFS_JOURNL_E_INVAL = 0, /* Invalid Journal type */ + CFLSH_USFS_JOURNAL_E_ADD_BLK = 1, /* Get Block Table Entry */ + CFLSH_USFS_JOURNAL_E_FREE_BLK = 2, /* Free Block Table Entry */ + CFLSH_USFS_JOURNAL_E_GET_INODE = 3, /* Free Inode */ + CFLSH_USFS_JOURNAL_E_FREE_INODE = 4, /* Free Inode */ + CFLSH_USFS_JOURNAL_E_ADD_INODE_DBLK = 5, /* Add data block to Inode */ + CFLSH_USFS_JOURNAL_E_FREE_INODE_DBLK = 6, /* Free data block from Inode */ + CFLSH_USFS_JOURNAL_E_FREE_DIR_E = 7, /* Free entry from directory */ + CFLSH_USFS_JOURNAL_E_INODE_FSIZE = 8, /* Inode file_size change */ +} cflsh_journal_entry_type_t; + +typedef struct cflsh_usfs_journal_entry_s { + uint64_t start_marker; +#define CLFSH_USFS_JOURNAL_E_SM __EYEC8('c','u','f','s','_','j','n','e') + cflsh_journal_entry_type_t type; + uint32_t reserved; /* Reserved for future use */ + union { + struct { /* For Add/Free Block Table Entry */ + uint64_t block_no; + } blk; + struct { /* For Free Inode Entry */ + uint64_t inode_num; + } inode; + struct { /* For Inode data block */ + uint64_t inode_num; + uint64_t block_num; + } inode_dblk; + struct { /* For Inode file size change */ + uint64_t old_file_size; + uint64_t new_file_size; + } inode_fsize; + struct { + uint64_t directory_inode_num; + uint64_t file_inode_num; + } dir; + + }; + +} cflsh_usfs_journal_entry_t; + + +typedef struct cflsh_usfs_journal_hdr_s { + uint64_t start_marker; +#define CLFSH_USFS_JOURNAL_SM __EYEC8('c','u','f','s','_','j','l','h') + uint32_t version; + uint32_t status; + uint64_t flags; + uint64_t reserved; + uint64_t num_entries; + cflsh_usfs_journal_entry_t entry[1]; + + +} cflsh_usfs_journal_hdr_t; + +typedef enum { + CFLSH_USFS_JOURNL_EV_INVAL = 0, /* Invalid Journal event type */ + CFLSH_USFS_JOURNAL_EV_ADD = 1, /* Add/write event to journal */ + CFLSH_USFS_JOURNAL_EV_REMOVE = 2, /* Remove event from journal */ + +} cflash_journal_event_type_t; + + +/************************************************************************/ +/* cflsh_usfs_journal_ev - The data structure for a journaled events */ +/* */ +/* . */ +/************************************************************************/ + + +typedef struct cflsh_usfs_journal_ev_s { + struct cflsh_usfs_journal_ev_s *prev; /* Previous chunk in list */ + struct cflsh_usfs_journal_ev_s *next; /* Next chunk in list */ + + int flags; + + cflsh_usfs_journal_entry_t journal_entry; + uint64_t offset; /* In case of a remove event */ + /* this contains the offset of */ + /* the event to be removed */ + eye_catch4b_t eyec; /* Eye catcher */ + +} cflsh_usfs_journal_ev_t; + +#define CFLSH_USFS_DIR_MAX_NAME 256 +/************************************************************************/ +/* Directory data structures for this filesystem. */ +/************************************************************************/ + +typedef struct cflsh_usfs_directory_entry_s { + uint32_t version; + uint32_t flags; +#define CFLSH_USFS_DIR_VAL 0x0001 /* Valid directory entry */ + uint64_t inode_number; + uint64_t reserved; + cflsh_usfs_inode_file_type_t type; + char filename[CFLSH_USFS_DIR_MAX_NAME]; + +} cflsh_usfs_directory_entry_t; + + +typedef struct cflsh_usfs_directory_s { + uint64_t start_marker; +#define CLFSH_USFS_DIRECTORY_SM __EYEC8('c','u','f','s','_','d','i','r') + uint64_t flags; + uint32_t version; + uint32_t status; + uint64_t num_entries; + cflsh_usfs_directory_entry_t entry[1]; + + +} cflsh_usfs_directory_t; + + + +/************************************************************************/ +/* Disk lba list entry */ +/************************************************************************/ +typedef struct cflsh_usfs_disk_lba_e_s { + uint64_t start_lba; /* Starting disk LBA/sector for */ + /* for this request. */ + uint64_t num_lbas; /* Number of disk LBAs/sectors */ +} cflsh_usfs_disk_lba_e_t; + + +/************************************************************************/ +/* Filesystem lba list entry */ +/************************************************************************/ +typedef struct cflsh_usfs_fs_lba_e_s { + uint64_t start_lba; /* Starting fs LBA/sector for */ + /* for this request. */ + uint64_t num_lbas; /* Number of fs LBAs/sectors */ +} cflsh_usfs_fs_lba_e_t; + + +/************************************************************************/ +/* Miscellaneous */ +/************************************************************************/ + +/* flags for cusfs_add_fs_blocks_to_inode_ptrs_cleanup */ + +#define CUSFS_UPDATE_SINGLE_INODE 0x2 +#define CUSFS_UPDATE_DOUBLE_INODE 0x4 +#define CUSFS_UPDATE_TRIPLE_INODE 0x8 + +#endif /* _H_CFLASH_USFS_DISK */ diff --git a/src/usfs/cflsh_usfs_inode.c b/src/usfs/cflsh_usfs_inode.c new file mode 100644 index 00000000..5f045a3a --- /dev/null +++ b/src/usfs/cflsh_usfs_inode.c @@ -0,0 +1,4776 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* bos720 src/bos/usr/ccs/lib/libcflsh_block/cflsh_block.c 1.8 */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#ifdef _AIX +static char sccsid[] = "%Z%%M% %I% %W% %G% %U%"; +#endif + + + +/* + * COMPONENT_NAME: (sysxcflashusfs) CAPI Flash user space filesystem library + * + * FUNCTIONS: + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +#define CFLSH_USFS_FILENUM 0x0400 + +#include "cflsh_usfs_internal.h" +#include "cflsh_usfs_protos.h" + + +/* + * NAME: cusfs_get_inode_stats + * + * FUNCTION: Get inode table statistics + * + * + * INPUTS: + * + * + * RETURNS: 0 Success, Otherwise failure + * + * + */ + +int cusfs_get_inode_stats(cflsh_usfs_t *cufs,uint64_t *total_inodes, uint64_t *available_inodes, + int flags) +{ + cflash_offset_t offset; + uint64_t num_blocks_read; + uint64_t rc = 0; + cflsh_usfs_inode_table_stats_t *inode_stat; + + + if (cufs == NULL) { + + + errno = EINVAL; + return -1; + } + + if (total_inodes == NULL) { + + errno = EINVAL; + return -1; + } + + if (available_inodes == NULL) { + + errno = EINVAL; + return -1; + } + + + CUSFS_LOCK(cufs->inode_table_lock); + + + /* + * Get inode table statistics + */ + + offset = CFLSH_USFS_INODE_TBL_STATS_OFFSET; + + num_blocks_read = 1; + + rc = cblk_read(cufs->chunk_id,cufs->inode_tbl_buf,offset,num_blocks_read,0); + + if (rc < num_blocks_read) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + inode_stat = (cflsh_usfs_inode_table_stats_t *) cufs->inode_tbl_buf; + + + if ((inode_stat->start_marker != CLFSH_USFS_IT_SM) || + (inode_stat->end_marker != CLFSH_USFS_IT_EM)) { + + CUSFS_TRACE_LOG_FILE(1,"corrupted inode table stats at lba = 0x%llx sm = 0x%llx em = 0x%llx", + offset,inode_stat->start_marker,inode_stat->end_marker); + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + + } + + *total_inodes = inode_stat->total_inodes; + *available_inodes = inode_stat->available_inodes; + + + + CUSFS_UNLOCK(cufs->inode_table_lock); + return 0; +} + + +/* + * NAME: cusfs_get_inode + * + * FUNCTION: Get free inode and mark as allocated. + * + * + * INPUTS: + * + * + * NOTES: This code assumes the following about the blocks + * array: + * + * 0-11: Are direct inode pointers + * 12: Is a single indirect inode pointer + * 13: Is a double indirect inode pointer + * 14: Is a triple indirect inode pointer. + * + * + * RETURNS: NULL failure, Otherwise success + * + * + */ + +cflsh_usfs_inode_t *cusfs_get_inode(cflsh_usfs_t *cufs, uint64_t parent_directory_inode_num, + cflsh_usfs_inode_file_type_t inode_type, + mode_t mode_flags, uid_t uid, gid_t gid, + uint64_t blocks[], int num_blocks, uint64_t file_size, + char *sym_link_name, + int flags) +{ + cflsh_usfs_inode_t *inode = NULL; + int buf_size; + char *buf; + int i; + uint64_t remaining_size; + cflash_offset_t offset; + uint64_t num_blocks_read; + uint64_t rc = 0; + cflsh_usfs_inode_table_stats_t *inode_stat; + uint64_t current_byte_offset; + cflsh_usfs_inode_t *inode_ptr; + uint64_t buf_offset; + + + if (cufs == NULL) { + + + errno = EINVAL; + return inode; + } + + if (blocks == NULL) { + + errno = EINVAL; + return inode; + } + + + CUSFS_LOCK(cufs->inode_table_lock); + + if (cflsh_usfs_master_lock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + + + buf_size = cufs->inode_tbl_buf_size; + + remaining_size = cufs->superblock.inode_table_size * cufs->disk_block_size; + + offset = cufs->superblock.inode_table_start_lba; + + if (buf_size % cufs->disk_block_size) { + + CUSFS_TRACE_LOG_FILE(1,"buf_size = 0x%x, is not a multiple of device block size", + buf_size,cufs->disk_block_size); + } + + num_blocks_read = MIN(buf_size/cufs->disk_block_size,cufs->max_xfer_size); + + + current_byte_offset = 0; + + while (remaining_size > 0) { + + + bzero(cufs->inode_tbl_buf,cufs->inode_tbl_buf_size); + + if (remaining_size < buf_size) { + + + num_blocks_read = remaining_size/cufs->disk_block_size; + + + if (remaining_size % cufs->disk_block_size) { + + num_blocks_read++; + } + + buf_size = remaining_size; + } + + + + rc = cblk_read(cufs->chunk_id,cufs->inode_tbl_buf,offset,num_blocks_read,0); + + if (rc < num_blocks_read) { + + CUSFS_TRACE_LOG_FILE(1,"reaad of inode table at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return inode; + } + + inode_ptr = cufs->inode_tbl_buf; + + while (inode_ptr < (cflsh_usfs_inode_t *)(cufs->inode_tbl_buf + buf_size)) { + + if (!(inode_ptr->flags & CFLSH_USFS_INODE_INUSE)) { + + + if ((inode_ptr->start_marker != CLFSH_USFS_INODE_SM) || + (inode_ptr->end_marker != CLFSH_USFS_INODE_EM)) { + + CUSFS_TRACE_LOG_FILE(1,"Invalid inode markers inode_ptr = %p offset = 0x%llx", + inode_ptr, offset); + continue; + } + + + inode = (cflsh_usfs_inode_t *) malloc(sizeof(cflsh_usfs_inode_t)); + + if (inode == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"malloc of inode failed with errno = %d", + errno); + + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return inode; + + } + + + inode_ptr->uid = uid; + inode_ptr->gid = gid; + + + if (inode_type == CFLSH_USFS_INODE_FILE_DIRECTORY) { + + mode_flags |= S_IFDIR; + + } else if (inode_type == CFLSH_USFS_INODE_FILE_REGULAR) { + + mode_flags |= S_IFREG; + + } else if (inode_type == CFLSH_USFS_INODE_FILE_SYMLINK) { + + mode_flags |= S_IFLNK; + + } + + inode_ptr->mode_flags = mode_flags; + inode_ptr->file_size = file_size; + inode_ptr->type = inode_type; + inode_ptr->num_links = 1; +#if !defined(__64BIT__) && defined(_AIX) + inode_ptr->atime = cflsh_usfs_time64(NULL); +#else + inode_ptr->atime = time(NULL); +#endif /* not 32-bit AIX */ + inode_ptr->ctime = inode_ptr->atime; + inode_ptr->mtime = inode_ptr->atime; + + + inode_ptr->directory_inode = parent_directory_inode_num; + + if (inode_type != CFLSH_USFS_INODE_FILE_SYMLINK) { + + num_blocks = MIN(num_blocks,CFLSH_USFS_NUM_TOTAL_INODE_PTRS); + + for (i=0; i< num_blocks; i++) { + + inode_ptr->ptrs[i] = blocks[i]; + } + + } else { + + if (sym_link_name == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"symlink is null"); + + free(inode); + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return NULL; + } + + + strcpy(inode_ptr->sym_link_path,sym_link_name); + } + + inode_ptr->flags = CFLSH_USFS_INODE_INUSE; + + *inode = *inode_ptr; + + + /* + * Write back this inode, but only write this specific LBA + * data + */ + + if (cusfs_get_inode_block_no_from_inode_index(cufs,inode->index,(uint64_t *)&offset)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get lba for inode->index = 0x%llx", + inode->index); + + free(inode); + CUSFS_UNLOCK(cufs->inode_table_lock); + + return NULL;; + + } + + /* + * Get pointer to beginning of this disk blocks data buffer + * with the updated inode + */ + + buf_offset = (uint64_t)inode_ptr; + + buf_offset &= ~((uint64_t)cufs->disk_block_size - 1); + + buf = (char*)buf_offset; + + /* + * Write updated disk block containing this inode back to disk + */ + + rc = cblk_write(cufs->chunk_id,buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"write of inode table at lba = 0x%llx failed with errno = %d", + offset,errno); + + free(inode); + free(inode); + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + CUSFS_UNLOCK(cufs->inode_table_lock); + + return NULL; + } + + + *inode = *inode_ptr; + + + break; + } + inode_ptr++; + } + + if (inode) { + + break; + } + + remaining_size -= buf_size; + + current_byte_offset += buf_size; + + + offset += num_blocks_read; + + + } /* while */ + + + /* + * ?? Add validation check that this lba is not the meta data + * at the beginning of the disk. + */ + + CUSFS_TRACE_LOG_FILE(5,"inode found = 0x%p on device = %s", + inode,cufs->device_name); + + + /* + * Update inode table statistics + */ + + offset = CFLSH_USFS_INODE_TBL_STATS_OFFSET; + + num_blocks_read = 1; + + rc = cblk_read(cufs->chunk_id,cufs->inode_tbl_buf,offset,num_blocks_read,0); + + if (rc < num_blocks_read) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + CUSFS_UNLOCK(cufs->inode_table_lock); + return NULL; + } + + inode_stat = (cflsh_usfs_inode_table_stats_t *) cufs->inode_tbl_buf; + + + if ((inode_stat->start_marker != CLFSH_USFS_IT_SM) || + (inode_stat->end_marker != CLFSH_USFS_IT_EM)) { + + CUSFS_TRACE_LOG_FILE(1,"corrupted inode table stats at lba = 0x%llx sm = 0x%llx em = 0x%llx", + offset,inode_stat->start_marker,inode_stat->end_marker); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return NULL; + + } + + inode_stat->available_inodes--; + + rc = cblk_write(cufs->chunk_id,cufs->inode_tbl_buf,offset,num_blocks_read,0); + + if (rc < num_blocks_read) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return NULL; + } + + + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + + CUSFS_UNLOCK(cufs->inode_table_lock); + return inode; +} + + +/* + * NAME: cusfs_release_inode + * + * FUNCTION: Release an inode for use by others + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise error. + * + * + */ + +int cusfs_release_inode(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode,int flags) +{ + int rc = 0; + cflash_offset_t offset; + cflsh_usfs_inode_t *inode_ptr; + cflsh_usfs_inode_table_stats_t *inode_stat; + int inode_index_in_block; + int num_inodes_per_block; + uint64_t index; + + + if (cufs == NULL) { + + + errno = EINVAL; + return -1; + } + + if (inode == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Node valid inode passed for disk %s", + cufs->device_name); + errno = EINVAL; + return -1; + } + + if (cusfs_get_inode_block_no_from_inode_index(cufs,inode->index,(uint64_t *)&offset)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get lba for inode->index = 0x%llx", + inode->index); + return -1; + + } + + CUSFS_LOCK(cufs->inode_table_lock); + + if (cflsh_usfs_master_lock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + + rc = cblk_read(cufs->chunk_id,cufs->inode_tbl_buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of inode table at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + + /* + * Get offset in sector for this inode + */ + + num_inodes_per_block = cufs->disk_block_size / sizeof(cflsh_usfs_inode_t); + + inode_index_in_block = inode->index % num_inodes_per_block; + + inode_ptr = cufs->inode_tbl_buf; + + if (inode_ptr[inode_index_in_block].index != inode->index) { + + CUSFS_TRACE_LOG_FILE(1,"inode on disk different from ours inode_ptr->index = 0x%llx, inode->index = 0x%llx", + inode_ptr->index,inode->index); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + + } + + if (inode_ptr[inode_index_in_block].type != inode->type) { + + CUSFS_TRACE_LOG_FILE(1,"inode types do not match inode_ptr->type = %d, inode->type = %d, inode->index = 0x%llx", + inode_ptr->type,inode->type,inode->index); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + if (inode_ptr[inode_index_in_block].num_links > 1) { + + inode_ptr[inode_index_in_block].num_links--; + + + } else { + + index = inode_ptr->index; + + bzero(&inode_ptr[inode_index_in_block],sizeof(*inode_ptr)); + + inode_ptr[inode_index_in_block].start_marker = CLFSH_USFS_INODE_SM; + + inode_ptr[inode_index_in_block].index = index; + + inode_ptr[inode_index_in_block].end_marker = CLFSH_USFS_INODE_EM; + + } + + rc = cblk_write(cufs->chunk_id,cufs->inode_tbl_buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"write of inode table at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + + free(inode); + + CUSFS_TRACE_LOG_FILE(5,"inode freed = 0x%p on device = %s", + inode,cufs->device_name); + + + + /* + * Update inode table statistics + */ + + offset = CFLSH_USFS_INODE_TBL_STATS_OFFSET; + + rc = cblk_read(cufs->chunk_id,cufs->inode_tbl_buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + inode_stat = (cflsh_usfs_inode_table_stats_t *) cufs->inode_tbl_buf; + + + if ((inode_stat->start_marker != CLFSH_USFS_IT_SM) || + (inode_stat->end_marker != CLFSH_USFS_IT_EM)) { + + CUSFS_TRACE_LOG_FILE(1,"corrupted inode table stats at lba = 0x%llx sm = 0x%llx em = 0x%llx", + offset,inode_stat->start_marker,inode_stat->end_marker); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + + } + + inode_stat->available_inodes++; + + rc = cblk_write(cufs->chunk_id,cufs->inode_tbl_buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return rc; +} + + +/* + * NAME: cusfs_release_inode_and_data_blocks + * + * FUNCTION: Release an inode for use by others. + * Also free all associated data blocks for this inode. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise error. + * + * + */ + +int cusfs_release_inode_and_data_blocks(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode,int flags) +{ + int rc = 0; + cflash_offset_t offset; + cflsh_usfs_inode_table_stats_t *inode_stat; + cflsh_usfs_inode_t *inode_ptr; + int inode_index_in_block; + int num_inodes_per_block; + uint64_t index; + int i; + + + if (cufs == NULL) { + + + errno = EINVAL; + return -1; + } + + if (inode == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Node valid inode passed for disk %s", + cufs->device_name); + errno = EINVAL; + return -1; + } + + if (cusfs_get_inode_block_no_from_inode_index(cufs,inode->index,(uint64_t *)&offset)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get lba for inode->index = 0x%llx", + inode->index); + return -1; + + } + + CUSFS_LOCK(cufs->inode_table_lock); + + if (cflsh_usfs_master_lock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + + + rc = cblk_read(cufs->chunk_id,cufs->inode_tbl_buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of inode table at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + + /* + * Get offset in sector for this inode + */ + + num_inodes_per_block = cufs->disk_block_size / sizeof(cflsh_usfs_inode_t); + + inode_index_in_block = inode->index % num_inodes_per_block; + + inode_ptr = cufs->inode_tbl_buf; + + if (inode_ptr[inode_index_in_block].index != inode->index) { + + CUSFS_TRACE_LOG_FILE(1,"inode on disk different from ours inode_ptr->index = 0x%llx, inode->index = 0x%llx", + inode_ptr->index,inode->index); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + + } + + if (inode_ptr[inode_index_in_block].type != inode->type) { + + CUSFS_TRACE_LOG_FILE(1,"inode types do not match inode_ptr->type = %d, inode->type = %d, inode->index = 0x%llx", + inode_ptr->type,inode->type,inode->index); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + if (inode_ptr[inode_index_in_block].num_links > 1) { + + inode_ptr[inode_index_in_block].num_links--; + + } else { + + index = inode_ptr[inode_index_in_block].index; + + if (inode_ptr[inode_index_in_block].type != CFLSH_USFS_INODE_FILE_SYMLINK) { + + for (i=0; i< CFLSH_USFS_NUM_DIRECT_INODE_PTRS; i++) { + + if (inode_ptr[inode_index_in_block].ptrs[i]) { + + cusfs_release_data_blocks(cufs,inode_ptr[inode_index_in_block].ptrs[i],1,0); + } + } + + if (inode_ptr[inode_index_in_block].ptrs[CFLSH_USFS_INDEX_SINGLE_INDIRECT]) { + + cusfs_remove_indirect_inode_ptr(cufs,inode_ptr[inode_index_in_block].ptrs[CFLSH_USFS_INDEX_SINGLE_INDIRECT],0); + } + if (inode_ptr[inode_index_in_block].ptrs[CFLSH_USFS_INDEX_DOUBLE_INDIRECT]) { + + cusfs_remove_indirect_inode_ptr(cufs,inode_ptr[inode_index_in_block].ptrs[CFLSH_USFS_INDEX_DOUBLE_INDIRECT], + CFLSH_USFS_INODE_FLG_INDIRECT_ENTRIES); + } + if (inode_ptr[inode_index_in_block].ptrs[CFLSH_USFS_INDEX_TRIPLE_INDIRECT]) { + + cusfs_remove_indirect_inode_ptr(cufs,inode_ptr[inode_index_in_block].ptrs[CFLSH_USFS_INDEX_TRIPLE_INDIRECT], + CFLSH_USFS_INODE_FLG_D_INDIRECT_ENTRIES); + } + + + } + + bzero(&inode_ptr[inode_index_in_block],sizeof(*inode_ptr)); + + inode_ptr[inode_index_in_block].start_marker = CLFSH_USFS_INODE_SM; + + inode_ptr[inode_index_in_block].index = index; + + inode_ptr[inode_index_in_block].end_marker = CLFSH_USFS_INODE_EM; + + } + + rc = cblk_write(cufs->chunk_id,cufs->inode_tbl_buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"write of inode table at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } else { + + rc = 0; + } + + + free(inode); + + CUSFS_TRACE_LOG_FILE(5,"inode freed = 0x%p on device = %s", + inode,cufs->device_name); + + + + /* + * Update inode table statistics + */ + + offset = CFLSH_USFS_INODE_TBL_STATS_OFFSET; + + rc = cblk_read(cufs->chunk_id,cufs->inode_tbl_buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + inode_stat = (cflsh_usfs_inode_table_stats_t *) cufs->inode_tbl_buf; + + + if ((inode_stat->start_marker != CLFSH_USFS_IT_SM) || + (inode_stat->end_marker != CLFSH_USFS_IT_EM)) { + + CUSFS_TRACE_LOG_FILE(1,"corrupted inode table stats at lba = 0x%llx sm = 0x%llx em = 0x%llx", + offset,inode_stat->start_marker,inode_stat->end_marker); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + + } + + inode_stat->available_inodes--; + + rc = cblk_write(cufs->chunk_id,cufs->inode_tbl_buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"write of free block table stats at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + + CUSFS_UNLOCK(cufs->inode_table_lock); + return rc; +} + +/* + * NAME: cusfs_get_inode_from_index + * + * FUNCTION: Get the inode structure from disk for + * the specified inode index (number). + * + * + * INPUTS: + * NONE + * + * RETURNS: NULL failure, otherwise success + * + * + */ + +cflsh_usfs_inode_t *cusfs_get_inode_from_index(cflsh_usfs_t *cufs, uint64_t inode_num,int flags) +{ + int rc = 0; + cflash_offset_t offset; + cflsh_usfs_inode_t *inode_ptr; + cflsh_usfs_inode_t *inode = NULL; + int inode_index_in_block; + int num_inodes_per_block; + + + + if (cufs == NULL) { + + + errno = EINVAL; + return NULL; + } + + + if (cusfs_get_inode_block_no_from_inode_index(cufs,inode_num,(uint64_t *)&offset)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get lba for inode->index = 0x%llx", + inode_num); + return NULL; + + } + + CUSFS_LOCK(cufs->inode_table_lock); + + + + rc = cblk_read(cufs->chunk_id,cufs->inode_tbl_buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of inode table at lba = 0x%llx failed with errno = %d", + offset,errno); + CUSFS_UNLOCK(cufs->inode_table_lock); + return NULL; + } + + + /* + * Get offset in sector for this inode + */ + + num_inodes_per_block = cufs->disk_block_size / sizeof(cflsh_usfs_inode_t); + + inode_index_in_block = inode_num % num_inodes_per_block; + + inode_ptr = cufs->inode_tbl_buf; + + if (inode_ptr[inode_index_in_block].index != inode_num) { + + CUSFS_TRACE_LOG_FILE(1,"inode index 0x%llx, does not match index 0x%llx requested", + inode[inode_index_in_block].index,inode_num); + CUSFS_UNLOCK(cufs->inode_table_lock); + return NULL; + } + + + inode = (cflsh_usfs_inode_t *) malloc(sizeof(cflsh_usfs_inode_t)); + + if (inode == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"malloc of inode failed with errno = %d", + errno); + + CUSFS_UNLOCK(cufs->inode_table_lock); + return inode; + } + + *inode = inode_ptr[inode_index_in_block]; + + + CUSFS_TRACE_LOG_FILE(5,"inode = 0x%p found on device = %s", + inode,cufs->device_name); + + + + CUSFS_UNLOCK(cufs->inode_table_lock); + return inode; +} + +/* + * NAME: cusfs_update_inode + * + * FUNCTION: Update a inode + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise error. + * + * + */ + +int cusfs_update_inode(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode,int flags) +{ + int rc = 0; + cflash_offset_t offset; + cflsh_usfs_inode_t *inode_ptr; + int inode_index_in_block; + int num_inodes_per_block; + + + + if (cufs == NULL) { + + + errno = EINVAL; + return -1; + } + + if (inode == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Node valid inode passed for disk %s", + cufs->device_name); + errno = EINVAL; + return -1; + } + + if (cusfs_get_inode_block_no_from_inode_index(cufs,inode->index,(uint64_t *)&offset)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get lba for inode->index = 0x%llx", + inode->index); + return -1; + + } + + CUSFS_LOCK(cufs->inode_table_lock); + + + if (cflsh_usfs_master_lock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + + rc = cblk_read(cufs->chunk_id,cufs->inode_tbl_buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"read of inode table at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + + /* + * Get offset in sector for this inode + */ + + num_inodes_per_block = cufs->disk_block_size / sizeof(cflsh_usfs_inode_t); + + inode_index_in_block = inode->index % num_inodes_per_block; + + inode_ptr = cufs->inode_tbl_buf; + + if (inode_ptr[inode_index_in_block].index != inode->index) { + + CUSFS_TRACE_LOG_FILE(1,"inode on disk different from ours inode_ptr->index = 0x%llx, inode->index = 0x%llx", + inode_ptr->index,inode->index); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + + } + + + if (inode_ptr[inode_index_in_block].type != inode->type) { + + CUSFS_TRACE_LOG_FILE(1,"inode types do not match inode_ptr->type = %d, inode->type = %d, inode->index = 0x%llx", + inode_ptr->type,inode->type,inode->index); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + /* + * Update inode change time + */ + +#if !defined(__64BIT__) && defined(_AIX) + inode->ctime = cflsh_usfs_time64(NULL); +#else + inode->ctime = time(NULL); +#endif /* not 32-bit AIX */ + + + inode_ptr[inode_index_in_block] = *inode; + + rc = cblk_write(cufs->chunk_id,cufs->inode_tbl_buf,offset,1,0); + + if (rc < 1) { + + CUSFS_TRACE_LOG_FILE(1,"write of inode table at lba = 0x%llx failed with errno = %d", + offset,errno); + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + + CUSFS_UNLOCK(cufs->inode_table_lock); + return -1; + } + + + + + CUSFS_TRACE_LOG_FILE(5,"updated inode = 0x%p on device = %s", + inode,cufs->device_name); + + + + if (cflsh_usfs_master_unlock(cufs->master_handle,CFLSH_USFS_MASTER_INODE_TABLE_LOCK,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unlock master lock for disk %s with errno = %d", + cufs->device_name,errno); + } + CUSFS_UNLOCK(cufs->inode_table_lock); + return 0; +} + + +/* + * NAME: cusfs_get_indirect_inode_ptr + * + * FUNCTION: Get a data block to be set up for indirect + * inode pointers. + * + * + * INPUTS: + * + * + * + * + * RETURNS: 0 failure, otherwise success. + * + * + */ + +uint64_t cusfs_get_indirect_inode_ptr(cflsh_usfs_t *cufs, + uint64_t blocks[], int *num_blocks, + int flags) +{ + int rc = 0; + int i = 0; + uint64_t block_num; + uint64_t disk_block_num; + int requested_num_blocks = 1; + uint64_t *disk_blocks; + char *buf; + int buf_size; + int num_disk_blocks; + + + if (cufs == NULL) { + + + errno = EINVAL; + return 0; + } + + if (blocks == NULL) { + + + errno = EINVAL; + return 0; + } + + if (num_blocks == NULL) { + + + errno = EINVAL; + return 0; + } + + buf_size = cufs->fs_block_size; + + buf = malloc(buf_size); + + if (buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"malloc of buf for indirect inode pointer failed with errno = %d for disk %s", + errno, cufs->device_name); + + return 0; + } + + + bzero(buf,buf_size); + + block_num = cusfs_get_data_blocks(cufs,&requested_num_blocks,0); + + if (num_blocks == 0) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block on disk %s, with errno %d", + cufs->device_name, errno); + return 0; + } + + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,block_num,&disk_block_num)) { + + cusfs_release_data_blocks(cufs,block_num,requested_num_blocks,0); + + free(buf); + return 0; + } + + disk_blocks = (uint64_t *)buf; + + *num_blocks = MIN(*num_blocks,cufs->fs_block_size/sizeof(uint64_t)); + + for (i = 0; i < *num_blocks; i++) { + + disk_blocks[i] = blocks[i]; + } + + //?? locking?? + + + + num_disk_blocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_write(cufs->chunk_id,buf,disk_block_num,num_disk_blocks,0); + + if (rc < num_disk_blocks) { + + CUSFS_TRACE_LOG_FILE(1,"write of of inode indirect pointer %d at lba = 0x%llx failed with errno = %d", + i,disk_block_num,errno); + cusfs_release_data_blocks(cufs,block_num,requested_num_blocks,0); + + block_num = 0; + } + + + free(buf); + + + return block_num; +} + + +/* + * NAME: cusfs_update_indirect_inode_ptr + * + * FUNCTION: Update a data block that is already set up for indirect + * inode pointers. + * + * + * INPUTS: + * + * + * + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_update_indirect_inode_ptr(cflsh_usfs_t *cufs, + uint64_t inode_ptr_block_no, + uint64_t blocks[], int *num_blocks, + int flags) +{ + int rc = 0; + int i = 0; + uint64_t block_num = inode_ptr_block_no; + uint64_t disk_block_num; + uint64_t *disk_blocks; + char *buf; + int buf_size; + int num_disk_blocks; + + + if (cufs == NULL) { + + + errno = EINVAL; + return 0; + } + + if (blocks == NULL) { + + + errno = EINVAL; + return 0; + } + + if (num_blocks == NULL) { + + + errno = EINVAL; + return 0; + } + + buf_size = cufs->fs_block_size; + + buf = malloc(buf_size); + + if (buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"malloc of buf for indirect inode pointer failed with errno = %d for disk %s", + errno, cufs->device_name); + + return 0; + } + + bzero(buf,buf_size); + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,block_num,&disk_block_num)) { + + free(buf); + return 0; + } + + + + + disk_blocks = (uint64_t *) buf; + + *num_blocks = MIN(*num_blocks,cufs->fs_block_size/sizeof(uint64_t)); + + for (i = 0; i < *num_blocks; i++) { + + disk_blocks[i] = blocks[i]; + } + + //?? locking?? + + + + num_disk_blocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_write(cufs->chunk_id,buf,disk_block_num,num_disk_blocks,0); + + if (rc < num_disk_blocks) { + + CUSFS_TRACE_LOG_FILE(1,"write of of inode indierct pointer %d at lba = 0x%llx failed with errno = %d", + i,disk_block_num,errno); + + block_num = 0; + } + + + free(buf); + + + return rc; +} + + +/* + * NAME: cusfs_add_blocks_to_indirect_inode_ptr + * + * FUNCTION: Update data block that is already set up for indirect + * inode pointers to add more data block pointers. + * + * + * INPUTS: + * + * + * + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_add_blocks_to_indirect_inode_ptr(cflsh_usfs_t *cufs, + uint64_t inode_ptr_block_no, + uint64_t blocks[], int *num_blocks, + int flags) +{ + int rc = 0; + int i,j; + uint64_t block_num = inode_ptr_block_no; + uint64_t disk_block_num; + uint64_t *disk_blocks; + char *buf; + int buf_size; + int num_disk_blocks; + int num_inode_ptrs; + + + if (cufs == NULL) { + + + errno = EINVAL; + return 0; + } + + if (blocks == NULL) { + + + errno = EINVAL; + return 0; + } + + if (num_blocks == NULL) { + + + errno = EINVAL; + return 0; + } + + buf_size = cufs->fs_block_size; + + buf = malloc(buf_size); + + if (buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"malloc of buf for indirect inode pointer failed with errno = %d for disk %s", + errno, cufs->device_name); + + return 0; + } + + bzero(buf,buf_size); + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,block_num,&disk_block_num)) { + + free(buf); + return 0; + } + + + + num_disk_blocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_read(cufs->chunk_id,buf,disk_block_num,num_disk_blocks,0); + + if (rc < num_disk_blocks) { + + CUSFS_TRACE_LOG_FILE(1,"read of inode indierct pointer at lba = 0x%llx failed with errno = %d", + disk_block_num,errno); + block_num = 0; + } + + + + disk_blocks = (uint64_t *) buf; + + /* + * Find first unused data inode_ptrs + * in this data block. + */ + + num_inode_ptrs = cufs->fs_block_size/sizeof(uint64_t); + + for (i = 0; i < num_inode_ptrs;i++) { + + if (disk_blocks[i] == 0x00LL) { + + break; + } + } + + + *num_blocks = MIN(*num_blocks,(cufs->fs_block_size/sizeof(uint64_t) - i)); + + for (j = i; j < *num_blocks; j++) { + + disk_blocks[j] = blocks[j]; + } + + + + // ?? cusfs_release_data_blocks(cufs,block_num,requested_num_blocks,0); + + + + rc = cblk_write(cufs->chunk_id,buf,disk_block_num,num_disk_blocks,0); + + if (rc < num_disk_blocks) { + + CUSFS_TRACE_LOG_FILE(1,"write of inode indierct pointer %d at lba = 0x%llx failed with errno = %d", + i,disk_block_num,errno); + + block_num = 0; + } + + + free(buf); + + + return rc; +} + +/* + * NAME: cusfs_remove_indirect_inode_ptr + * + * FUNCTION: Remove a data block that is already set up for indirect + * inode pointers. This can also remove all the data blocks + * referenced from this indirect inode pointer. + * + * + * INPUTS: + * + * + * + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int cusfs_remove_indirect_inode_ptr(cflsh_usfs_t *cufs, + uint64_t inode_ptr_block_no, + int flags) +{ + int rc = 0; + int i = 0; + uint64_t block_num = inode_ptr_block_no; + uint64_t *disk_blocks; + uint64_t disk_block_num; + char *buf; + int buf_size; + int num_disk_blocks; + int num_inode_ptrs; + + + if (cufs == NULL) { + + + errno = EINVAL; + return 0; + } + + buf_size = cufs->fs_block_size; + + buf = malloc(buf_size); + + if (buf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"malloc of buf for indirect inode pointer failed with errno = %d for disk %s", + errno, cufs->device_name); + + return 0; + } + + bzero(buf,buf_size); + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,block_num,&disk_block_num)) { + + free(buf); + return 0; + } + + + num_disk_blocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_read(cufs->chunk_id,buf,disk_block_num,num_disk_blocks,0); + + if (rc < num_disk_blocks) { + + CUSFS_TRACE_LOG_FILE(1,"read of inode indierct pointer %d at lba = 0x%llx failed with errno = %d", + i,disk_block_num,errno); + + block_num = 0; + } + + + disk_blocks = (uint64_t *) buf; + + num_inode_ptrs = cufs->fs_block_size/sizeof(uint64_t); + + + for (i = 0; i < num_inode_ptrs;i++) { + + if (disk_blocks[i]) { + + if (flags & CFLSH_USFS_INODE_FLG_INDIRECT_ENTRIES) { + cusfs_remove_indirect_inode_ptr(cufs, disk_blocks[i],0); + } else if (flags & CFLSH_USFS_INODE_FLG_D_INDIRECT_ENTRIES) { + cusfs_remove_indirect_inode_ptr(cufs, disk_blocks[i], + CFLSH_USFS_INODE_FLG_INDIRECT_ENTRIES); + } else { + cusfs_release_data_blocks(cufs,disk_blocks[i],1,0); + } + } + } + + //?? locking?? + + + cusfs_release_data_blocks(cufs,inode_ptr_block_no,1,0); + + free(buf); + + + return 0; +} + +/* + * NAME: cusfs_get_inode_block_no_from_inode_index + * + * FUNCTION: Read the disk sector (in terms of cufs->disk_block_size) + * which contains the inode with the inode_index specfied. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, non-zero otherwise. + * + * + */ + +int cusfs_get_inode_block_no_from_inode_index(cflsh_usfs_t *cufs, uint64_t inode_index,uint64_t *lba) +{ + int rc = 0; + + + *lba = inode_index * sizeof(cflsh_usfs_inode_t)/cufs->disk_block_size + cufs->superblock.inode_table_start_lba; + + if ((*lba < cufs->superblock.inode_table_start_lba) || + (*lba > (cufs->superblock.inode_table_start_lba + cufs->superblock.inode_table_size))) { + + CUSFS_TRACE_LOG_FILE(1,"Invalid inode lba = 0x%llx inode_start_lba = 0x%llx, inode_end_lba = 0x%llx", + *lba,cufs->superblock.inode_table_start_lba, + (cufs->superblock.inode_table_start_lba + cufs->superblock.inode_table_size)); + + errno = ERANGE; + return -1; + } + return rc; +} + + +/* + * NAME: cusfs_get_inode_ptr_from_file_offset + * + * FUNCTION: Determine which inode pointer in the inode is + * associated with the byte offset in the file. + * + * + * INPUTS: + * ?? Remove this routine? + * + * RETURNS: -1 failure, otherwise success + * + * + */ + +int cusfs_get_inode_ptr_from_file_offset(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + uint64_t offset, int *indirect_inode_ptr, int flags) +{ + int inode_ptr_index = -1; + uint64_t start_single_indirect_inode_offset; + + if (cufs == NULL) { + + errno = EINVAL; + return inode_ptr_index; + + } + + if (file == NULL) { + + errno = EINVAL; + return inode_ptr_index; + } + + inode_ptr_index = offset/cufs->fs_block_size; + + + if (inode_ptr_index >= CFLSH_USFS_NUM_DIRECT_INODE_PTRS) { + + start_single_indirect_inode_offset = CFLSH_USFS_NUM_DIRECT_INODE_PTRS * cufs->fs_block_size; + + + *indirect_inode_ptr = (offset - start_single_indirect_inode_offset)/ cufs->fs_block_size; + + if (*indirect_inode_ptr > cufs->fs_block_size/(sizeof(uint64_t))) { + + + /* + * ?? TODO: This code does not handle double and triple indirection in inode ponters + */ + + return -1; + } + + inode_ptr_index = CFLSH_USFS_INDEX_SINGLE_INDIRECT; + + + + + } + + + + return inode_ptr_index; +} + +/* + * NAME: cusfs_rdwr_block_inode_pointers + * + * FUNCTION: Read/Write in/out fs block size block of inode pointers. + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ +int cusfs_rdwr_block_inode_pointers(cflsh_usfs_t *cufs, + cflsh_usfs_inode_t *inode, + void *buffer, + uint64_t fs_lba, int nblocks, int flags) +{ + int rc = 0; + uint64_t disk_lba; + + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + /* + * Convert fs_block_size sector to disk_block_size sector + */ + + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,fs_lba, + &disk_lba)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to get disk blocks for fs_lba 0x%ll for inode of type %d on disk %s", + fs_lba,inode->type, cufs->device_name); + + + return -1; + } + if (flags & CFLSH_USFS_RDWR_WRITE) { + rc = cblk_write(cufs->chunk_id,buffer,disk_lba,nblocks,0); + } else { + rc = cblk_read(cufs->chunk_id,buffer,disk_lba,nblocks,0); + } + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"%s at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + ((flags & CFLSH_USFS_RDWR_WRITE) ? "Write":" Read"), + disk_lba,inode->type, cufs->device_name); + + + return -1; + } + + return 0; + +} + + +/* + * NAME: cusfs_get_disk_lbas_from_inode_ptr_for_transfer + * + * FUNCTION: For the specified transfer, get a list of + * disk LBA ranges that are associated. + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_get_disk_lbas_from_inode_ptr_for_transfer(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t offset, size_t num_bytes, + cflsh_usfs_disk_lba_e_t **disk_lbas, uint64_t *num_disk_lbas, + int flags) +{ + int rc = 0; + int i,j; + uint64_t start_fs_lba; + int single_index =0; + int double_index = 0; + int triple_index = 0; + uint64_t start_inode_ptr; + uint64_t start_offset_in_inode_ptr; + uint64_t end_offset_in_inode_ptr; + uint64_t num_inode_ptrs; + uint64_t num_inode_ptrs_per_fs_block; + uint64_t *inode_single_block_list = NULL; + uint64_t *inode_double_block_list = NULL; + uint64_t *inode_triple_block_list = NULL; + uint64_t start_disk_lba; + cflsh_usfs_disk_lba_e_t *lba_list; + uint64_t num_lbas, num_lbas2; + int nblocks; + + + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + + nblocks = cufs->fs_block_size/cufs->disk_block_size; + + /* + * First determine out of the whole collection + * inode pointers (this is independent of whether + * these are direct or indirect inode pointers), + * which one is the starting + * inode pointer and which one is the ending + * inode pointer. In this initial work, + * we are only counting inode pointers that directly + * point to data blocks. Thus if a an indirect inode pointer + * is involved, this calculation is not directly looking + * indirect inode pointers, but instead looking at direct + * inode pointers that are referenced by the indirect + * inode pointers. + */ + + + start_inode_ptr = offset/cufs->fs_block_size; + + start_offset_in_inode_ptr = offset % cufs->fs_block_size; + + + num_inode_ptrs = num_bytes/cufs->fs_block_size; + + if (num_bytes % cufs->fs_block_size) { + num_inode_ptrs++; + } + + if ((offset % cufs->fs_block_size) && + (offset/cufs->fs_block_size != (offset+num_bytes)/cufs->fs_block_size)) { + + /* + * If the offset is not aligned on a filesystem block and + * the start and end of the transfer are not on the same + * filesystem block, then we need to span an additional + * filesystem block. + */ + num_inode_ptrs++; + } + + *num_disk_lbas = num_inode_ptrs; + + + end_offset_in_inode_ptr = (offset + num_bytes) % cufs->fs_block_size; + + + /* + * First create list of inode pointer in fs_block_size LBAs, + * then we'll convert the fs_block_size LBAs to disk_block_size + * LBAs in the middle inode pointers. And we'll adjust the + * the start and ending offset to the closest disk_block_size + * lba. + */ + + + lba_list = malloc(sizeof(cflsh_usfs_disk_lba_e_t) * num_inode_ptrs); + + if (lba_list == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Malloc of disk_lbas of for num_inode_ptrs = %d of on disk %s failed with errno %d ", + cufs->device_name,errno); + + + + return -1; + } + + CUSFS_TRACE_LOG_FILE(9,"type %d on disk %s, offset = 0x%llx, start_inode_ptr = 0x%llx, num_inode_ptrs = 0x%llx", + inode->type, cufs->device_name,offset,start_inode_ptr,num_inode_ptrs); + + /* + * This code assumes each inode pointer to points to exactly one fs_block_size + * block. + */ + + + for (i = start_inode_ptr,j=0; i < (start_inode_ptr + num_inode_ptrs); i++,j++) { + + + lba_list[j].num_lbas = cufs->fs_block_size/cufs->disk_block_size; + + + if (i < CFLSH_USFS_NUM_DIRECT_INODE_PTRS) { + + start_fs_lba = inode->ptrs[i]; + + CUSFS_TRACE_LOG_FILE(9,"start_fs_lba = 0x%llx, i = 0x%x ", + start_fs_lba,i,cufs->device_name); + + + } else { + + + if (inode_single_block_list == NULL) { + + inode_single_block_list = malloc(cufs->fs_block_size); + + if (inode_single_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of single_indirect of for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->device_name,errno); + + return -1; + } + + /* + * Read in single indirect inode ptrs + */ + + if (cusfs_rdwr_block_inode_pointers(cufs,inode,inode_single_block_list, + inode->ptrs[CFLSH_USFS_INDEX_SINGLE_INDIRECT], + nblocks,0)) { + + if (inode_single_block_list) { + free(inode_single_block_list); + } + if (inode_double_block_list) { + free(inode_double_block_list); + } + if (inode_triple_block_list) { + free(inode_triple_block_list); + } + free(lba_list); + return -1; + } + + + } + + + /* + * Since we flatten out the inode pointers to just + * the direct ones eliminating the indirect varieties + */ + + if ((i >= CFLSH_USFS_NUM_DIRECT_INODE_PTRS) && + (i < (num_inode_ptrs_per_fs_block + CFLSH_USFS_NUM_DIRECT_INODE_PTRS))) { + + /* + * These inode pointers are in the single inode pointer list + */ + + + start_fs_lba = inode_single_block_list[i - CFLSH_USFS_NUM_DIRECT_INODE_PTRS]; + + + CUSFS_TRACE_LOG_FILE(9,"start_fs_lba = 0x%llx, i = 0x%x on disk %s", + start_fs_lba,i,cufs->device_name); + + } else if ((i >= num_inode_ptrs_per_fs_block + CFLSH_USFS_NUM_DIRECT_INODE_PTRS) && + (i < (num_inode_ptrs_per_fs_block * num_inode_ptrs_per_fs_block + CFLSH_USFS_NUM_DIRECT_INODE_PTRS))) { + + + /* + * These inode pointers are in the double inode pointer list + */ + + if (inode_double_block_list == NULL) { + + double_index = 0; + single_index = 0; + + inode_double_block_list = malloc(cufs->fs_block_size); + + if (inode_double_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of single_indirect of for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->device_name,errno); + + return -1; + } + + /* + * Read in double indirect inode ptrs + */ + + + if (cusfs_rdwr_block_inode_pointers(cufs,inode,inode_double_block_list, + inode->ptrs[CFLSH_USFS_INDEX_DOUBLE_INDIRECT], + nblocks,0)) { + + if (inode_single_block_list) { + free(inode_single_block_list); + } + if (inode_double_block_list) { + free(inode_double_block_list); + } + if (inode_triple_block_list) { + free(inode_triple_block_list); + } + free(lba_list); + return -1; + } + + } + + if (single_index == 0) { + /* + * Read in another block of single indirect inode pointers + */ + + if (cusfs_rdwr_block_inode_pointers(cufs,inode,inode_single_block_list, + inode_double_block_list[double_index], + nblocks,0)) { + + if (inode_single_block_list) { + free(inode_single_block_list); + } + if (inode_double_block_list) { + free(inode_double_block_list); + } + if (inode_triple_block_list) { + free(inode_triple_block_list); + } + free(lba_list); + return -1; + } + + } + + start_fs_lba = inode_single_block_list[single_index]; + + + CUSFS_TRACE_LOG_FILE(9,"start_fs_lba = 0x%llx, i = 0x%x on disk %s", + start_fs_lba,i,cufs->device_name); + + if (single_index < num_inode_ptrs_per_fs_block) { + + single_index++; + } else { + + single_index = 0; + + double_index++; + } + + + } else if ((i >= (num_inode_ptrs_per_fs_block * num_inode_ptrs_per_fs_block + CFLSH_USFS_NUM_DIRECT_INODE_PTRS)) && + (i < (num_inode_ptrs_per_fs_block * num_inode_ptrs_per_fs_block * num_inode_ptrs_per_fs_block + CFLSH_USFS_NUM_DIRECT_INODE_PTRS))) { + + + /* + * These inode pointers are in the triple inode pointer list + */ + + + if (inode_triple_block_list == NULL) { + + triple_index = 0; + double_index = 0; + single_index = 0; + + inode_triple_block_list = malloc(cufs->fs_block_size); + + if (inode_triple_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of triple_indirect of for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->device_name,errno); + + return -1; + } + + /* + * Read in triple indirect inode ptrs + */ + + + if (cusfs_rdwr_block_inode_pointers(cufs,inode,inode_triple_block_list, + inode->ptrs[CFLSH_USFS_INDEX_TRIPLE_INDIRECT], + nblocks,0)) { + + if (inode_single_block_list) { + free(inode_single_block_list); + } + if (inode_double_block_list) { + free(inode_double_block_list); + } + if (inode_triple_block_list) { + free(inode_triple_block_list); + } + free(lba_list); + return -1; + } + + } + + + if (double_index == 0) { + /* + * Read in another block of single indirect inode pointers + */ + + if (cusfs_rdwr_block_inode_pointers(cufs,inode,inode_double_block_list, + inode_triple_block_list[triple_index], + nblocks,0)) { + + if (inode_single_block_list) { + free(inode_single_block_list); + } + if (inode_double_block_list) { + free(inode_double_block_list); + } + if (inode_triple_block_list) { + free(inode_triple_block_list); + } + free(lba_list); + return -1; + } + + } + + if (single_index == 0) { + /* + * Read in another block of single indirect inode pointers + */ + + if (cusfs_rdwr_block_inode_pointers(cufs,inode,inode_single_block_list, + inode_double_block_list[double_index], + nblocks,0)) { + + if (inode_single_block_list) { + free(inode_single_block_list); + } + if (inode_double_block_list) { + free(inode_double_block_list); + } + if (inode_triple_block_list) { + free(inode_triple_block_list); + } + free(lba_list); + return -1; + } + + } + + start_fs_lba = inode_single_block_list[single_index]; + + + CUSFS_TRACE_LOG_FILE(9,"start_fs_lba = 0x%llx, i = 0x%x on disk %s", + start_fs_lba,i,cufs->device_name); + + if (single_index < num_inode_ptrs_per_fs_block) { + + single_index++; + } else { + + single_index = 0; + + if (double_index < num_inode_ptrs_per_fs_block) { + + double_index++; + } else { + + double_index = 0; + + triple_index++; + } + } + + + } else { + + CUSFS_TRACE_LOG_FILE(1,"Invalid inode pointer %d in main inode of type %d on disk %s", + i,inode->type, cufs->device_name); + + + if (inode_single_block_list) { + free(inode_single_block_list); + } + if (inode_double_block_list) { + free(inode_double_block_list); + } + if (inode_triple_block_list) { + free(inode_triple_block_list); + } + free(lba_list); + return -1; + + } + } + + + if (start_fs_lba == 0) { + + + CUSFS_TRACE_LOG_FILE(1,"LBA of 0 found for index i = %d on device , ", + i,cufs->device_name); + + + if (inode_single_block_list) { + free(inode_single_block_list); + } + if (inode_double_block_list) { + free(inode_double_block_list); + } + if (inode_triple_block_list) { + free(inode_triple_block_list); + } + free(lba_list); + return -1; + } + + + + /* + * Convert block number from and filesystem block size + * block number, to a disk block size block number. + */ + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,start_fs_lba,&start_disk_lba)) { + + + CUSFS_TRACE_LOG_FILE(1,"type %d on disk %s failed to get disk blocks for 0x%llx, ", + inode->type, cufs->device_name,start_fs_lba); + + + if (inode_single_block_list) { + free(inode_single_block_list); + } + if (inode_double_block_list) { + free(inode_double_block_list); + } + if (inode_triple_block_list) { + free(inode_triple_block_list); + } + free(lba_list); + return -1; + } + + + + lba_list[j].start_lba = start_disk_lba; + + num_lbas = lba_list[j].num_lbas; + + if ((i == start_inode_ptr) && + (start_offset_in_inode_ptr)) { + + /* + * Adjust the transfer's start lba to the closest disk lba in this + * fs lba. + */ + start_disk_lba += start_offset_in_inode_ptr/cufs->disk_block_size; + + lba_list[j].num_lbas -= (start_disk_lba - lba_list[j].start_lba); + + lba_list[j].start_lba = start_disk_lba; + } + + if ((i == (num_inode_ptrs - 1)) && + (end_offset_in_inode_ptr)) { + + + /* + * Adjust the transfer's end lba to the closest disk lba in this + * fs lba + */ + + num_lbas2 = end_offset_in_inode_ptr/cufs->disk_block_size; + + if (end_offset_in_inode_ptr % cufs->disk_block_size) { + num_lbas2++; + } + + if (num_lbas2 < num_lbas) { + + lba_list[j].num_lbas -= (num_lbas - num_lbas2); + } + + } + + } + + *disk_lbas = lba_list; + + if (inode_single_block_list) { + free(inode_single_block_list); + } + if (inode_double_block_list) { + free(inode_double_block_list); + } + if (inode_triple_block_list) { + free(inode_triple_block_list); + } + return rc; +} + + +/* + * NAME: cusfs_get_all_disk_lbas_from_inode + * + * FUNCTION: For the specified inode, get a list of + * all disk LBA ranges that are associated with it. + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_get_all_disk_lbas_from_inode(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + cflsh_usfs_disk_lba_e_t **disk_lbas, uint64_t *num_disk_lbas, + int flags) +{ + int level; + int rc; + size_t num_bytes; + + CUSFS_TRACE_LOG_FILE(9,"inode->type %d on disk = %s", + inode->type, cufs->device_name); + + + if (cusfs_num_data_blocks_in_inode(cufs,inode,num_disk_lbas,&level,0)) { + + *num_disk_lbas = 0; + return -1; + } + + num_bytes = (*num_disk_lbas) * cufs->fs_block_size; + + CUSFS_TRACE_LOG_FILE(9,"num_disk_lbas = 0x%llx", + *num_disk_lbas); + + rc = cusfs_get_disk_lbas_from_inode_ptr_for_transfer(cufs,inode,0,num_bytes,disk_lbas,num_disk_lbas,0); + + CUSFS_TRACE_LOG_FILE(9,"rc = %d, errno = %d", + rc, errno); + return rc; +} + + +/* + * NAME: cusfs_get_inode_ptr_for_transfer + * + * FUNCTION: Determine which inode pointer in the inode are + * associated with this transfer + * + * + * INPUTS: ?? Remove thie routine? + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_get_inode_ptr_from_for_transfer(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + uint64_t offset, size_t num_bytes, + int *start_inode_ptr_index, + int *start_inode_ptr_byte_offset, + int *end_inode_ptr_index, + int *end_inode_ptr_byte_offset, + int *num_inode_ptr_indices, + int flags) +{ + int indirect_inode_ptr; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (file == NULL) { + + errno = EINVAL; + return -1; + } + + if ((start_inode_ptr_index == NULL) || + (end_inode_ptr_index == NULL)) { + + + errno = EINVAL; + return -1; + } + + + *start_inode_ptr_index = cusfs_get_inode_ptr_from_file_offset(cufs,file,offset,&indirect_inode_ptr,0); + + if (*start_inode_ptr_index < 0) { + + + CUSFS_TRACE_LOG_FILE(1,"For file = %s on disk %s failed to get starting inode_ptr for offset = 0x%llx, ", + file->filename,cufs->device_name,offset); + + + + return -1; + } + + /* + * ?? TODO: Currently this code discards the value of indirect_inode_ptr + */ + + *start_inode_ptr_byte_offset = offset % cufs->fs_block_size; + + + + *end_inode_ptr_index = cusfs_get_inode_ptr_from_file_offset(cufs,file,(offset + num_bytes),&indirect_inode_ptr,0); + + if (*end_inode_ptr_index < 0) { + + + CUSFS_TRACE_LOG_FILE(1,"For file = %s on disk %s failed to get starting inode_ptr for offset = 0x%llx, ", + file->filename,cufs->device_name,(offset + num_bytes)); + + + + return -1; + } + + /* + * ?? TODO: Currently this code discards the value of indirect_inode_ptr + */ + + *end_inode_ptr_byte_offset = offset % cufs->fs_block_size; + + + + *num_inode_ptr_indices = end_inode_ptr_index - start_inode_ptr_index + 1; + + + + return 0; +} + + +/* + * NAME: cusfs_find_disk_lbas_in_inode_ptr_for_transfer + * + * FUNCTION: For the specified inode pointer index, determine + * which disk lba's are associated with this transfer + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_find_disk_lbas_in_inode_ptr_for_transfer(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + int inode_ptr_index, + uint64_t *start_disk_lba, + uint64_t *num_disk_blocks, + int start_inode_ptr, + int end_inode_ptr, + int transfer_start_offset, + int transfer_end_offset, + int flags) +{ + uint64_t end_lba; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (file == NULL) { + + errno = EINVAL; + return -1; + } + + if ((start_disk_lba == NULL) || + (num_disk_blocks == NULL)) { + + + errno = EINVAL; + return -1; + } + + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,file->inode->ptrs[inode_ptr_index],start_disk_lba)) { + + + CUSFS_TRACE_LOG_FILE(1,"file = %s of type %d on disk %s failed to get disk blocks for 0x%llx, ", + file->filename,file->inode->type, cufs->device_name,file->inode->ptrs[inode_ptr_index]); + + + + return -1; + } + + + end_lba = *start_disk_lba + cufs->fs_block_size/cufs->disk_block_size; + + + if (inode_ptr_index == start_inode_ptr) { + + *start_disk_lba += transfer_start_offset/cufs->disk_block_size; + } + + if (inode_ptr_index == end_inode_ptr) { + + *num_disk_blocks = transfer_end_offset/cufs->disk_block_size; + if (transfer_end_offset % cufs->disk_block_size ) { + (*num_disk_blocks)++; + } + }else { + + *num_disk_blocks = end_lba - *start_disk_lba + 1; + + + } + + return 0; + +} + +/* + * NAME: cusfs_num_data_blocks_single_indirect_inode_ptr + * + * FUNCTION: Determine the number of data blocks currently + * referenced by this single indirect inode pointer + * + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_num_data_blocks_single_indirect_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t single_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags) +{ + + + int rc = 0; + int i; + uint64_t disk_lba; + int nblocks; + uint64_t num_inode_ptrs_per_fs_block; + uint64_t *inode_single_block_list = NULL; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + + if (single_indirect_inode_fs_lba == 0x0LL) { + + errno = EINVAL; + return -1; + } + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + + inode_single_block_list = malloc(cufs->fs_block_size); + + if (inode_single_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of single_indirect of size 0x%x, for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->fs_block_size,cufs->device_name,errno); + + return -1; + } + + + + /* + * Convert fs_block_size sector to disk_block_size sector + */ + + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,single_indirect_inode_fs_lba, + &disk_lba)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to get disk blocks for fs_lba 0x%ll for inode of type %d on disk %s", + single_indirect_inode_fs_lba,inode->type, cufs->device_name); + + free(inode_single_block_list); + return -1; + } + + nblocks = cufs->fs_block_size/cufs->disk_block_size; + + CUSFS_TRACE_LOG_FILE(9,"inode->type %d nblocks = 0x%llx, disk_lba = 0x%llx,on disk = %s", + inode->type, nblocks,disk_lba,cufs->device_name); + + rc = cblk_read(cufs->chunk_id,inode_single_block_list,disk_lba,nblocks,0); + + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Read at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + + free(inode_single_block_list); + return -1; + } + + CUSFS_TRACE_LOG_FILE(9,"inode->type %d rc = 0x%llx, num_inode_ptrs_per_fs_block = 0x%llx,on disk = %s", + inode->type, rc,num_inode_ptrs_per_fs_block,cufs->device_name); + + + for (i = 0; i < num_inode_ptrs_per_fs_block;i++) { + + if (inode_single_block_list[i] == 0x0LL) { + + *num_fs_blocks = i; + break; + } + + } + + + CUSFS_TRACE_LOG_FILE(9,"*num_fs_blocks = 0x%llx", + *num_fs_blocks); + + free(inode_single_block_list); + return 0; + + + +} + + + +/* + * NAME: cusfs_num_data_blocks_double_indirect_inode_ptr + * + * FUNCTION: Determine the number of data blocks currently + * referenced by this double indirect inode pointer + * + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_num_data_blocks_double_indirect_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t double_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags) +{ + + + int rc = 0; + int i; + uint64_t disk_lba; + int nblocks; + uint64_t num_inode_ptrs_per_fs_block; + uint64_t *inode_double_block_list = NULL; + uint64_t num_indirect_blocks; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + if (double_indirect_inode_fs_lba == 0x0LL) { + + errno = EINVAL; + return -1; + } + + *num_fs_blocks = 0; + + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + + inode_double_block_list = malloc(cufs->fs_block_size); + + if (inode_double_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of double_indirect of size 0x%x, for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->fs_block_size,cufs->device_name,errno); + + return -1; + } + + + + /* + * Convert fs_block_size sector to disk_block_size sector + */ + + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,double_indirect_inode_fs_lba, + &disk_lba)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to get disk blocks for fs_lba 0x%ll for inode of type %d on disk %s", + double_indirect_inode_fs_lba,inode->type, cufs->device_name); + + free(inode_double_block_list); + return -1; + } + + nblocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_read(cufs->chunk_id,inode_double_block_list,disk_lba,nblocks,0); + + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Read at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + + free(inode_double_block_list); + return -1; + } + + + for (i = 0; i < num_inode_ptrs_per_fs_block;i++) { + + + if (cusfs_num_data_blocks_single_indirect_inode_ptr(cufs,inode, + inode_double_block_list[i], + &num_indirect_blocks,0)) { + + + rc = -1; + *num_fs_blocks += num_indirect_blocks; + break; + + + } + + if (num_indirect_blocks < num_inode_ptrs_per_fs_block) { + + *num_fs_blocks += num_indirect_blocks; + break; + } + + + *num_fs_blocks += num_indirect_blocks; + + } + + + + free(inode_double_block_list); + return 0; + + + +} + + + +/* + * NAME: cusfs_num_data_blocks_triple_indirect_inode_ptr + * + * FUNCTION: Determine the number of data blocks currently + * referenced by this triple indirect inode pointer + * + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_num_data_blocks_triple_indirect_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t triple_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags) +{ + + + int rc = 0; + int i; + uint64_t disk_lba; + int nblocks; + uint64_t num_inode_ptrs_per_fs_block; + uint64_t *inode_triple_block_list = NULL; + uint64_t num_indirect_blocks; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + if (triple_indirect_inode_fs_lba == 0x0LL) { + + errno = EINVAL; + return -1; + } + + *num_fs_blocks = 0; + + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + + inode_triple_block_list = malloc(cufs->fs_block_size); + + if (inode_triple_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of triple_indirect of size 0x%x, for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->fs_block_size,cufs->device_name,errno); + + return -1; + } + + + + /* + * Convert fs_block_size sector to disk_block_size sector + */ + + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,triple_indirect_inode_fs_lba, + &disk_lba)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to get disk blocks for fs_lba 0x%ll for inode of type %d on disk %s", + triple_indirect_inode_fs_lba,inode->type, cufs->device_name); + + free(inode_triple_block_list); + return -1; + } + + nblocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_read(cufs->chunk_id,inode_triple_block_list,disk_lba,nblocks,0); + + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Read at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + + free(inode_triple_block_list); + return -1; + } + + + for (i = 0; i < num_inode_ptrs_per_fs_block;i++) { + + + if (cusfs_num_data_blocks_single_indirect_inode_ptr(cufs,inode, + inode_triple_block_list[i], + &num_indirect_blocks,0)) { + + + rc = -1; + *num_fs_blocks += num_indirect_blocks; + break; + + + } + + if (num_indirect_blocks < num_inode_ptrs_per_fs_block) { + + *num_fs_blocks += num_indirect_blocks; + break; + } + + + *num_fs_blocks += num_indirect_blocks; + + } + + + + free(inode_triple_block_list); + return 0; + + + +} + + + +/* + * NAME: cusfs_num_data_blocks_in_inode + * + * FUNCTION: Determine the number of data blocks currently + * referenced by this triple indirect inode pointer + * + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_num_data_blocks_in_inode(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t *num_fs_blocks, int *level,int flags) +{ + + + int rc = 0; + uint64_t num_data_blocks; + int i; + uint64_t num_inode_ptrs_per_fs_block; + uint64_t num_indirect_blocks; + + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + num_data_blocks = 0; + + + CUSFS_TRACE_LOG_FILE(9,"inode->type %d on disk = %s", + inode->type, cufs->device_name); + + for (i = 0; i < CFLSH_USFS_NUM_TOTAL_INODE_PTRS; i++) { + + if (i < CFLSH_USFS_NUM_DIRECT_INODE_PTRS) { + + if (inode->ptrs[i] == 0x0LL) { + + *level = (CFLSH_USFS_NUM_DIRECT_INODE_PTRS -1); + + CUSFS_TRACE_LOG_FILE(9,"i = 0x%x, level = 0x%x ", + i,*level); + break; + } + + num_data_blocks++; + + } else if (i < CFLSH_USFS_INDEX_DOUBLE_INDIRECT) { + + if (inode->ptrs[i] == 0x0LL) { + + *level = (CFLSH_USFS_NUM_DIRECT_INODE_PTRS -1); + CUSFS_TRACE_LOG_FILE(9,"i = 0x%x, level = 0x%x ", + i,*level); + break; + } + + if (cusfs_num_data_blocks_single_indirect_inode_ptr(cufs,inode, + inode->ptrs[i], + &num_indirect_blocks,0)) { + + rc = -1; + break; + } + + num_data_blocks += num_indirect_blocks; + + + if (num_indirect_blocks < num_inode_ptrs_per_fs_block) { + + + *level = CFLSH_USFS_INDEX_SINGLE_INDIRECT; + CUSFS_TRACE_LOG_FILE(9,"i = 0x%x, level = 0x%x ", + i,*level); + break; + } + + } else if (i < CFLSH_USFS_INDEX_TRIPLE_INDIRECT) { + + + + if (inode->ptrs[i] == 0x0LL) { + + *level = CFLSH_USFS_INDEX_SINGLE_INDIRECT; + CUSFS_TRACE_LOG_FILE(9,"i = 0x%x, level = 0x%x ", + i,*level); + break; + } + + if (cusfs_num_data_blocks_double_indirect_inode_ptr(cufs,inode, + inode->ptrs[i], + &num_indirect_blocks,0)) { + + rc = -1; + + break; + } + + num_data_blocks += num_indirect_blocks; + + + if (num_indirect_blocks < num_inode_ptrs_per_fs_block) { + + *level = CFLSH_USFS_INDEX_DOUBLE_INDIRECT; + break; + } + + } else if (i == CFLSH_USFS_INDEX_TRIPLE_INDIRECT) { + + + if (inode->ptrs[i] == 0x0LL) { + + *level = CFLSH_USFS_INDEX_DOUBLE_INDIRECT; + CUSFS_TRACE_LOG_FILE(9,"i = 0x%x, level = 0x%x ", + i,*level); + break; + } + + if (cusfs_num_data_blocks_triple_indirect_inode_ptr(cufs,inode, + inode->ptrs[i], + &num_indirect_blocks,0)) { + rc = -1; + + break; + } + + + *level = CFLSH_USFS_INDEX_TRIPLE_INDIRECT; + num_data_blocks += num_indirect_blocks; + + } + + } /* for */ + + + *num_fs_blocks = num_data_blocks; + + CUSFS_TRACE_LOG_FILE(9,"num_data_blocks = 0x%llx, level = 0x%x ", + num_data_blocks,*level); + return rc; +} +/* + * NAME: cusfs_get_next_inode_data_block + * + * FUNCTION: Add up to num_fs_lbas data blocks to this + * single indirect inode pointer. + * + * + * + * INPUTS: + * + * + * RETURNS: 0 failure, otherwise success + * + * + */ + +uint64_t cusfs_get_next_inode_data_block(cflsh_usfs_t *cufs, uint64_t **block_list, + uint64_t *block_list_index,uint64_t num_needed_blocks, int flags) +{ + uint64_t block_list_num; + uint64_t *local_block_list; + + if (cufs == NULL) { + + errno = EINVAL; + return 0; + + } + + if (block_list_index == NULL) { + + errno = EINVAL; + return 0; + + } + + if (num_needed_blocks < 1) { + + + errno = EINVAL; + return 0; + } + + + if (*block_list == NULL) { + + + block_list_num = num_needed_blocks; + + *block_list_index = 0; + + if (cusfs_get_data_blocks_list(cufs, &block_list_num, block_list,0)) { + + + CUSFS_TRACE_LOG_FILE(1,"failed to get requested blocks 0x%llx on disk %s", + num_needed_blocks, cufs->device_name); + return 0; + } + + if (block_list_num < num_needed_blocks) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get requested blocks 0x%llx, insteaed got 0x%llx on disk %s", + num_needed_blocks, block_list_num,cufs->device_name); + free(*block_list); + return 0; + } + + } else { + + + if (*block_list_index >= num_needed_blocks) { + + CUSFS_TRACE_LOG_FILE(1,"block_list_index = 0x%llx is too large *block_list_num = 0x%llx on disk %s", + *block_list_index,num_needed_blocks, cufs->device_name); + + errno = EINVAL; + return 0; + + + } + + } + + local_block_list = *block_list; + + CUSFS_TRACE_LOG_FILE(9,"local_block_list[0x%llx] = 0x%llx on disk %s", + block_list_index,local_block_list[*block_list_index], cufs->device_name); + + return (local_block_list[(*block_list_index)++]); +} + +/* + * NAME: cusfs_add_fs_blocks_to_single_inode_ptr + * + * FUNCTION: Add up to num_fs_lbas data blocks to this + * single indirect inode pointer. + * + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_add_fs_blocks_to_single_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t single_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags) +{ + int rc = 0; + int fail_for = FALSE; + int i; + uint64_t disk_lba; + int nblocks; + uint64_t num_inode_ptrs_per_fs_block; + uint64_t *inode_single_block_list = NULL; + int set_num_needed_blocks = FALSE; + int num_needed_blocks = 0; /* Number of disk blocks needed at this level */ + uint64_t *block_list = NULL; + uint64_t block_list_index = 0; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + if (single_indirect_inode_fs_lba == 0x0LL) { + + errno = EINVAL; + return -1; + } + + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + + inode_single_block_list = malloc(cufs->fs_block_size); + + if (inode_single_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of single_indirect of size 0x%x, for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->fs_block_size,cufs->device_name,errno); + + return -1; + } + + /* + * Convert fs_block_size sector to disk_block_size sector + */ + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,single_indirect_inode_fs_lba, + &disk_lba)) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get disk blocks for fs_lba 0x%ll for inode of type %d on disk %s", + single_indirect_inode_fs_lba,inode->type, cufs->device_name); + + free(inode_single_block_list); + return -1; + } + + nblocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_read(cufs->chunk_id,inode_single_block_list,disk_lba,nblocks,0); + + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Read at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + free(inode_single_block_list); + return -1; + } + + + if (flags & CFLSH_USFS_INODE_FLG_NEW_INDIRECT_BLK) { + + /* + * This the first time accessing this indirect + * block. So initialize it. + */ + + bzero(inode_single_block_list,cufs->fs_block_size); + + } + + for (i = 0; i < num_inode_ptrs_per_fs_block && *num_fs_blocks > 0;i++) { + + if (inode_single_block_list[i] == 0x0LL) { + + if (!set_num_needed_blocks) { + + set_num_needed_blocks = TRUE; + + num_needed_blocks = MIN((num_inode_ptrs_per_fs_block - i),*num_fs_blocks); + + } + + inode_single_block_list[i] = cusfs_get_next_inode_data_block(cufs,&block_list,&block_list_index, + num_needed_blocks,0); + + if (inode_single_block_list[i] == 0x0LL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block for indirect_inode_ptr for inode of type %d on disk %s", + inode->type, cufs->device_name); + fail_for = TRUE; + break; + + } + + (*num_fs_blocks)--; + } + + } + + + if (block_list) { + + free(block_list); + } + + rc = cblk_write(cufs->chunk_id,inode_single_block_list,disk_lba,nblocks,0); + + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Write at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + rc = -1; + + } else { + + rc = 0; + } + + if (fail_for) { + + rc = -1; + } + + free(inode_single_block_list); + return 0; +} + + +/* + * NAME: cusfs_add_fs_blocks_to_double_inode_ptr + * + * FUNCTION: Add up to num_fs_lbas data blocks to this + * double indirect inode pointer. + * + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_add_fs_blocks_to_double_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t double_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags) +{ + int rc = 0; + int fail_for = FALSE; + int i; + uint64_t disk_lba; + int nblocks; + uint64_t num_inode_ptrs_per_fs_block; + uint64_t *inode_double_block_list = NULL; + int local_flags; + int set_num_needed_blocks = FALSE; + int num_needed_blocks = 0; /* Number of disk blocks needed at this level */ + uint64_t *block_list = NULL; + uint64_t block_list_index = 0; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + if (double_indirect_inode_fs_lba == 0x0LL) { + + errno = EINVAL; + return -1; + } + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + + inode_double_block_list = malloc(cufs->fs_block_size); + + if (inode_double_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of double_indirect of size 0x%x, for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->fs_block_size,cufs->device_name,errno); + + return -1; + } + + /* + * Convert fs_block_size sector to disk_block_size sector + */ + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,double_indirect_inode_fs_lba, + &disk_lba)) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get disk blocks for fs_lba 0x%ll for inode of type %d on disk %s", + double_indirect_inode_fs_lba,inode->type, cufs->device_name); + + free(inode_double_block_list); + return -1; + } + + nblocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_read(cufs->chunk_id,inode_double_block_list,disk_lba,nblocks,0); + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Read at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + free(inode_double_block_list); + return -1; + } + + + if (flags & CFLSH_USFS_INODE_FLG_NEW_INDIRECT_BLK) { + + /* + * This the first time accessing this indirect + * block. So initialize it. + */ + + bzero(inode_double_block_list,cufs->fs_block_size); + + } + + for (i = 0; i < num_inode_ptrs_per_fs_block && *num_fs_blocks > 0;i++) { + + if (inode_double_block_list[i] == 0x0LL) { + + if (!set_num_needed_blocks) { + + set_num_needed_blocks = TRUE; + + num_needed_blocks = MIN((num_inode_ptrs_per_fs_block - i),*num_fs_blocks); + + } + + inode_double_block_list[i] = cusfs_get_next_inode_data_block(cufs,&block_list,&block_list_index, + num_needed_blocks,0); + + if (inode_double_block_list[i] == 0x0LL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block for indirect_inode_ptr for inode of type %d on disk %s", + inode->type, cufs->device_name); + fail_for = TRUE; + break; + + } + + local_flags = CFLSH_USFS_INODE_FLG_NEW_INDIRECT_BLK; + } else { + + local_flags = 0; + + } + + if (cusfs_add_fs_blocks_to_single_inode_ptr(cufs,inode,inode_double_block_list[i],num_fs_blocks,local_flags)) { + + fail_for = TRUE; + break; + } + + } + + + if (block_list) { + + free(block_list); + } + + + rc = cblk_write(cufs->chunk_id,inode_double_block_list,disk_lba,nblocks,0); + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Write at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + rc = -1; + } else { + + rc = 0; + } + + + if (fail_for) { + + rc = -1; + } + + free(inode_double_block_list); + return rc; +} + + +/* + * NAME: cusfs_add_fs_blocks_to_triple_inode_ptr + * + * FUNCTION: Add up to num_fs_lbas data blocks to this + * triple indirect inode pointer. + * + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_add_fs_blocks_to_triple_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t triple_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags) +{ + int rc = 0; + int fail_for = FALSE; + int i; + uint64_t disk_lba; + int nblocks; + uint64_t num_inode_ptrs_per_fs_block; + uint64_t *inode_triple_block_list = NULL; + int local_flags; + int set_num_needed_blocks = FALSE; + int num_needed_blocks = 0; /* Number of disk blocks needed at this level */ + uint64_t *block_list = NULL; + uint64_t block_list_index = 0; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + if (triple_indirect_inode_fs_lba == 0x0LL) { + + errno = EINVAL; + return -1; + } + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + + inode_triple_block_list = malloc(cufs->fs_block_size); + + if (inode_triple_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of triple_indirect of size 0x%x, for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->fs_block_size,cufs->device_name,errno); + + return -1; + } + + /* + * Convert fs_block_size sector to disk_block_size sector + */ + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,triple_indirect_inode_fs_lba, + &disk_lba)) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get disk blocks for fs_lba 0x%ll for inode of type %d on disk %s", + triple_indirect_inode_fs_lba,inode->type, cufs->device_name); + + free(inode_triple_block_list); + return -1; + } + + nblocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_read(cufs->chunk_id,inode_triple_block_list,disk_lba,nblocks,0); + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Read at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + free(inode_triple_block_list); + return -1; + } + + + + + if (flags & CFLSH_USFS_INODE_FLG_NEW_INDIRECT_BLK) { + + /* + * This the first time accessing this indirect + * block. So initialize it. + */ + + bzero(inode_triple_block_list,cufs->fs_block_size); + + } + + for (i = 0; i < num_inode_ptrs_per_fs_block && *num_fs_blocks > 0;i++) { + + if (inode_triple_block_list[i] == 0x0LL) { + + if (!set_num_needed_blocks) { + + set_num_needed_blocks = TRUE; + + num_needed_blocks = MIN((num_inode_ptrs_per_fs_block - i),*num_fs_blocks); + + } + + inode_triple_block_list[i] = cusfs_get_next_inode_data_block(cufs,&block_list,&block_list_index, + num_needed_blocks,0); + + if (inode_triple_block_list[i] == 0x0LL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block for indirect_inode_ptr for inode of type %d on disk %s", + inode->type, cufs->device_name); + fail_for = TRUE; + break; + + } + + local_flags = CFLSH_USFS_INODE_FLG_NEW_INDIRECT_BLK; + } else { + + local_flags = 0; + + } + + if (cusfs_add_fs_blocks_to_double_inode_ptr(cufs,inode,inode_triple_block_list[i],num_fs_blocks,local_flags)) { + + fail_for = TRUE; + break; + } + } + + + if (block_list) { + + free(block_list); + } + + + rc = cblk_write(cufs->chunk_id,inode_triple_block_list,disk_lba,nblocks,0); + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Write at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + rc = -1; + } + + if (fail_for) { + + rc = -1; + } else { + + rc = 0; + } + + + free(inode_triple_block_list); + return rc; +} + +/* + * NAME: cusfs_add_fs_blocks_to_inode_ptrs + * + * FUNCTION: Add num_fs_blocks filesystem data blocks to + * the end of the inode pointers. This code + * will also get data blocks from + * the free block table for others to use. + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_add_fs_blocks_to_inode_ptrs(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t num_fs_lbas,int flags) +{ + int rc = 0; + int i; + int local_flags; + int first_unused_inode_ptr = 0; + int num_needed_blocks = 1; /* Number of disk blocks needed at this level */ + uint64_t *block_list = NULL; + uint64_t block_list_index = 0; + uint64_t num_inode_ptrs_per_fs_block; + + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_lbas < 1) { + + + errno = EINVAL; + return -1; + } + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + for (i = 0; i < CFLSH_USFS_NUM_TOTAL_INODE_PTRS && num_fs_lbas > 0; i++) { + + if (i < CFLSH_USFS_NUM_DIRECT_INODE_PTRS) { + + if (inode->ptrs[i] == 0x0LL) { + + if (!first_unused_inode_ptr) { + + first_unused_inode_ptr = i; + + /* + * Determine number of data blocks needed + * at this level to satisfy the requested number + * fs blocks. + */ + + num_needed_blocks = MIN((CFLSH_USFS_NUM_DIRECT_INODE_PTRS - i + 1),num_fs_lbas); + + + if (num_fs_lbas > (num_inode_ptrs_per_fs_block + (CFLSH_USFS_NUM_DIRECT_INODE_PTRS - i))) { + + /* + * We also need a double indirect pointer + */ + num_needed_blocks++; + + } + + if (num_fs_lbas > (num_inode_ptrs_per_fs_block * num_inode_ptrs_per_fs_block + num_inode_ptrs_per_fs_block + (CFLSH_USFS_NUM_DIRECT_INODE_PTRS - i))) { + + /* + * We also need a triple indirect pointer + */ + num_needed_blocks++; + + } + + + } + + inode->ptrs[i] = cusfs_get_next_inode_data_block(cufs,&block_list,&block_list_index,num_needed_blocks,0); + + if (inode->ptrs[i] == 0x0LL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block for indirect_inode_ptr for inode of type %d on disk %s", + inode->type, cufs->device_name); + break; + } + num_fs_lbas--; + } + + } else if (i < CFLSH_USFS_INDEX_DOUBLE_INDIRECT) { + + if (inode->ptrs[i] == 0x0LL) { + + + if (!first_unused_inode_ptr) { + + num_needed_blocks = 1; + + if (num_fs_lbas > num_inode_ptrs_per_fs_block) { + + /* + * We also need a double indirect pointer + */ + num_needed_blocks++; + + } + + if (num_fs_lbas > (num_inode_ptrs_per_fs_block * num_inode_ptrs_per_fs_block)) { + + /* + * We also need a triple indirect pointer + */ + num_needed_blocks++; + + } + } + + inode->ptrs[i] = cusfs_get_next_inode_data_block(cufs,&block_list,&block_list_index,num_needed_blocks,0); + + if (inode->ptrs[i] == 0x0LL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block for indirect_inode_ptr for inode of type %d on disk %s", + inode->type, cufs->device_name); + break; + } + + local_flags = CFLSH_USFS_INODE_FLG_NEW_INDIRECT_BLK; + } else { + + local_flags = 0; + + } + + if (cusfs_add_fs_blocks_to_single_inode_ptr(cufs,inode,inode->ptrs[i],&num_fs_lbas,local_flags)) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block for indirect_inode_ptr for inode of type %d on disk %s", + inode->type, cufs->device_name); + break; + } + + + } else if (i < CFLSH_USFS_INDEX_TRIPLE_INDIRECT) { + + + if (inode->ptrs[i] == 0x0LL) { + + + if (!first_unused_inode_ptr) { + + num_needed_blocks = 1; + + if (num_fs_lbas > (num_inode_ptrs_per_fs_block * num_inode_ptrs_per_fs_block)) { + + /* + * We also need a triple indirect pointer + */ + num_needed_blocks++; + + } + + } + + inode->ptrs[i] = cusfs_get_next_inode_data_block(cufs,&block_list,&block_list_index,num_needed_blocks,0); + + if (inode->ptrs[i] == 0x0LL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block for indirect_inode_ptr for inode of type %d on disk %s", + inode->type, cufs->device_name); + break; + } + + local_flags = CFLSH_USFS_INODE_FLG_NEW_INDIRECT_BLK; + } else { + + local_flags = 0; + + } + + if (cusfs_add_fs_blocks_to_double_inode_ptr(cufs,inode,inode->ptrs[i],&num_fs_lbas,local_flags)) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block for indirect_inode_ptr for inode of type %d on disk %s", + inode->type, cufs->device_name); + break; + } + + + } else if (i == CFLSH_USFS_INDEX_TRIPLE_INDIRECT) { + + + if (inode->ptrs[i] == 0x0LL) { + + if (!first_unused_inode_ptr) { + + num_needed_blocks = 1; + } + + inode->ptrs[i] = cusfs_get_next_inode_data_block(cufs,&block_list,&block_list_index,num_needed_blocks,0); + + if (inode->ptrs[i] == 0x0LL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block for indirect_inode_ptr for inode of type %d on disk %s", + inode->type, cufs->device_name); + break; + } + + local_flags = CFLSH_USFS_INODE_FLG_NEW_INDIRECT_BLK; + } else { + + local_flags = 0; + + } + + if (cusfs_add_fs_blocks_to_triple_inode_ptr(cufs,inode,inode->ptrs[i],&num_fs_lbas,local_flags)) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to get data block for indirect_inode_ptr for inode of type %d on disk %s", + inode->type, cufs->device_name); + break; + } + + } + } + + if (block_list) { + + free(block_list); + } + + + rc = cusfs_update_inode(cufs,inode,0); + + + if ((rc == 0) && + (num_fs_lbas > 0)) { + + rc = -1; + } + return rc; +} + + +/* + * NAME: cusfs_remove_fs_blocks_from_single_inode_ptr + * + * FUNCTION: Remove up to num_fs_lbas data blocks from this + * single indirect inode pointer. + * + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_remove_fs_blocks_from_single_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t single_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags) +{ + + + int rc = 0; + int i; + uint64_t disk_lba; + int nblocks; + uint64_t num_inode_ptrs_per_fs_block; + uint64_t *inode_single_block_list = NULL; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + + inode_single_block_list = malloc(cufs->fs_block_size); + + if (inode_single_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of single_indirect of size 0x%x, for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->fs_block_size,cufs->device_name,errno); + + return -1; + } + + + + /* + * Convert fs_block_size sector to disk_block_size sector + */ + + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,single_indirect_inode_fs_lba, + &disk_lba)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to get disk blocks for fs_lba 0x%ll for inode of type %d on disk %s", + single_indirect_inode_fs_lba,inode->type, cufs->device_name); + + free(inode_single_block_list); + return -1; + } + + nblocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_read(cufs->chunk_id,inode_single_block_list,disk_lba,nblocks,0); + + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Read at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + + free(inode_single_block_list); + return -1; + } + + + for (i = num_inode_ptrs_per_fs_block; i > 0 && *num_fs_blocks > 0;i--) { + + if (inode_single_block_list[i] != 0x0LL) { + + + cusfs_release_data_blocks(cufs,inode_single_block_list[i],1,0); + + inode_single_block_list[i] = 0; + + (*num_fs_blocks)--; + } + + } + + + rc = cblk_write(cufs->chunk_id,inode_single_block_list,disk_lba,nblocks,0); + + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Write at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + + rc = -1; + } + + free(inode_single_block_list); + return rc; + + + +} + + + +/* + * NAME: cusfs_remove_fs_blocks_from_double_inode_ptr + * + * FUNCTION: Remove up to num_fs_lbas data blocks from this + * double indirect inode pointer. + * + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_remove_fs_blocks_from_double_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t double_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags) +{ + + + int rc = 0; + int i; + uint64_t disk_lba; + int nblocks; + uint64_t num_inode_ptrs_per_fs_block; + uint64_t *inode_double_block_list = NULL; + + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + + inode_double_block_list = malloc(cufs->fs_block_size); + + if (inode_double_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of double_indirect of size 0x%x, for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->fs_block_size,cufs->device_name,errno); + + return -1; + } + + + + /* + * Convert fs_block_size sector to disk_block_size sector + */ + + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,double_indirect_inode_fs_lba, + &disk_lba)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to get disk blocks for fs_lba 0x%ll for inode of type %d on disk %s", + double_indirect_inode_fs_lba,inode->type, cufs->device_name); + + free(inode_double_block_list); + return -1; + } + + nblocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_read(cufs->chunk_id,inode_double_block_list,disk_lba,nblocks,0); + + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Read at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + + free(inode_double_block_list); + return -1; + } + + + for (i = num_inode_ptrs_per_fs_block; i > 0 && *num_fs_blocks > 0;i--) { + + if (inode_double_block_list[i] != 0x0LL) { + + if (cusfs_remove_fs_blocks_from_single_inode_ptr(cufs,inode, + inode_double_block_list[i],num_fs_blocks,0)) { + + /* + * Do best effort to write out the updated double indirect + * block. + */ + + cblk_write(cufs->chunk_id,inode_double_block_list,disk_lba,nblocks,0); + free(inode_double_block_list); + return -1; + } + + + inode_double_block_list[i] = 0; + + + } + + } + + + rc = cblk_write(cufs->chunk_id,inode_double_block_list,disk_lba,nblocks,0); + + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Write at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + rc = -1; + } + + + free(inode_double_block_list); + return rc; + +} + + + + + +/* + * NAME: cusfs_remove_fs_blocks_from_triple_inode_ptr + * + * FUNCTION: Remove up to num_fs_lbas data blocks from this + * triple indirect inode pointer. + * + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_remove_fs_blocks_from_triple_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t triple_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags) +{ + + + int rc = 0; + int i; + uint64_t disk_lba; + int nblocks; + uint64_t num_inode_ptrs_per_fs_block; + uint64_t *inode_triple_block_list = NULL; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_blocks == NULL) { + + errno = EINVAL; + return -1; + } + + + num_inode_ptrs_per_fs_block = cufs->fs_block_size/sizeof(uint64_t); + + + inode_triple_block_list = malloc(cufs->fs_block_size); + + if (inode_triple_block_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Malloc of triple_indirect of size 0x%x, for num_inode_ptrs = %d on disk %s failed with errno %d ", + cufs->fs_block_size,cufs->device_name,errno); + + return -1; + } + + + + /* + * Convert fs_block_size sector to disk_block_size sector + */ + + + if (cusfs_get_disk_block_no_from_fs_block_no(cufs,triple_indirect_inode_fs_lba, + &disk_lba)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to get disk blocks for fs_lba 0x%ll for inode of type %d on disk %s", + triple_indirect_inode_fs_lba,inode->type, cufs->device_name); + + free(inode_triple_block_list); + return -1; + } + + nblocks = cufs->fs_block_size/cufs->disk_block_size; + + rc = cblk_read(cufs->chunk_id,inode_triple_block_list,disk_lba,nblocks,0); + + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Read at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + + free(inode_triple_block_list); + return -1; + } + + + for (i = num_inode_ptrs_per_fs_block; i > 0 && *num_fs_blocks > 0;i--) { + + if (inode_triple_block_list[i] != 0x0LL) { + + if (cusfs_remove_fs_blocks_from_double_inode_ptr(cufs,inode, + inode_triple_block_list[i],num_fs_blocks,0)) { + + /* + * Do best effort to write out the updated triple indirect + * block. + */ + cblk_write(cufs->chunk_id,inode_triple_block_list,disk_lba,nblocks,0); + free(inode_triple_block_list); + return -1; + } + + inode_triple_block_list[i] = 0; + + + } + + } + + + rc = cblk_write(cufs->chunk_id,inode_triple_block_list,disk_lba,nblocks,0); + + + if (rc < nblocks) { + + + CUSFS_TRACE_LOG_FILE(1,"Write at lba 0x%llx faild for indirect_inode_ptrt %d for inode of type %d on disk %s", + disk_lba,inode->type, cufs->device_name); + + + rc = -1; + } + + + free(inode_triple_block_list); + return rc; + + + +} + + +/* + * NAME: cusfs_remove_fs_blocks_to_inode_ptrs + * + * FUNCTION: Remove num_fs_blocks filesystem data blocks from + * the end of the inode pointers. This code + * will also make this data blocks available + * in the free block table for others to use. + * blocks to an inode, by update the inode. + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_remove_fs_blocks_from_inode_ptrs(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t num_fs_lbas,int flags) +{ + int rc = 0; + int i; + int level; + uint64_t num_data_blocks; + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (inode == NULL) { + + errno = EINVAL; + return -1; + } + + if (num_fs_lbas < 1) { + + + errno = EINVAL; + return -1; + } + + + /* + * First determine how many data blocks are + * being referenced from this inode. + * + * The maximum possible total of such inode pointers + * is the direct inode pointers in the + * inode itself (CFLSH_USFS_NUM_DIRECT_INODE_PTRS) + + * the number of inode pointers in the first single indirect + * inode pointer (num_inode_ptrs_per_fs_block) + the + * number of inode pointers in the double indirect + * inode pointer (num_inode_ptrs_per_fs_block * num_inode_ptrs_per_fs_block) + * + the number of inode pinters in the triple indirect inode pointer + * (num_inode_ptrs_per_fs_block * num_inode_ptrs_per_fs_block * num_inode_ptrs_per_fs_block). + */ + + + + num_data_blocks = 0; + + if (cusfs_num_data_blocks_in_inode(cufs,inode,&num_data_blocks,&level,0)) { + + return -1; + } + + if (num_data_blocks < 1) { + + CUSFS_TRACE_LOG_FILE(1,"No inode pointers in use for this inode on disk %s", + cufs->device_name); + + errno = EINVAL; + return -1; + } + + if (num_fs_lbas > num_data_blocks) { + + + CUSFS_TRACE_LOG_FILE(1,"num_fs_lbas 0x%x > max_possible_inode_ptrs 0x%x on disk %s", + num_fs_lbas, num_data_blocks,cufs->device_name); + + errno = EINVAL; + return -1; + } + + + switch (level) { + + case CFLSH_USFS_INDEX_TRIPLE_INDIRECT: + + /* + * Remove up to num_fs_lbas from the + * triple indirect blocks + */ + + if (cusfs_remove_fs_blocks_from_triple_inode_ptr(cufs, inode, + inode->ptrs[CFLSH_USFS_INDEX_TRIPLE_INDIRECT], + &num_fs_lbas,0)) { + break; + } + + if (num_fs_lbas == 0) { + break; + } + + /* Drop through */ + + case CFLSH_USFS_INDEX_DOUBLE_INDIRECT: + + + /* + * Remove up to num_fs_lbas from the + * double indirect blocks + */ + + if (cusfs_remove_fs_blocks_from_double_inode_ptr(cufs, inode, + inode->ptrs[CFLSH_USFS_INDEX_DOUBLE_INDIRECT], + &num_fs_lbas,0)) { + break; + } + + if (num_fs_lbas == 0) { + break; + } + + /* Drop through */ + case CFLSH_USFS_INDEX_SINGLE_INDIRECT: + + + /* + * Remove up to num_fs_lbas from the + * double indirect blocks + */ + + if (cusfs_remove_fs_blocks_from_single_inode_ptr(cufs, inode, + inode->ptrs[CFLSH_USFS_INDEX_SINGLE_INDIRECT], + &num_fs_lbas,0)) { + break; + } + + if (num_fs_lbas == 0) { + break; + } + + /* Drop through */ + + case (CFLSH_USFS_NUM_DIRECT_INODE_PTRS - 1): + + for (i = (CFLSH_USFS_NUM_DIRECT_INODE_PTRS - 1); i >=0; i--) { + + + cusfs_release_data_blocks(cufs,inode->ptrs[i],1,0); + + inode->ptrs[i] = 0; + + num_fs_lbas--; + } + break; + default: + CUSFS_TRACE_LOG_FILE(1,"Invalid level %d for disk %s", + level,cufs->device_name); + + errno = EINVAL; + return -1; + + } + + rc = cusfs_update_inode(cufs,inode,0); + + return rc; +} diff --git a/src/usfs/cflsh_usfs_internal.h b/src/usfs/cflsh_usfs_internal.h new file mode 100644 index 00000000..c809c870 --- /dev/null +++ b/src/usfs/cflsh_usfs_internal.h @@ -0,0 +1,1026 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* bos720 src/bos/usr/ccs/lib/libcflsh_block/cflsh_usfs_internal.h 1.6 */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/* %Z%%M% %I% %W% %G% %U% */ + + +/* + * COMPONENT_NAME: (sysxcflashusfs) CAPI Flash user space file system library + * + * FUNCTIONS: Data structure used by library for its various internal + * operations. + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +#ifndef _H_CFLASH_USFS_INT +#define _H_CFLASH_USFS_INT + +#ifndef _AIX +#define _GNU_SOURCE +#endif /* !_AIX */ + +#include "cflsh_usfs_disk.h" +#include "cflsh_usfs_wrapper.h" + + +#ifndef FALSE +#define FALSE 0 +#endif + +#ifndef TRUE +#define TRUE 1 +#endif + + + +/************************************************************************/ +/* Compile options */ +/************************************************************************/ +#ifdef _NOT_YET +#define _MASTER_LOCK 1 /* require master to coordinate locking of meta */ + /* data in filesystem */ +#endif /* NOT_YET */ + + + +/************************************************************************/ +/* Userspace filesyste library limits */ +/************************************************************************/ +#define MAX_NUM_THREAD_LOGS 4095 /* Maximum number of thread logs allowed per */ + /* process */ + +#define CUSFS_BITS_PER_BYTE 8 + + +#define CUSFS_PAGE_ALIGN 0xfffffffffffff000LL +/************************************************************************/ +/* CAPI user space filesyste library eye catchers */ +/************************************************************************/ +#define CFLSH_USFS_EYEC_GLOBAL __EYEC4('c','u','G','L') /* cuGL */ +#define CFLSH_USFS_EYEC_FS __EYEC4('c','u','F','S') /* cuFS */ +#define CFLSH_USFS_EYEC_FL __EYEC4('c','u','F','L') /* cuFL */ +#define CFLSH_USFS_EYEC_AIO __EYEC4('c','u','A','I') /* cuAI */ + +#define CFLSH_EYECATCH_AIO(aio) ((aio)->eyec != CFLSH_USFS_EYEC_AIO) + +#define CFLSH_EYECATCH_FILE(file) ((file)->eyec != CFLSH_USFS_EYEC_FL) + +#define CFLSH_EYECATCH_FS(cufs) ((cufs)->eyec != CFLSH_USFS_EYEC_FS) + +#define CFLSH_EYECATCH_GLOBAL(global) ((global)->eyec != CFLSH_USFS_EYEC_GLOBAL) + +/************************************************************************/ +/* Miscellaneous */ +/************************************************************************/ + +#define CFLSH_USFS_WAIT_AIO_BG_DLY 10 /* Time to delay in microseconds*/ + /* for background thread to wait*/ + /* for AIO to drain. */ + +#define CFLSH_USFS_WAIT_AIO_DELAY 100 /* Time to delay in microseconds*/ + /* to wait for AIO to drain. */ + +#define CFLSH_USFS_WAIT_AIO_RETRY 100000/* Retries to wait for AIO to */ + /* drain. */ + +#define CFLSH_USFS_WAIT_READ_AHEAD 100 /* Time to delay in microseconds*/ + /* for read ahead to complete */ + +#define CFLSH_USFS_WAIT_READ_AHEAD_RETRY 600000 /* Retries to wait for */ + /* read ahead to complete. */ + +#define CFLSH_USFS_PTHREAD_SIG 0x1 /* Send a pthread_cond_signal */ + +#define CFLSH_USFS_ONLY_ASYNC_CMPLT 0x1 /* only look for async completion */ + +#define CFLSH_USFS_ALLOC_READ_AHEAD 1 /* Allocate Read Ahead Buffer */ + +/************************************************************************/ +/* Delays/Retries */ +/************************************************************************/ + +#define CFLSH_USFS_RDWR_WRITE 0x001 /* Indicates write */ +#define CFLSH_USFS_RDWR_DIRECT 0x002 /* Indicates direct I/O */ + /* to user's buffer. */ + +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + + +#ifndef _AIX +/* + * fetch_and_or and fetch_and_add are not available for linux user space. + * It appears that __sync_fetch_and_or, and __sync_fetch_and_add are doing the + * required functionality, respectively. + */ + +#define fetch_and_or(ptr,val) (__sync_fetch_and_or(ptr,val)) + +#define fetch_and_add(ptr,val) (__sync_fetch_and_add(ptr,val)) + + +#endif /* !_AIX */ + + +/************************************************************************/ +/* Tracing/logging data structues and maroc */ +/************************************************************************/ +extern char *cusfs_log_filename; +extern int cusfs_log_verbosity; +extern FILE *cusfs_logfp; +extern FILE *cusfs_dumpfp; +extern int cusfs_dump_level; +extern int cusfs_notify_log_level; +extern int dump_sequence_num; + +extern pthread_mutex_t cusfs_log_lock; + +extern uint32_t num_thread_logs; /* Number of thread log files */ + + +/* + * For thread trace tables, using pthread_t does not work + * work well since the values can be very large (64-bit) + * and there is not an obvious way to hash them into something + * more manageable. So for thread trace table we use + * gettid for non-AIX and pthread_getthreadid_np. pthread_getthreadid_np + * is a non-portible pthread interface to get thread sequence numbers, + * It is available in some BSD unixes, but not AIX nor Linux. + * However AIX does support a close derivative pthread_getunique_np, which + * can be used to implement it. For linux we use gettid via the syscall + * interface. + */ +#ifdef _AIX + +inline int pthread_getthreadid_np(void) +{ + int tid = 0; + pthread_t my_pthread = pthread_self(); + + if (pthread_getunique_np(&my_pthread,&tid)) { + + tid = 0; + } + + return tid; +} + + + +#define CFLSH_BLK_GETTID (pthread_getthreadid_np()) +#else +#define CFLSH_BLK_GETTID (syscall(SYS_gettid)) +#endif /* ! AIX */ + +/* + * Trace/log the information to common file. If thread tracing + * is also on then log information to the hashed thread log file. + */ +#define CUSFS_TRACE_LOG_FILE(verbosity,msg, ...) \ +do { \ + if ((cusfs_log_filename != NULL) && \ + (verbosity <= cusfs_log_verbosity)) { \ + pthread_mutex_lock(&cusfs_log_lock); \ + if (cusfs_global.flags & CFLSH_G_SYSLOG) { \ + char msg1[PATH_MAX]; \ + /* \ + * If we are using syslog \ + */ \ + sprintf(msg1,"%s,%s,%d,%s",__FILE__,__FUNCTION__,__LINE__,msg); \ + syslog(LOG_DEBUG,msg1,## __VA_ARGS__); \ + \ + } else { \ + /* \ + * If we are not using syslog, but tracing to files \ + */ \ + cusfs_trace_log_data_ext(&(cusfs_global.trace_ext), \ + cusfs_logfp, \ + __FILE__,(char *)__FUNCTION__,__LINE__, \ + msg,## __VA_ARGS__); \ + if (num_thread_logs) { \ + int thread_index = CFLSH_BLK_GETTID & cusfs_global.thread_log_mask; \ + \ + \ + if ((thread_index >=0) && \ + (thread_index < num_thread_logs)) { \ + \ + cusfs_global.thread_logs[thread_index].ext_arg.log_number = cusfs_global.trace_ext.log_number; \ + cusfs_trace_log_data_ext(&(cusfs_global.thread_logs[thread_index].ext_arg), \ + cusfs_global.thread_logs[thread_index].logfp, \ + __FILE__,(char *)__FUNCTION__, \ + __LINE__,msg,## __VA_ARGS__); \ + \ + } \ + } \ + } \ + \ + pthread_mutex_unlock(&cusfs_log_lock); \ + } \ +} while (0) + +typedef struct cflsh_thread_log_s { + trace_log_ext_arg_t ext_arg; + FILE *logfp; +} cflsh_thread_log_t; + + + +/************************************************************************/ +/* Live dump macro */ +/************************************************************************/ + +#define CUSFS_LIVE_DUMP_THRESHOLD(level,msg) \ +do { \ + \ + if (level <= cusfs_dump_level) { \ + cusfs_dump_debug_data(msg,__FILE__,__FUNCTION__,__LINE__,__DATE__); \ + } \ + \ +} while (0) + +#ifdef _REMOVE +/************************************************************************/ +/* Notify/Log macro: This is only used when we want to control */ +/* amount of logging. */ +/************************************************************************/ + +#define CUSFS_NOTIFY_LOG_THRESHOLD(level,chunk,path_index,error_num,out_rc,reason, cmd) \ +do { \ + \ + if (level <= cusfs_notify_log_level) { \ + cusfs_notify_mc_err((chunk),(path_index),(error_num),(out_rc),(reason),(cmd)); \ + } \ + \ +} while (0) + +#endif /* _REMOVE */ +/************************************************************************/ +/* OS specific LWSYNC macros */ +/************************************************************************/ + +#ifdef _AIX +#define CUSFS_LWSYNC() asm volatile ("lwsync") +#elif _MACOSX +#define CUSFS_LWSYNC() asm volatile ("mfence") +#else +#define CUSFS_LWSYNC() __asm__ __volatile__ ("lwsync") +#endif + + + +/*************************************************************************** + * + * cusfs_lock + * + * Wrapper structure for mutex locks and accompanying macros + * + ***************************************************************************/ +typedef +struct cusfs_lock_s { + pthread_mutex_t plock; + pthread_t thread; /* Thread id of lock holder */ + int count; /* Non-zero when someone holding this lock */ + char *filename;/* Filename string of lock taker */ + uint file; /* File number of lock taker */ + uint line; /* Line number of lock taker */ +} cusfs_lock_t; + +/* + * Initialize the mutex lock fields + * + * cflsh_lock - cflsh_blk_lock_t structure + */ +#define CUSFS_LOCK_INIT(cflsh_lock) \ +do { \ + pthread_mutex_init(&((cflsh_lock).plock), NULL); \ + (cflsh_lock).count = 0; \ + (cflsh_lock).file = 0; \ + (cflsh_lock).line = 0; \ + (cflsh_lock).thread = 0; \ +} while (0) + +#define CUSFS_LOCK_IFREE(cflsh_lock) \ +do { \ + pthread_mutex_destroy(&((cflsh_lock).plock)); \ + (cflsh_lock).count = 0; \ + (cflsh_lock).file = 0; \ + (cflsh_lock).line = 0; \ + (cflsh_lock).thread = NULL; \ +} while (0) + +#define CUSFS_LOCK(cflsh_lock) \ +do { \ + \ + pthread_mutex_lock(&((cflsh_lock).plock)); \ + \ + if (cusfs_log_verbosity >= 5) { \ + (cflsh_lock).thread = pthread_self(); \ + } \ + (cflsh_lock).file = CFLSH_USFS_FILENUM; \ + (cflsh_lock).filename = __FILE__; \ + (cflsh_lock).line = __LINE__; \ + \ + (cflsh_lock).count++; \ + \ +} while (0) + + +#define CUSFS_UNLOCK(cflsh_lock) \ +do { \ + \ + (cflsh_lock).count--; \ + \ + (cflsh_lock).thread = 0; \ + (cflsh_lock).filename = NULL; \ + (cflsh_lock).file = 0; \ + (cflsh_lock).line = 0; \ + \ + pthread_mutex_unlock(&((cflsh_lock).plock)); \ + \ +} while (0) + + + + + +/*************************************************************************** + * + * cflsh_blk_rwlock + * + * Wrapper structure for mutex rwlocks and accompanying macros + * + ***************************************************************************/ +typedef +struct cusfs_rwlock_s { + pthread_rwlock_t plock; + pthread_t thread; /* Thread id of lock holder */ + int count; /* Non-zero when someone holding this lock */ + char *filename;/* Filename string of lock taker */ + uint file; /* File number of lock taker */ + uint line; /* Line number of lock taker */ +} cflsh_blk_rwlock_t; + +#ifdef _USE_RW_LOCK +/* + * Initialize the mutex read/write lock fields + * + * cflsh_lock - cflsh_blk_rwlock_t structure + */ +#define CUSFS_RWLOCK_INIT(cflsh_lock) \ +do { \ + pthread_rwlock_init(&((cflsh_lock).plock), NULL); \ + (cflsh_lock).count = 0; \ + (cflsh_lock).file = 0; \ + (cflsh_lock).line = 0; \ + (cflsh_lock).thread = 0; \ +} while (0) + +#define CUSFS_RWLOCK_IFREE(cflsh_lock) \ +do { \ + pthread_rwlock_destroy(&((cflsh_lock).plock)); \ + (cflsh_lock).count = 0; \ + (cflsh_lock).file = 0; \ + (cflsh_lock).line = 0; \ + (cflsh_lock).thread = NULL; \ +} while (0) + +#define CUSFS_RD_RWLOCK(cflsh_lock) \ +do { \ + \ + pthread_rwlock_rdlock(&((cflsh_lock).plock)); \ + \ + if (cusfs_log_verbosity >= 5) { \ + (cflsh_lock).thread = pthread_self(); \ + } \ + (cflsh_lock).file = CFLSH_USFS_FILENUM; \ + (cflsh_lock).filename = __FILE__; \ + (cflsh_lock).line = __LINE__; \ + \ + (cflsh_lock).count++; \ + \ +} while (0) + + +#define CUSFS_WR_RWLOCK(cflsh_lock) \ +do { \ + \ + pthread_rwlock_wrlock(&((cflsh_lock).plock)); \ + \ + if (cusfs_log_verbosity >= 5) { \ + (cflsh_lock).thread = pthread_self(); \ + } \ + (cflsh_lock).file = CFLSH_USFS_FILENUM; \ + (cflsh_lock).filename = __FILE__; \ + (cflsh_lock).line = __LINE__; \ + \ + (cflsh_lock).count++; \ + \ +} while (0) + +#define CUSFS_RWUNLOCK(cflsh_lock) \ +do { \ + \ + (cflsh_lock).count--; \ + \ + (cflsh_lock).thread = 0; \ + (cflsh_lock).filename = NULL; \ + (cflsh_lock).file = 0; \ + (cflsh_lock).line = 0; \ + \ + pthread_rwlock_unlock(&((cflsh_lock).plock)); \ + \ +} while (0) + +#else +/* + * Do not use read/write locks use mutex locks instead + */ +#define CUSFS_RWLOCK_INIT(cflsh_lock) CUSFS_LOCK_INIT(cflsh_lock) +#define CUSFS_RWLOCK_IFREE(cflsh_lock) CUSFS_LOCK_IFREE(cflsh_lock) +#define CUSFS_RD_RWLOCK(cflsh_lock) CUSFS_LOCK(cflsh_lock) +#define CUSFS_WR_RWLOCK(cflsh_lock) CUSFS_LOCK(cflsh_lock) +#define CUSFS_RWUNLOCK(cflsh_lock) CUSFS_UNLOCK(cflsh_lock) + +#endif + + + +/******************************************************************** + * Queueing Macros + * + * The CUSFS_Q/DQ_NODE macros enqueue and dequeue nodes to the head + * or tail of a doubly-linked list. They assume that 'next' and 'prev' + * are the names of the queueing pointers. + * + *******************************************************************/ + +/* + * Give a node and the head and tail pointer for a list, enqueue + * the node at the head of the list. Assumes the list is + * doubly-linked and NULL-terminated at both ends, and that node + * is non-NULL. Casts to void allow commands of different data + * types than the list to be queued into the list. This is useful + * if one data type is a subset of another. + */ +#define CUSFS_Q_NODE_HEAD(head, tail, node,_node_prev,_node_next) \ +do { \ + \ + /* If node is valid */ \ + if ((node) != NULL) \ + { \ + (node)->_node_prev = NULL; \ + (node)->_node_next = (head); \ + \ + if ((head) == NULL) { \ + \ + /* List is empty; 'node' is also the tail */ \ + (tail) = (node); \ + \ + } else { \ + \ + /* List isn't empty;old head must point to 'node' */ \ + (head)->_node_prev = (node); \ + } \ + \ + (head) = (node); \ + } \ + \ +} while (0) + + + +/* + * Give a node and the head and tail pointer for a list, enqueue + * the node at the tail of the list. Assumes the list is + * doubly-linked and NULL-terminated at both ends, and that node + * is non-NULL. Casts to void allow commands of different data + * types than the list to be queued into the list. This is useful + * if one data type is a subset of another. + */ +#define CUSFS_Q_NODE_TAIL(head, tail, node,_node_prev,_node_next) \ +do { \ + \ + /* If node is valid */ \ + if ((node) != NULL) \ + { \ + \ + (node)->_node_prev = (tail); \ + (node)->_node_next = NULL; \ + \ + if ((tail) == NULL) { \ + \ + /* List is empty; 'node' is also the head */ \ + (head) = (node); \ + \ + } else { \ + \ + /* List isn't empty;old tail must point to 'node' */ \ + (tail)->_node_next = (node); \ + } \ + \ + (tail) = (node); \ + } \ + \ + \ + \ +} while (0) + + +/* + * Given a node and the head and tail pointer for a list, dequeue + * the node from the list. Assumes the list is doubly-linked and + * NULL-terminated at both ends, and that node is non-NULL. + * + * Casts to void allow commands of different data types than the + * list to be dequeued into the list. This is useful if one data + * type is a subset of another. + */ +#define CUSFS_DQ_NODE(head, tail, node,_node_prev,_node_next) \ +do { \ + /* If node is valid */ \ + if ((node) != NULL) \ + { \ + /* If node was head, advance the head to node's next*/ \ + if ((head) == (node)) \ + { \ + (head) = ((node)->_node_next); \ + } \ + \ + /* If node was tail, retract the tail to node's prev */ \ + if ((tail) == (node)) \ + { \ + (tail) = ((node)->_node_prev); \ + } \ + \ + /* A follower's predecessor is now node's predecessor*/ \ + if ((node)->_node_next) \ + { \ + (node)->_node_next->_node_prev = ((node)->_node_prev); \ + } \ + \ + /* A predecessor's follower is now node's follower */ \ + if ((node)->_node_prev) \ + { \ + (node)->_node_prev->_node_next = ((node)->_node_next); \ + } \ + \ + (node)->_node_next = NULL; \ + (node)->_node_prev = NULL; \ + } \ + \ + \ +} while(0) + + + +/************************************************************************/ +/* AIOCB defines and macros */ +/************************************************************************/ + +#define CUSFS_AIOCB32_FLG 1 /* Indicates this is an aiocb and not aiocb64 */ + +#ifdef _AIX +#define CUSFS_RET_AIOCB(_aiocbp_,_rc_,_errno_) \ +do { \ + \ + (_aiocbp_)->aio_errno = (_errno_); \ + \ + (_aiocbp_)->aio_return = (_rc_); \ + \ +} while (0) + +#else + +#define CUSFS_RET_AIOCB(_aiocbp_,_rc_,_errno_) \ +do { \ + \ + (_aiocbp_)->__error_code = (_errno_); \ + \ + (_aiocbp_)->__return_value = (_rc_); \ + \ +} while (0) + + +#endif /* !_AIX */ + + + + +typedef enum { + CFLSH_USFS_AIO_E_INVAL = 0x0, /* Invalid operation */ + CFLSH_USFS_AIO_E_READ = 0x1, /* Read operations */ + CFLSH_USFS_AIO_E_WRITE = 0x2, /* Write operations */ +} cflsh_usfs_aio_op_t; + +/************************************************************************/ +/* cflsh_usfs_aio_entry - The data structure for a async I/O requests */ +/* */ +/* */ +/* */ +/************************************************************************/ +typedef struct cflsh_usfs_aio_entry_s { + struct cflsh_usfs_aio_entry_s *free_prev;/* Previous free async */ + /* I/O */ + struct cflsh_usfs_aio_entry_s *free_next;/* Next free async */ + /* I/O */ + struct cflsh_usfs_aio_entry_s *act_prev; /* Previous active async */ + /* I/O. */ + struct cflsh_usfs_aio_entry_s *act_next; /* Next active async I/O */ + struct cflsh_usfs_aio_entry_s *cmplt_prev;/* Previous completed */ + /* async I/O. */ + struct cflsh_usfs_aio_entry_s *cmplt_next;/* Next completed */ + /* async I/O. */ + struct cflsh_usfs_aio_entry_s *file_prev;/* Previous async I/O */ + /* for this file. */ + struct cflsh_usfs_aio_entry_s *file_next;/* Next async I/O */ + /* for this file */ + + int flags; +#define CFLSH_USFS_AIO_E_AIOCB 0x0001 /* This aio_entry is associated */ + /* with an aiocb. */ +#define CFLSH_USFS_AIO_E_AIOCB64 0x0002 /* This aio_entry is associated */ + /* with an aiocb64. */ +#define CFLSH_USFS_AIO_E_LBUF 0x0004 /* This aio_entry is using the */ + /* local_buffer. */ +#define CFLSH_USFS_AIO_E_ISSUED 0x0008/* Command has been issued */ + cflsh_usfs_aio_op_t op; /* Operation */ + int state; + uint64_t aio_offset; /* aiocb's aio_offset */ + uint64_t aio_nbytes; /* aiocb's aio_nbytes */ + struct aiocb64 *aiocbp; /* Pointer to associated aiocb */ + struct sigevent aio_sigevent; /* aiocb's aio_sigevent */ + struct cflsh_usfs_data_obj_s *file;/* Associated file structure */ + void *caller_buffer; /* Buffer provided by caller */ + /* to this library. */ + uint64_t caller_buffer_len; /* Length of caller buffer in */ + /* bytes. */ + void *local_buffer; /* Our local (internal) buffer */ + uint64_t local_buffer_len; /* Length of local buffer in */ + /* bytes. */ + uint64_t local_buffer_len_used; /* Amount of bytes used by this*/ + /* Async I/O of the */ + /* local_buffer. */ + int tag; /* tag returned by cblk_aread */ + /* or cblk_awrite. */ + int index; /* Index in async pool */ + size_t nblocks; /* Size of transfer in blocks */ + uint64_t offset; /* Offset in file */ + uint64_t start_lba; /* Starting LBA for this async */ + cblk_arw_status_t cblk_status; /* Status for this command */ + + eye_catch4b_t eyec; /* Eye catcher */ + +} cflsh_usfs_aio_entry_t; + + +struct cflsh_usfs_data_read_ahead { + int flags; +#define CFLSH_USFS_D_RH_ACTIVE 0x0001 /* Read ahead request is active*/ + int tag; /* Tag of async I/O */ + uint64_t start_offset; /* Starting disk LBA of read ahead */ + uint64_t start_lba; /* Starting disk LBA of read ahead */ + uint64_t nblocks; /* Size of read ahead in blocks */ + void *local_buffer; /* Our local (internal) buffer */ + uint64_t local_buffer_len; /* Length of local buffer in */ + /* bytes. */ + uint64_t local_buffer_len_used;/* Amount of bytes used by this */ + /* Request */ + + cblk_arw_status_t cblk_status;/* Status for this command */ + +}; + + + +/************************************************************************/ +/* cflsh_usfs_data_obj - The data structure for a data object in the */ +/* in the filesystem (i.e. file, directory, */ +/* symlink, etc). */ +/* */ +/************************************************************************/ +typedef struct cflsh_usfs_data_obj_s { + struct cflsh_usfs_data_obj_s *prev; /* Previous file in cufs list */ + struct cflsh_usfs_data_obj_s *next; /* Next file in cufs list */ + struct cflsh_usfs_data_obj_s *fprev; /* Previous file in global list*/ + struct cflsh_usfs_data_obj_s *fnext; /* Next file in global list */ + int flags; +#define CFLSH_USFS_ROOT_DIR 0x00001 +#define CFLSH_USFS_DIRECT_IO 0x00002 /* This file was open with */ + /* O_DIRECT flag set. */ +#define CFLSH_USFS_SYNC_IO 0x00004 /* This file was open with */ + /* O_SYNC flag set. */ +#define CFLSH_USFS_IN_HASH 0x00008 /* In file hash table */ + + + cflsh_usfs_inode_file_type_t file_type; + cflsh_usfs_inode_t *inode; + int index; + FILE *fp; + char filename[PATH_MAX]; /* Filename */ + + cflash_offset_t offset; /* offset in this file */ + + void *data_buf; /* Data buffer of file */ + int data_buf_size; /* Data buffer size */ + cusfs_lock_t lock; + + struct cflsh_usfs_s *fs; + + cflsh_usfs_disk_lba_e_t *disk_lbas; /* Disk LBAs for this file */ + uint64_t num_disk_lbas; /* Number of entries in the*/ + /* disk_lbas list. */ + struct cflsh_usfs_data_read_ahead seek_ahead; /* Read ahead on seek */ + struct { + int flags; +#define CFLSH_USFS_D_INODE_MTIME 0x0001 /* Modification time for */ + /* inode needs to be updated */ + /* to this value. */ +#define CFLSH_USFS_D_INODE_ATIME 0x0002 /* Last access time for */ + /* inode needs to be updated */ + /* to this value. */ +#if !defined(__64BIT__) && defined(_AIX) + time64_t mtime; /* Last time file content */ + /* was modified. */ + time64_t atime; /* Last time file was */ + /* accessed. */ +#else + time_t mtime; /* Last time file content */ + /* was modified. */ + time_t atime; /* Last time file was */ + /* accessed. */ +#endif + + } inode_updates; + + cflsh_usfs_aio_entry_t *head_act; /* Head of active async */ + /* I/O for this file. */ + cflsh_usfs_aio_entry_t *tail_act; /* Tail of active async */ + /* I/O for this file. */ + eye_catch4b_t eyec; /* Eye catcher */ + + +} cflsh_usfs_data_obj_t; + + + +/************************************************************************/ +/* CUFS hashing defines */ +/************************************************************************/ +#define CUSFS_BAD_ADDR_MASK 0xfff /* CUFS' should be allocated page aligned */ + /* this mask allows us to check for bad */ + /* chunk addresses */ + +#define MAX_NUM_CUSFS_HASH 64 +#define MAX_NUM_CUSFS_FILE_HASH 64 + +#define CUSFS_HASH_MASK 0x0000003f + +typedef struct cflsh_usfs_async_thread_s { + struct cflsh_usfs_s *cufs; /* Cufs associated with this */ + /* thread. */ + +} cflsh_usfs_async_thread_t; + + +struct cflsh_usfs_stat { + struct cflsh_usfs_s *cufs; /* Cufs associated with this */ + /* thread. */ + uint64_t num_areads; /* Total number of async reads */ + /* issued via cblk_aread interface */ + uint64_t num_awrites; /* Total number of async writes */ + /* issued via cblk_awrite interface*/ + uint32_t num_act_areads; /* Current number of reads active */ + /* via cblk_read interface */ + uint32_t num_act_awrites; /* Current number of writes active */ + /* via cblk_write interface */ + uint32_t max_num_act_awrites; /* High water mark on the maximum */ + /* number of asyync writes active */ + /* at once. */ + uint32_t max_num_act_areads; /* High water mark on the maximum */ + uint64_t num_error_areads; /* Total number of async reads */ + /* failed via cblk_aread interface */ + uint64_t num_error_awrites; /* Total number of async writes */ + /* failed via cblk_awrite interface*/ + uint64_t num_success_threads; /* Total number of pthread_creates */ + /* that succeed. */ + uint64_t num_failed_threads; /* Total number of pthread_creates */ + /* that failed. */ + uint64_t num_active_threads; /* Current number of threads */ + /* running. */ + + +}; + + + +/************************************************************************/ +/* cflsh_usfs - The data structure for a file system */ +/* for this process */ +/* . */ +/************************************************************************/ +typedef struct cflsh_usfs_s { + struct cflsh_usfs_s *prev; /* Previous filesystem in list */ + struct cflsh_usfs_s *next; /* Next filesystem in list */ + uint8_t in_use; /* This filessytem is in use */ + int flags; /* Flags for this filesystem */ +#define CFLSH_USFS_FLG_SB_VAL 0x0001 /* Superblock valid */ +#define CFLSH_USFS_FLG_FT_VAL 0x0002 /* Free block table valid */ +#define CFLSH_USFS_FLG_IT_VAL 0x0004 /* Inode table valid */ +#define CFLSH_USFS_FLG_FAIL_FORK 0x0008 /* The child process this */ + /* file system is associated with */ + /* encountered errors on the fork */ + int index; /* Index of this cufs */ + chunk_id_t chunk_id; /* Chunk index number of disk */ + /* for this filesystem */ + + + uint64_t fs_block_size; /* Block size referenced by inodes */ + uint64_t num_blocks; /* Size of file system in blocks */ + uint64_t num_inode_data_blocks; /* Number of data blocks (where */ + /* the block size is inode block */ + /* size, which is the block_size */ + /* field in this struct. */ + uint64_t size_free_blk_entries; /* size of free block table */ + /* entries in bytes */ + + uint32_t disk_block_size; /* Block size used by disk */ + + int max_requests; + int max_xfer_size; /* Maxium transfer size in blocks */ + void *buf; /* Buffer for this filesystem */ + int buf_size; /* Size of memory referenced by buf*/ + void *free_blk_tbl_buf; + int free_blk_tbl_buf_size; + void *inode_tbl_buf; + int inode_tbl_buf_size; + chunk_stats_t disk_stats; + cflsh_usfs_super_block_t superblock; + //?? stats + cflsh_usfs_data_obj_t *root_directory; /* Root directory of filesystem */ + + cflsh_usfs_data_obj_t *filelist_head; /* Head of queue of files in */ + /* use on this filesystem */ + + cflsh_usfs_data_obj_t *filelist_tail; /* Tail of queue of files in */ + /* use on this filesystem */ + + cflsh_usfs_journal_ev_t *add_journal_ev_head;/* Head of journal events */ + /* to be added for this */ + /* filesystem */ + + cflsh_usfs_journal_ev_t *add_journal_ev_tail;/* Tail of journal events */ + /* to be added for this */ + /* filesystem */ + char device_name[PATH_MAX]; + char *udid; + + pthread_t thread_id; /* Async completion thread id */ + int thread_flags; /* Flags passed to background */ + /* async I/O handler. */ +#define CFLSH_CUFS_EXIT_ASYNC 0x0001 /* Thread should exit */ + pthread_cond_t thread_event; /* Thread event for this file- */ + /* system. */ + struct cflsh_usfs_async_thread_s async_data;/* Data passed o */ + /* async thread handler. */ + + cflsh_usfs_master_hndl_t master_handle; + cusfs_lock_t lock; + cusfs_lock_t free_table_lock; + cusfs_lock_t inode_table_lock; + cusfs_lock_t journal_lock; + + int async_pool_size; /* Size of async I/O poll */ + /* in bytes. */ + int async_pool_num_cmds; /* Number of async I/O */ + /* commands in async_poll. */ + cflsh_usfs_aio_entry_t *async_pool;/* Pool of async I/O */ + /* requests */ + cflsh_usfs_aio_entry_t *head_free;/* Head of free async */ + /* I/O */ + cflsh_usfs_aio_entry_t *tail_free;/* Tail of free async */ + /* I/O */ + cflsh_usfs_aio_entry_t *head_act; /* Head of active async */ + /* I/O */ + cflsh_usfs_aio_entry_t *tail_act; /* Tail of active async */ + /* I/O */ + cflsh_usfs_aio_entry_t *head_cmplt; /* Head of complete */ + /* asyc I/O */ + cflsh_usfs_aio_entry_t *tail_cmplt; /* Tail of complete */ + /* async I/O */ + + struct cflsh_usfs_stat stats; /* Statistics for this fs */ + eye_catch4b_t eyec; /* Eye catcher */ + +} cflsh_usfs_t; + +#ifndef _AIX +#define CFLSH_USFS_HOST_TYPE_FILE "/proc/cpuinfo" + + + +#endif /* ! _AIX */ + +typedef +enum { + CFLSH_USFS_HOST_UNKNOWN = 0, /* Unknown host type */ + CFLSH_USFS_HOST_NV = 1, /* Bare Metal (or No virtualization */ + /* host type. */ + CFLSH_USFS_HOST_PHYP = 2, /* pHyp host type */ + CFLSH_USFS_HOST_KVM = 3, /* KVM host type */ +} cflsh_usfs_host_type_t; + + +/************************************************************************/ +/* cusfs_global - Global library data structure per process */ +/************************************************************************/ + +typedef struct cusfs_global_s { +#ifdef _USE_RW_LOCK + cusfs_rwlock_t global_lock; +#else + cusfs_lock_t global_lock; +#endif + int flags; /* Global flags for this chunk */ +#define CFLSH_G_SYSLOG 0x0001 /* Use syslog for all tracing */ + int next_cusfs_id; /* Id of next allocated */ + /* filesystem. */ + int next_file_id; /* Id of next allocated */ + /* file. */ + + cflsh_usfs_host_type_t host_type; /* Host type */ + cflsh_usfs_os_type_t os_type; /* Current OS type */ + + pid_t caller_pid; /* Process ID of caller of */ + /* this library. */ + + char *process_name; /* Name of process using this */ + /* library if known. */ + int num_active_cufs; /* Number of active filesystems*/ + int num_max_active_cufs; /* Maximum number of active */ + /* filesystems seen at a time. */ + int num_bad_file_ids; /* Number of times we see a */ + /* a bad cufs id. */ + int num_bad_cusfs_ids; /* Number of times we see a */ + /* a bad cufs id. */ + + cflsh_usfs_data_obj_t *file_hash[MAX_NUM_CUSFS_FILE_HASH]; + cflsh_usfs_t *hash[MAX_NUM_CUSFS_HASH]; + + + trace_log_ext_arg_t trace_ext; /* Extended argument for trace */ + uint32_t thread_log_mask; /* Mask used to hash thread */ + /* logs into specific files. */ + cflsh_thread_log_t *thread_logs; /* Array of log files per thread*/ + +#ifdef _SKIP_READ_CALL + int adap_poll_delay;/* Adapter poll delay time in */ + /* microseconds */ +#endif /* _SKIP_READ_CALL */ + eye_catch4b_t eyec; /* Eye catcher */ + + +} cusfs_global_t; + +extern cusfs_global_t cusfs_global; + + +/* Compile time check suitable for use in a function */ +#define CFLSH_USFS_COMPILE_ASSERT(test) \ +do { \ + struct __Fo0 { char v[(test) ? 1 : -1]; } ; \ +} while (0) + + + +#endif /* _H_CFLASH_USFS_INT */ diff --git a/src/usfs/cflsh_usfs_protos.h b/src/usfs/cflsh_usfs_protos.h new file mode 100644 index 00000000..9c8fbce5 --- /dev/null +++ b/src/usfs/cflsh_usfs_protos.h @@ -0,0 +1,297 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* bos720 src/bos/usr/ccs/lib/libcflsh_block/cflsh_usfs_protos.h 1.6 */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/* %Z%%M% %I% %W% %G% %U% */ + + +/* + * COMPONENT_NAME: (sysxcflashufs) CAPI Flash user space file system library + * + * FUNCTIONS: API for users of this library. + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +#ifndef _H_CFLASH_USFS_PROTOS +#define _H_CFLASH_USFS_PROTOS + + +/* cflsh_usfs.c internal only protos */ + + +ssize_t _cusfs_rdwr(int fd, struct aiocb64 *aiocbp, + void *buffer,size_t nbytes,int flags); + +/* cflsh_usfs_utils.c protos */ + +#if !defined(__64BIT__) && defined(_AIX) +time64_t cflsh_usfs_time64(time64_t *tp); + +#endif /* 32-bit AIX */ + +void cusfs_setup_trace_files(int new_process); +int cusfs_valid_endianess(void); +void cusfs_get_host_type(void); +void cusfs_chunk_sigsev_handler (int signum, siginfo_t *siginfo, void *uctx); +void cusfs_prepare_fork (void); +void cusfs_parent_post_fork (void); +void cusfs_child_post_fork (void); +void cusfs_trace_log_data_ext(trace_log_ext_arg_t *ext_arg, + FILE *logfp,char *filename, char *function, + uint line_num,char *msg, ...); + +void cusfs_display_stats(cflsh_usfs_t *cufs, int verbosity); +int cusfs_setup_dump_file(void); +void cusfs_dump_debug_data(const char *reason,const char *reason_filename, + const char *reason_function, + int reason_line_num, const char *reason_date); + +void cusfs_signal_dump_handler(int signum, siginfo_t *siginfo, void *uctx); +int cusfs_setup_sigsev_dump(void); +int cusfs_setup_sigusr1_dump(void); +cflsh_usfs_os_type_t cusfs_get_os_type(void); +int cusfs_get_fs_block_size(cflsh_usfs_t *cufs, int flags); +int cusfs_get_inode_stats(cflsh_usfs_t *cufs,uint64_t *total_inodes, uint64_t *available_inodes, + int flags); +int cusfs_get_inode_table_size(cflsh_usfs_t *cufs, int flags); +cflsh_usfs_t *cusfs_get_cufs(char *device_name, int flags); +void cusfs_release_cufs(cflsh_usfs_t *cufs, int flags); +cflsh_usfs_data_obj_t *cusfs_alloc_file(cflsh_usfs_t *cufs, char *filename, + int flags); +void cusfs_free_file(cflsh_usfs_data_obj_t *file); +int cusfs_get_aio(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + cflsh_usfs_aio_entry_t **aio,int flags); +int cusfs_free_aio(cflsh_usfs_t *cufs, cflsh_usfs_aio_entry_t *aio,int flags); +int cusfs_issue_aio(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, uint64_t start_lba, struct aiocb64 *aiocbp, + void *buffer, size_t nbytes, int flags); +int cusfs_check_active_aio_complete(cflsh_usfs_t *cufs, int flags); +void cusfs_cleanup_all_completed_aiocbs_for_file(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, int flags); +int cusfs_wait_for_aio_for_file(cflsh_usfs_t *cufs, + cflsh_usfs_data_obj_t *file, int flags); +int cusfs_get_status_all_aio_for_aiocb(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + struct aiocb64 *aiocbp, int flags); +int cusfs_return_status_all_aio_for_aiocb(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, struct aiocb64 *aiocbp, + int flags); + +int cusfs_start_aio_thread_for_cufs(cflsh_usfs_t *cufs, int flags); +void cusfs_stop_aio_thread_for_cufs(cflsh_usfs_t *cufs, int flags); +void *cusfs_aio_complete_thread(void *data); + +cflsh_usfs_t *cusfs_access_fs(char *device_name, int flags); +void cusfs_add_file_to_hash(cflsh_usfs_data_obj_t *file, int flags); +void cusfs_remove_file_from_hash(cflsh_usfs_data_obj_t *file, int flags); +cflsh_usfs_data_obj_t *cusfs_find_data_obj_from_inode(cflsh_usfs_t *cufs, uint64_t inode_num, int flags); + +cflsh_usfs_data_obj_t *cusfs_find_data_obj_from_filename(cflsh_usfs_t *cufs, char *filename, int flags); + +int cusfs_rdwr_data_object(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, void *buf, int *buf_size, int flags); + +int cusfs_read_data_object(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, void *buf, int *buf_size, int flags); + +int cusfs_read_data_object_for_inode_no(cflsh_usfs_t *cufs, uint64_t inode_num, + void **buf, int *buf_size, int flags); + +int cusfs_write_data_object(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, void *buf, int *buf_size, int flags); +cflsh_usfs_data_obj_t *cusfs_lookup_data_obj(cflsh_usfs_t *cufs, + char *pathname, int flags); +char *cusfs_get_only_filename_from_diskpath(char *disk_pathname, int flags); +cflsh_usfs_t *cusfs_find_fs_for_disk(char *device_name, int flags); +cflsh_usfs_t *cusfs_find_fs_from_path(char *disk_pathname, int flags); +cflsh_usfs_data_obj_t *cusfs_open_disk_filename(char *disk_pathname, int flags); +int cusfs_flush_inode_timestamps(cflsh_usfs_t *cusfs, cflsh_usfs_data_obj_t *file); + +int cusfs_get_disk_lbas_for_transfer(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + uint64_t offset, size_t num_bytes, + cflsh_usfs_disk_lba_e_t **disk_lbas, uint64_t *num_disk_lbas, + int flags); +void cusfs_fork_manager(void); + +void cusfs_aio_cleanup(cflsh_usfs_t *cufs); + +void cusfs_seek_read_ahead(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + uint64_t start_offset, int flags); + +/* cflsh_usfs_disk.c protos */ +cflsh_usfs_fs_id_t cusfs_generate_fs_unique_id(void); +int cusfs_get_disk_block_no_from_fs_block_no(cflsh_usfs_t *cufs, uint64_t fs_block_no, uint64_t *disk_block_no); +uint64_t cusfs_get_free_block_table_size(cflsh_usfs_t *cufs, + uint64_t num_blocks, + int flags); +int cusfs_create_superblock(cflsh_usfs_t *cufs, int flags); +int cusfs_update_superblock(cflsh_usfs_t *cufs, int flags); +int cusfs_remove_superblock(cflsh_usfs_t *cufs, int flags); +int cusfs_validate_superblock(cflsh_usfs_t *cufs, int flags); +int cusfs_query_superblock(cflsh_usfs_t *cufs, cflsh_usfs_super_block_t *superblock,int flags); +int cusfs_create_free_block_table(cflsh_usfs_t *cufs, int flags); +int cusfs_create_inode_table(cflsh_usfs_t *cufs, int flags); +int cusfs_create_journal(cflsh_usfs_t *cufs, int flags); +int cusfs_add_journal_event(cflsh_usfs_t *cufs, + cflsh_journal_entry_type_t journal_type, + uint64_t block_no, uint64_t inode_num, + uint64_t old_file_size, uint64_t new_file_size, + uint64_t directory_inode_num, + int flags); +int cusfs_flush_journal_events(cflsh_usfs_t *cufs, int flags); +int cusfs_get_free_table_stats(cflsh_usfs_t *cufs,uint64_t *total_blocks, uint64_t *available_blocks, + int flags); +uint64_t cusfs_get_data_blocks(cflsh_usfs_t *cufs, int *num_blocks,int flags); +int cusfs_get_data_blocks_list(cflsh_usfs_t *cufs, uint64_t *num_blocks, uint64_t **block_list, int flags); +int cusfs_release_data_blocks(cflsh_usfs_t *cufs, uint64_t lba,int num_blocks,int flags); +int cusfs_add_directory_entry(cflsh_usfs_t *cufs,uint64_t directory_inode_num,cflsh_usfs_inode_t *inode, + char *filename,int flags); +cflsh_usfs_inode_t *cusfs_find_inode_num_from_directory_by_name(cflsh_usfs_t *cufs, + char *name,char *buf, + int buf_size,int flags); +int cusfs_remove_directory_entry(cflsh_usfs_t *cufs,cflsh_usfs_inode_t *inode, + char *filename,int flags); +int cusfs_check_directory_empty(cflsh_usfs_t *cufs,cflsh_usfs_inode_t *inode, + char *filename,int flags); +int cusfs_list_files_directory(cflsh_usfs_t *cufs,cflsh_usfs_inode_t *inode, + char *filename,int flags); +int cusfs_get_next_directory_entry(cflsh_usfs_t *cufs,cflsh_usfs_data_obj_t *directory, + cflsh_usfs_directory_entry_t *return_dir_entry, cflash_offset_t *offset, + int flags); +int cusfs_initialize_directory(cflsh_usfs_t *cufs,cflsh_usfs_inode_t *inode,int flags); +int cusfs_clear_data_obj(cflsh_usfs_t *cufs,cflsh_usfs_inode_t *inode,int flags); + +cflsh_usfs_data_obj_t *cusfs_create_data_obj(cflsh_usfs_t *cufs, + char *filename, + cflsh_usfs_inode_file_type_t inode_type, + uint64_t parent_directory_inode_num, + mode_t mode_flags, uid_t uid, gid_t gid, + char *sym_link_name, + int flags); + +int cusfs_free_data_obj(cflsh_usfs_t *cufs,cflsh_usfs_data_obj_t *file,int flags); +int cusfs_mv_data_obj(cflsh_usfs_t *cufs,cflsh_usfs_data_obj_t *file,char *new_full_pathname,int flags); + +cflsh_usfs_inode_t *cusfs_find_root_inode(cflsh_usfs_t *cufs, int flags); + + +/* cflsh_usfs_inode.c */ + +cflsh_usfs_inode_t *cusfs_get_inode(cflsh_usfs_t *cufs, uint64_t parent_directory_inode_num, + cflsh_usfs_inode_file_type_t inode_type, + mode_t mode_flags, uid_t uid, gid_t gid, + uint64_t blocks[], int num_blocks, uint64_t file_size, + char *sym_link_name, + int flags); + +int cusfs_release_inode(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode,int flags); +int cusfs_release_inode_and_data_blocks(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode,int flags); +cflsh_usfs_inode_t *cusfs_get_inode_from_index(cflsh_usfs_t *cufs, uint64_t inode_num,int flags); +int cusfs_update_inode(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode,int flags); + +uint64_t cusfs_get_indirect_inode_ptr(cflsh_usfs_t *cufs, + uint64_t blocks[], int *num_blocks, + int flags); +int cusfs_update_indirect_inode_ptr(cflsh_usfs_t *cufs, + uint64_t inode_ptr_block_no, + uint64_t blocks[], int *num_blocks, + int flags); +int cusfs_add_blocks_to_indirect_inode_ptr(cflsh_usfs_t *cufs, + uint64_t inode_ptr_block_no, + uint64_t blocks[], int *num_blocks, + int flags); +int cusfs_remove_indirect_inode_ptr(cflsh_usfs_t *cufs, + uint64_t inode_ptr_block_no, + int flags); + +int cusfs_get_inode_block_no_from_inode_index(cflsh_usfs_t *cufs, uint64_t inode_index,uint64_t *lba); +int cusfs_get_inode_ptr_from_file_offset(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + uint64_t offset, int *indirect_inode_ptr, int flags); +int cusfs_rdwr_block_inode_pointers(cflsh_usfs_t *cufs, + cflsh_usfs_inode_t *inode, + void *buffer, + uint64_t fs_lba, int nblocks, int flags); + +int cusfs_get_disk_lbas_from_inode_ptr_for_transfer(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t offset, size_t num_bytes, + cflsh_usfs_disk_lba_e_t **disk_lbas, uint64_t *num_disk_lbas, + int flags); +int cusfs_get_all_disk_lbas_from_inode(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + cflsh_usfs_disk_lba_e_t **disk_lbas, uint64_t *num_disk_lbas, + int flags); +int cusfs_get_inode_ptr_from_for_transfer(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + uint64_t offset, size_t num_bytes, + int *start_inode_ptr_index, + int *start_inode_ptr_byte_offset, + int *end_inode_ptr_index, + int *end_inode_ptr_byte_offset, + int *num_inode_ptr_indices, + int flags); +int cusfs_find_disk_lbas_in_inode_ptr_for_transfer(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + int inode_ptr_index, + uint64_t *start_disk_lba, + uint64_t *num_disk_blocks, + int start_inode_ptr, + int end_inode_ptr, + int transfer_start_offset, + int transfer_end_offset, + int flags); +int cusfs_num_data_blocks_single_indirect_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t single_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags); +int cusfs_num_data_blocks_double_indirect_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t double_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags); +int cusfs_num_data_blocks_triple_indirect_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t triple_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags); +int cusfs_num_data_blocks_in_inode(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t *num_fs_blocks, int *level,int flags); +uint64_t cusfs_get_next_inode_data_block(cflsh_usfs_t *cufs, uint64_t **block_list, + uint64_t *block_list_index,uint64_t num_needed_blocks, int flags); +int cusfs_add_fs_blocks_to_single_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t single_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags); +int cusfs_add_fs_blocks_to_double_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t double_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags); +int cusfs_add_fs_blocks_to_triple_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t triple_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags); +int cusfs_add_fs_blocks_to_inode_ptrs(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t num_fs_lbas,int flags); +int cusfs_remove_fs_blocks_from_single_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t single_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags); +int cusfs_remove_fs_blocks_from_double_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t double_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags); +int cusfs_remove_fs_blocks_from_triple_inode_ptr(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t triple_indirect_inode_fs_lba, + uint64_t *num_fs_blocks, int flags); +int cusfs_remove_fs_blocks_from_inode_ptrs(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, + uint64_t num_fs_lbas,int flags); +#endif /* _H_CFLASH_USFS_PROTOS */ diff --git a/src/usfs/cflsh_usfs_utils.c b/src/usfs/cflsh_usfs_utils.c new file mode 100644 index 00000000..def5679c --- /dev/null +++ b/src/usfs/cflsh_usfs_utils.c @@ -0,0 +1,5890 @@ +// IBM_PROLOG_BEGIN_TAG +// This is an automatically generated prolog. +// +// $Source: src/usfs/cflsh_usfs_utils.c $ +// +// IBM CONFIDENTIAL +// +// COPYRIGHT International Business Machines Corp. 2016 +// +// p1 +// +// Object Code Only (OCO) source materials +// Licensed Internal Code Source Materials +// IBM Surelock Licensed Internal Code +// +// The source code for this program is not published or other- +// wise divested of its trade secrets, irrespective of what has +// been deposited with the U.S. Copyright Office. +// +// Origin: 30 +// +// IBM_PROLOG_END_TAG */ +#ifdef _AIX +static char sccsid[] = "%Z%%M% %I% %W% %G% %U%"; +#endif + + + +/* + * COMPONENT_NAME: (sysxcflashusfs) CAPI Flash user space filesystem library + * + * FUNCTIONS: + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +#define CFLSH_USFS_FILENUM 0x0300 + + +#include "cflsh_usfs_internal.h" +#include "cflsh_usfs_protos.h" + + +#ifdef _AIX +#include + +#if !defined(__64BIT__) +#include +#endif +#endif + +char cusfs_filename[PATH_MAX]; + +extern void cflsh_usfs_init(void); + +#if !defined(__64BIT__) && defined(_AIX) +static time64_t (*real_time64)(time64_t *tp) = NULL; + + +/* + * NAME: time64 + * + * FUNCTION: For some levels of AIX, time64 is not + * available for 32-bit kernel. This + * function tries to determine if time64 + * is available. If so it will use it. Otherwise + * it will use time. + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +time64_t cflsh_usfs_time64(time64_t *tp) +{ + time64_t rc; + time_t *tp32; + + /* + * See if time64 is available on this system + */ + + real_time64 = (time64_t(*)(time64_t *))dlsym(RTLD_DEFAULT,"time64"); + + if (real_time64) { + + /* + * This system has time64 for 32-bit applications. + */ + rc = real_time64(tp); + + } else { + + /* + * This system does not have time64 for 32-bit applications. + */ + + if (tp) { + + tp32 = (time_t *)tp; + + /* + * Move to low order word of tp; + */ + + tp32++; + + rc = time(tp32); + + + } else { + rc = time(NULL); + } + + } + + return rc; + +} + +#endif /* 32-bit AIX */ + +/* ---------------------------------------------------------------------------- + * + * NAME: cusfs_setup_trace_files + * + * FUNCTION: Set up trace files + * + * + * + * + * + * RETURNS: NONE + * + * ---------------------------------------------------------------------------- + */ +void cusfs_setup_trace_files(int new_process) +{ + int i; + char *env_verbosity = getenv("CFLSH_USFS_TRC_VERBOSITY"); + char *env_use_syslog = getenv("CFLSH_USFS_TRC_SYSLOG"); + char *env_trace_append = getenv("CFLSH_USFS_TRC_APPEND"); + char *log_pid = getenv("CFLSH_USFS_TRACE_PID"); + char *env_num_thread_logs = getenv("CFLSH_USFS_TRACE_TID"); + char *env_user = getenv("USER"); + uint32_t thread_logs = 0; + char filename[PATH_MAX]; + char *filename_ptr = filename; + int open_flags = 0; + + +#ifndef _AIX + cusfs_global.flags |= CFLSH_G_SYSLOG; + + if (env_use_syslog) { + + if (strcmp(env_use_syslog,"ON")) { + + /* + * Don't use syslog tracing. Instead + * use tracing to a file. + */ + cusfs_global.flags &= ~CFLSH_G_SYSLOG; + + + /* + * By default for linux, enable appending of + * of log file if someone turns off the default + * of syslog. + */ + + open_flags |= TRACE_LOG_EXT_OPEN_APPEND_FLG; + } + } +#endif /* !_AIX */ + + + + + if (new_process) { + + if ((log_pid == NULL) || + (cusfs_global.flags & CFLSH_G_SYSLOG)) { + + /* + * If this is a new process (forked process) + * and we are not using traces per process, + * or we are logging via syslog + * then continue to the use the tracing + * in place for the parent process. + */ + + return; + } + + strcpy(filename,cusfs_log_filename); + } + + if (env_trace_append) { + + + if (!strcmp(env_trace_append,"ON")) { + + /* + * Append to existing trace file. The default + * it to truncate and overwrite. + */ + open_flags |= TRACE_LOG_EXT_OPEN_APPEND_FLG; + } else { + + open_flags &= ~TRACE_LOG_EXT_OPEN_APPEND_FLG; + } + + } + + if (env_verbosity) { + cusfs_log_verbosity = atoi(env_verbosity); + + } else { +#ifdef _AIX + cusfs_log_verbosity = 0; +#else + cusfs_log_verbosity = 1; +#endif + } + + cusfs_log_filename = getenv("CFLSH_USFS_TRACE"); + if (cusfs_log_filename == NULL) + { + sprintf(cusfs_filename, "/tmp/%s.cflsh_usfs_trc", env_user); + cusfs_log_filename = cusfs_filename; + } + + if ((log_pid) && !(cusfs_global.flags & CFLSH_G_SYSLOG)) { + + /* + * Use different filename for each process, when + * not using syslogging. + */ + + sprintf(cusfs_filename,"%s.%d",cusfs_log_filename,getpid()); + + if ((new_process) && + !strcmp(cusfs_log_filename,filename)) { + + /* + * If this is a new process (forked process) + * and the process trace filename is same as before, + * then return here, since we are already set up. + * This situation can occur if there are multiple chunks + * that are cloned after a fork. Only the first + * one would change the trace file. + */ + + return; + } + + cusfs_log_filename = cusfs_filename; + + } + + bzero((void *)&(cusfs_global.trace_ext),sizeof(trace_log_ext_arg_t)); + + /* + * We need to serialize access to this log file + * while we are setting it up. + */ + + pthread_mutex_lock(&cusfs_log_lock); + + if (cusfs_global.flags & CFLSH_G_SYSLOG) { + + openlog("CXLBLK",LOG_PID,LOG_USER); + + + } else if (cusfs_log_verbosity) { + + if (setup_trace_log_file_ext(&cusfs_log_filename,&cusfs_logfp,cusfs_log_filename,open_flags)) { + + //fprintf(stderr,"Failed to set up tracing for filename = %s\n",cusfs_log_filename); + + /* + * Turn off tracing if this fails. + */ + cusfs_log_verbosity = 0; + } + } + + + + if ((env_num_thread_logs) && !(cusfs_global.flags & CFLSH_G_SYSLOG)) { + + /* + * This indicates they want a trace log file per thread + * and we are not using syslog. + * We will still trace all threads in one common file, + * but also provide a thread log per thread too. + */ + + if ((new_process) && (num_thread_logs)) { + + /* + * If this is a new process (i.e. forked + * process), then we need to free up + * the resources from parent first. + */ + + free(cusfs_global.thread_logs); + } + + num_thread_logs = atoi(env_num_thread_logs); + + num_thread_logs = MIN(num_thread_logs, MAX_NUM_THREAD_LOGS); + + if (num_thread_logs) { + + /* + * Allocate there array of thread_log file pointers: + */ + + cusfs_global.thread_logs = (cflsh_thread_log_t *) malloc(num_thread_logs * sizeof(cflsh_thread_log_t)); + + if (cusfs_global.thread_logs) { + + bzero((void *)cusfs_global.thread_logs, num_thread_logs * sizeof(cflsh_thread_log_t)); + + + for (i=0; i< num_thread_logs;i++) { + + sprintf(filename,"%s.%d",cusfs_log_filename,i); + + + if (setup_trace_log_file(&filename_ptr,&cusfs_global.thread_logs[i].logfp,filename)) { + + fprintf(stderr,"Failed to set up tracing for filename = %s\n",filename); + free(cusfs_global.thread_logs); + + num_thread_logs = 0; + break; + } + + cusfs_global.thread_logs[i]. ext_arg.flags |= TRACE_LOG_NO_USE_LOG_NUM; + + } /* for */ + + /* + * We need to create a mask to allow us to hash + * thread ids into the our various thread log files. + * Thus we need mask that is based on the number_thread_log + * files. We'll create a mask that is contains a 1 for + * every bit up to the highest bit used to represent the number + * thread log files. + */ + + thread_logs = num_thread_logs; + cusfs_global.thread_log_mask = 0; + + while (thread_logs) { + + cusfs_global.thread_log_mask = (cusfs_global.thread_log_mask << 1) | 1; + thread_logs >>= 1; + + } /* while */ + + } else { + + /* + * If we fail to allocate the thread trace log, then + * set num_thread_logs back to 0. + */ + num_thread_logs = 0; + } + } + + } + + pthread_mutex_unlock(&cusfs_log_lock); + + + return; +} + + +/* ---------------------------------------------------------------------------- + * + * NAME: cusfs_valid_endianess + * + * FUNCTION: Determines the Endianess of the host that + * the binary is running on. + * + * + * + * CALLED BY: + * + * + * INTERNAL PROCEDURES CALLED: + * + * + * + * EXTERNAL PROCEDURES CALLED: + * + * + * + * RETURNS: 1 Host endianess matches compile flags + * 0 Host endianess is invalid based on compile flags + * + * ---------------------------------------------------------------------------- + */ +int cusfs_valid_endianess(void) +{ + int rc = FALSE; + short test_endian = 0x0102; + char *ptr; + char byte; + + ptr = (char *) &test_endian; + + byte = ptr[0]; + + if (byte == 0x02) { + + /* + * In a Little Endian host, the first indexed + * byte will be 0x2 + */ +#ifdef CFLASH_LITTLE_ENDIAN_HOST + rc = TRUE; +#else + rc = FALSE; +#endif /* !CFLASH_LITTLE_ENDIAN_HOST */ + + + } else { + + /* + * In a Big Endian host, the first indexed + * byte will be 0x1 + */ + +#ifdef CFLASH_LITTLE_ENDIAN_HOST + rc = FALSE; +#else + rc = TRUE; +#endif /* !CFLASH_LITTLE_ENDIAN_HOST */ + + + } + + + + return rc; +} + +/* ---------------------------------------------------------------------------- + * + * NAME: cusfs_get_host_type + * + * FUNCTION: Determine if this OS is running on Bare Metal, pHyp + * or PowerKVM. + * + * + * + * CALLED BY: + * + * + * INTERNAL PROCEDURES CALLED: + * + * + * + * EXTERNAL PROCEDURES CALLED: + * + * + * + * RETURNS: + * + * ---------------------------------------------------------------------------- + */ + + +void cusfs_get_host_type(void) +{ +#ifdef _AIX + + /* + * For AIX always assume pHyp platform type + */ + + cusfs_global.host_type = CFLSH_USFS_HOST_PHYP; + return; + +#else +#define CFLSH_USFS_HOST_BUF_SIZE 256 + FILE *fp; + char buf[CFLSH_USFS_HOST_BUF_SIZE]; + char *platform_line; + + /* + * Set host type to unknown until we had a chance + * to deterine otherwise. + */ + + cusfs_global.host_type = CFLSH_USFS_HOST_UNKNOWN; + + if ((fp = fopen(CFLSH_USFS_HOST_TYPE_FILE,"r")) == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Can not determine host type, failed to open = %s", + CFLSH_USFS_HOST_TYPE_FILE); + return; + } + + + + while (fgets(buf,CFLSH_USFS_HOST_BUF_SIZE,fp)) { + + + /* + * The /proc/cpuinfo file will have a line + * that starts with the format: + * + * platform : + * + * after the ":" will be the platform/host type + * + * So we first search for platform, then extract out + * platform/host type. + */ + + platform_line = strstr(buf,"platform"); + + if ((platform_line) && + (strstr(platform_line,":")) ) { + + CUSFS_TRACE_LOG_FILE(9,"Platform_line = %s",platform_line); + + + /* + * NOTE: The order below is import, because two of the + * platform strings start with pSeries. + */ + + if (strstr(platform_line,"PowerNV")) { + + CUSFS_TRACE_LOG_FILE(9,"BML host type"); + cusfs_global.host_type = CFLSH_USFS_HOST_NV; + + break; + } else if (strstr(platform_line,"pSeries (emulated by qemu)")) { + + CUSFS_TRACE_LOG_FILE(9,"PowerKVM host type"); + cusfs_global.host_type = CFLSH_USFS_HOST_KVM; + + } else if (strstr(platform_line,"pSeries")) { + + CUSFS_TRACE_LOG_FILE(9,"pHyp host type"); + cusfs_global.host_type = CFLSH_USFS_HOST_PHYP; + + } else { + CUSFS_TRACE_LOG_FILE(1,"Unknown platform string= %s", + platform_line); + } + + break; + + + } + + } + + if (cusfs_global.host_type == CFLSH_USFS_HOST_UNKNOWN) { + + CUSFS_TRACE_LOG_FILE(9,"could not determine host type"); + } + + + fclose (fp); + + return; +#endif +} + +/* ---------------------------------------------------------------------------- + * + * NAME: cusfs_chunk_sigsev_handler + * + * FUNCTION: Since a failing CAPI adapter, can generate SIGSEV + * for a now invalid MMIO address, let us collect some + * debug information here in this SIGSEGV hanndler + * to determine this. + * + * + * + * + * + * RETURNS: NONE + * + * ---------------------------------------------------------------------------- + */ +void cusfs_chunk_sigsev_handler (int signum, siginfo_t *siginfo, void *uctx) +{ + int i; + cflsh_usfs_t *cufs; + + CUSFS_TRACE_LOG_FILE(1,"si_code = %d, si_addr = 0x%p", + siginfo->si_code,siginfo->si_addr); + + switch (siginfo->si_code) { +#ifdef _AIX + case SEGV_MAPERR: + CUSFS_TRACE_LOG_FILE(1,"Address not mapped, address = 0x%p", + siginfo->si_addr); + + break; +#endif /* _AIX */ + case SEGV_ACCERR: + + CUSFS_TRACE_LOG_FILE(1,"Invalid permissions, address = 0x%p", + siginfo->si_addr); + + break; + default: + + CUSFS_TRACE_LOG_FILE(1,"Unknown si_code = %d, address = 0x%p", + siginfo->si_code,siginfo->si_addr); + } + + + for (i=0; i < MAX_NUM_CUSFS_HASH; i++) { + + cufs = cusfs_global.hash[i]; + + + while (cufs) { +#ifdef _REMOVE + + for (j=0; j < chunk->num_paths;j++) { + if ((chunk->flags & CFLSH_CHNK_SIGH) && + (chunk->path[j]) &&) { + + longjmp(chunk->path[j]->jmp_mmio,1); + } + + } /* for */ +#endif /* _REMOVE */ + cufs = cufs->next; + + } /* while */ + + + } /* for */ + + + /* + * If we get here then SIGSEGV is mostly + * likely not associated with a bad MMIO + * address (due to adapter reset or + * UE. Issue default signal. + */ + + signal(signum,SIG_DFL); + kill(getpid(),signum); + + return; +} + + +/* ---------------------------------------------------------------------------- + * + * NAME: cusfs_prepare_fork + * + * FUNCTION: If a process using this library does a fork, then + * this routine will be invoked + * prior to fork to the library into a consistent state + * that will be preserved across fork. + * + * + * + * + * + * RETURNS: NONE + * + * ---------------------------------------------------------------------------- + */ +void cusfs_prepare_fork (void) +{ + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_data_obj_t *file = NULL; + int i; + + + /* + * ?? TODO: Is anything needed here for semop + * or socket locks? + */ + + pthread_mutex_lock(&cusfs_log_lock); + + CUSFS_WR_RWLOCK(cusfs_global.global_lock); + + + for (i=0; i < MAX_NUM_CUSFS_HASH; i++) { + + cufs = cusfs_global.hash[i]; + + + while (cufs) { + + CUSFS_LOCK(cufs->lock); + CUSFS_LOCK(cufs->free_table_lock); + CUSFS_LOCK(cufs->inode_table_lock); + CUSFS_LOCK(cufs->journal_lock); + + + file = cufs->filelist_head; + + while (file) { + + CUSFS_LOCK(file->lock); + + file = file->next; + } + + + cufs = cufs->next; + + } /* while */ + + + } /* for */ + + + return; +} + + +/* ---------------------------------------------------------------------------- + * + * NAME: cusfs_parent_post_fork + * + * FUNCTION: If a process using this library does a fork, then + * this routine will be run on the parent after fork + * to release locks. + * + * + * + * + * + * RETURNS: NONE + * + * ---------------------------------------------------------------------------- + */ +void cusfs_parent_post_fork (void) +{ + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_data_obj_t *file = NULL; + int i; + int rc; + + + + + /* + * ?? TODO: Is anything needed here for semop + * or socket locks? + */ + + for (i=0; i < MAX_NUM_CUSFS_HASH; i++) { + + cufs = cusfs_global.hash[i]; + + + while (cufs) { + + CUSFS_UNLOCK(cufs->lock); + CUSFS_UNLOCK(cufs->free_table_lock); + CUSFS_UNLOCK(cufs->inode_table_lock); + CUSFS_UNLOCK(cufs->journal_lock); + + + file = cufs->filelist_head; + + while (file) { + + CUSFS_UNLOCK(file->lock); + + file = file->next; + } + + cufs = cufs->next; + + } /* while */ + + + } /* for */ + + + + rc = pthread_mutex_unlock(&cusfs_log_lock); + + if (rc) { + + /* + * Find the first chunk do a notify + * against it. + */ + + for (i=0; i < MAX_NUM_CUSFS_HASH; i++) { + + cufs = cusfs_global.hash[i]; + + + if (cufs) { + + break; + } + + + } /* for */ + +#ifdef _REMOVE + if (cufs) { + +//?? cusfs_notify_mc_err(cufs,0,0x205,rc, CUSFS_GLOBAL_NOTIFY_SFW_ERR,NULL); + + } +#endif /* _REMOVE */ + } + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + return; +} + +/* ---------------------------------------------------------------------------- + * + * NAME: cusfs_aio_cleanup() + * + * FUNCTION: If a process using this library does a fork, then + * this routine will be run on the child after fork + * to clean up all aio related threads and resources + * + * RETURNS: NONE + * + * ---------------------------------------------------------------------------- + */ + +void cusfs_aio_cleanup ( cflsh_usfs_t *cufs ) +{ + + cflsh_usfs_aio_entry_t *aio; + cflsh_usfs_aio_entry_t *next_aio; + cflsh_usfs_data_obj_t *file; + + aio = cufs->head_act; + + while (aio) + { + next_aio = aio->act_next; + + file = aio->file; + + /* caller of cusfs_free_aio() should have file lock */ + + if (file) + CUSFS_LOCK(file->lock); + + if( cusfs_free_aio(cufs,aio,0)) + { + cufs->flags = ECHILD; + + if (file) + CUSFS_UNLOCK(file->lock); + break; + } + + if (file) + CUSFS_UNLOCK(file->lock); + + aio = next_aio; + } + + CUSFS_TRACE_LOG_FILE(9,"aio cleanup called for disk %s, flags %d",cufs->device_name,cufs->flags); + + return ; + +} +/* ---------------------------------------------------------------------------- + * + * NAME: cusfs_fork_manager + * + * FUNCTION: If a process using this library does a fork, then + * this routine will be run on the child after fork + * to manage the fork() semantics. + * + * RETURNS: NONE + * + * ---------------------------------------------------------------------------- + */ + +void cusfs_fork_manager(void) +{ + int i; +#ifdef _REMOVE + chunk_ext_arg_t ext = NULL; +#endif /* _REMOVE */ + cflsh_usfs_t *cufs = NULL; + + for (i=0; i < MAX_NUM_CUSFS_HASH; i++) + { + cufs = cusfs_global.hash[i]; + + while (cufs) + { + CUSFS_LOCK(cufs->lock); + + cufs->stats.num_success_threads = 0; + + cufs->stats.num_active_threads = 0; + +#ifdef _REMOVE + cufs->chunk_id = cblk_open(cufs->device_name,cufs->max_requests, + O_RDWR,ext,0); + if (cufs->chunk_id == NULL_CHUNK_ID) + { + CUSFS_TRACE_LOG_FILE(1,"cblk_open failed for device= %s",cufs->device_name); + } + +#endif /* REMOVE */ + if (cufs->async_pool_num_cmds) + { + cusfs_aio_cleanup(cufs); + } + + if (cusfs_start_aio_thread_for_cufs(cufs,0)) + { + cufs->flags |= CFLSH_USFS_FLG_FAIL_FORK; + CUSFS_TRACE_LOG_FILE(1,"aio_thread fail to start for disk %s",cufs->device_name); + + } + + CUSFS_UNLOCK(cufs->lock); + + cufs = cufs->next; + } + } + + return ; +} + +/* ---------------------------------------------------------------------------- + * + * NAME: cusfs_child_post_fork + * + * FUNCTION: If a process using this library does a fork, then + * this routine will be run on the child after fork + * to release locks. + * + * + * + * + * + * RETURNS: NONE + * + * ---------------------------------------------------------------------------- + */ +void cusfs_child_post_fork (void) +{ + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_data_obj_t *file = NULL; + int i; + int rc; + + + + /* + * ?? TODO: Is anything needed here for semop + * or socket locks? + */ + + for (i=0; i < MAX_NUM_CUSFS_HASH; i++) { + + cufs = cusfs_global.hash[i]; + + + while (cufs) { + + CUSFS_UNLOCK(cufs->lock); + CUSFS_UNLOCK(cufs->free_table_lock); + CUSFS_UNLOCK(cufs->inode_table_lock); + CUSFS_UNLOCK(cufs->journal_lock); + + + file = cufs->filelist_head; + + while (file) { + + CUSFS_UNLOCK(file->lock); + + file = file->next; + } + + + cufs = cufs->next; + + } /* while */ + + + } /* for */ + + + + rc = pthread_mutex_unlock(&cusfs_log_lock); + + if (rc) { + + + /* + * Find the first chunk do a notify + * against it. + */ + + for (i=0; i < MAX_NUM_CUSFS_HASH; i++) { + + cufs = cusfs_global.hash[i]; + + + if (cufs) { + + break; + } + + + } /* for */ +#ifdef _REMOVE + if (cufs) { + +//?? cusfs_notify_mc_err(cufs,0,0x206,rc, CUSFS_GLOBAL_NOTIFY_SFW_ERR,NULL); + + } +#endif /* _REMOVE */ + } else { + + cusfs_global.caller_pid = getpid(); + + /* + * Since we forked the child, if we have tracing turned + * on for a trace file per process, then we need to + * open the new file for this child's PID. The routine + * cblk_setup_trace_files will handle the situation + * where multiple chunks are cloned and using the same + * new trace file. + */ + + cusfs_setup_trace_files(TRUE); + + CUSFS_TRACE_LOG_FILE(9,"PID = 0x%"PRIx64"",(uint64_t)cusfs_global.caller_pid); + + /* + * We have forked the child and we need to re-open + * device using cblk_open and get the chunk id again. + * Child process does not inherit any pending or + * on-going asynchronous IO openration fron its parent + * process; aio active comamnd will be cleaned up and + * aio handler thread will re-started again for child + */ + cusfs_fork_manager(); + + + } + + + CUSFS_RWUNLOCK(cusfs_global.global_lock); + + return; +} + + + + +/* ---------------------------------------------------------------------------- + * + * NAME: cusfs_trace_log_data_ext + * + * FUNCTION: This is a function call (as opposed to inlined) version + * of trace_log_data_ext from trace_log.h. It uses + * the same setup (setup_log_file in trace_log.h) and defines. + * + * Print a message to trace log. This + * function is the same as trace_log_data, except + * this function requires the caller to maintain + * the static variables via the extended argument. In addition + * it gives the caller additional control over logging. + * + * + * + * CALLED BY: + * + * + * INTERNAL PROCEDURES CALLED: + * + * + * + * EXTERNAL PROCEDURES CALLED: + * + * + * + * RETURNS: + * + * ---------------------------------------------------------------------------- + */ +void cusfs_trace_log_data_ext(trace_log_ext_arg_t *ext_arg, FILE *logfp,char *filename, char *function, + uint line_num,char *msg, ...) +{ + va_list ap; + struct timeb cur_time, log_time, delta_time; + uint print_log_number; + char timebuf[TIMELEN+1]; +#if !defined(__64BIT__) && defined(_AIX) + time64_t curtime; +#else + time_t curtime; +#endif /* not 32-bit AIX */ + + if (ext_arg == NULL) { + + return; + } + + if (logfp == NULL) { + + return; + } + + + if (ext_arg->flags & TRACE_LOG_NO_USE_LOG_NUM) { + + if (ext_arg->log_number > 0) { + print_log_number = ext_arg->log_number - 1; + } else { + print_log_number = 0; + } + + } else { + print_log_number = ext_arg->log_number; + } + + ftime(&cur_time); + + if (!(ext_arg->flags & TRACE_LOG_START_VALID)) { + + /* + * If start time is not set, then + * set it now. + */ + + ext_arg->start_time = cur_time; + + + ext_arg->flags |= TRACE_LOG_START_VALID; + + + log_time.time = 0; + log_time.millitm = 0; + + delta_time.time = 0; + delta_time.millitm = 0; + + + /* + * Print header + */ + fprintf(logfp,"---------------------------------------------------------------------------\n"); + fprintf(logfp,"Date for %s is %s at %s\n",__FILE__,__DATE__,__TIME__); + +#if !defined(__64BIT__) && defined(_AIX) + curtime = cflsh_usfs_time64(NULL); + fprintf(logfp,"Trace started at %s\n",ctime64_r(&curtime,timebuf)); +#else + curtime = time(NULL); + fprintf(logfp,"Trace started at %s\n",ctime_r(&curtime,timebuf)); +#endif /* not 32-bit AIX */ + + fprintf(logfp,"Index Sec msec delta dmsec Filename function, line ...\n"); + fprintf(logfp,"------- ----- ----- ----- ----- -------------------- ---------------------\n"); + + } else { + + /* + * Find time offset since starting time. + */ + + log_time.time = cur_time.time - ext_arg->start_time.time; + log_time.millitm = cur_time.millitm - ext_arg->start_time.millitm; + + delta_time.time = log_time.time - ext_arg->last_time.time; + delta_time.millitm = log_time.millitm - ext_arg->last_time.millitm; + } + + fprintf(logfp,"%7d %5d.%05d %5d.%05d %-25s %-35s line:%5d :", + print_log_number,(int)log_time.time,log_time.millitm,(int)delta_time.time,delta_time.millitm,filename, function, line_num); + /* + * Initialize ap to store arguments after msg + */ + + va_start(ap,msg); + vfprintf(logfp, msg, ap); + va_end(ap); + + fprintf(logfp,"\n"); + + fflush(logfp); + + if (!(ext_arg->flags & TRACE_LOG_NO_USE_LOG_NUM)) { + ext_arg->log_number++; + } + + ext_arg->last_time = log_time; + + return; + +} + + +/* + * NAME: cusfs_display_stats + * + * FUNCTION: This routine is called whenever an RRQ has been processed. + * + * + * NOTE; This routine assumes the caller is holding chunk->lock. + * + * RETURNS: None + * + * + */ + +void cusfs_display_stats(cflsh_usfs_t *cufs, int verbosity) +{ + CUSFS_TRACE_LOG_FILE(verbosity,"\nCUFS STATISTICS ..."); + + +#ifdef CFLASH_LITTLE_ENDIAN_HOST + CUSFS_TRACE_LOG_FILE(verbosity,"Little Endian"); +#else + CUSFS_TRACE_LOG_FILE(verbosity,"Big Endian"); +#endif + CUSFS_TRACE_LOG_FILE(verbosity,"cusfs_log_verbosity 0x%x",cusfs_log_verbosity); +#if !defined(__64BIT__) && defined(_AIX) + CUSFS_TRACE_LOG_FILE(verbosity,"32-bit app support "); +#else + CUSFS_TRACE_LOG_FILE(verbosity,"64-bit app support "); + +#endif + CUSFS_TRACE_LOG_FILE(verbosity,"flags 0x%x",cusfs_global.flags); + CUSFS_TRACE_LOG_FILE(verbosity,"num_areads 0x%llx",cufs->stats.num_areads); + CUSFS_TRACE_LOG_FILE(verbosity,"num_awrites 0x%llx",cufs->stats.num_awrites); + CUSFS_TRACE_LOG_FILE(verbosity,"num_error_areads 0x%llx",cufs->stats.num_error_areads); + CUSFS_TRACE_LOG_FILE(verbosity,"num_error_awrites 0x%llx",cufs->stats.num_error_awrites); + CUSFS_TRACE_LOG_FILE(verbosity,"num_act_areads 0x%x",cufs->stats.num_act_areads); + CUSFS_TRACE_LOG_FILE(verbosity,"num_act_awrites 0x%x",cufs->stats.num_act_awrites); + CUSFS_TRACE_LOG_FILE(verbosity,"max_num_act_areads 0x%x",cufs->stats.max_num_act_areads); + CUSFS_TRACE_LOG_FILE(verbosity,"max_num_act_awrites 0x%x",cufs->stats.max_num_act_awrites); + CUSFS_TRACE_LOG_FILE(verbosity,"num_success_threads 0x%llx",cufs->stats.num_success_threads); + CUSFS_TRACE_LOG_FILE(verbosity,"num_failed_threads 0x%llx",cufs->stats.num_failed_threads); + CUSFS_TRACE_LOG_FILE(verbosity,"num_active_threads 0x%llx",cufs->stats.num_active_threads); + + return; +} + +/* + * NAME: cusfs_setup_dump_file + * + * FUNCTION: This routine dump data structures for + * the block library. + * + * + * + * RETURNS: 0 - Success, Otherwise error. + * + * + */ + +int cusfs_setup_dump_file(void) +{ + int rc = 0; + char *env_user = getenv("USER"); + char *dump_filename = getenv("CFLSH_USFS_DUMP"); + char *log_pid = getenv("CFLSH_USFS_DUMP_PID"); + char filename[PATH_MAX]; + + + + + if (cusfs_dumpfp) { + + /* + * If dump file pointer is setup, + * then do not set it up again. + */ + + return rc; + + } + + + if (dump_filename == NULL) + { + sprintf(filename, "/tmp/%s.cflsh_usfs_dump", env_user); + dump_filename = filename; + } + + + if (log_pid) { + + /* + * Use different filename for each process + */ + + sprintf(filename,"%s.%d",dump_filename,getpid()); + + + } + + + + if ((cusfs_dumpfp = fopen(dump_filename, "a")) == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to open dump_filename file %s",dump_filename); + + cusfs_dumpfp = NULL; + rc = -1; + } + + + return rc; +} + +/* + * NAME: cusfs_dump_debug_data + * + * FUNCTION: This routine dump data structures for + * the block library. + * + * NOTE: This routine does not use any locking to serialize with the rest + * of the library, since it may be called to debug a range of + * library issues--including deadlock. Thus there is risk, + * that it could hit a segmentation fault. As result, it needs + * to call fflush periodically to ensure any data it's able to get + * is written out the dump file. In addition this code has taken + * some steps to order and check things to minimize the likelihood + * of it hitting a segmentation fault, but there is still no + * guarantee this can be avoid. + * + * RETURNS: None + * + * + */ + +void cusfs_dump_debug_data(const char *reason,const char *reason_filename,const char *reason_function, + int reason_line_num, const char *reason_date) +{ + int i; + cflsh_usfs_t *cufs; + + + char timebuf[TIMELEN+1]; +#if !defined(__64BIT__) && defined(_AIX) + time64_t cur_time; +#else + time_t cur_time; +#endif /* not 32-bit AIX */ + + + + + if (cusfs_dumpfp == NULL) { + + return; + } + + /* + * Print header + */ + fprintf(cusfs_dumpfp,"---------------------------------------------------------------------------\n"); + fprintf(cusfs_dumpfp,"Build date for %s is %s at %s\n",__FILE__,__DATE__,__TIME__); + + fflush(cusfs_dumpfp); + +#if !defined(__64BIT__) && defined(_AIX) + cur_time = cflsh_usfs_time64(NULL); + fprintf(cusfs_dumpfp,"Dump occurred at %s\n",ctime64_r(&cur_time,timebuf)); +#else + cur_time = time(NULL); + fprintf(cusfs_dumpfp,"Dump occurred at %s\n",ctime_r(&cur_time,timebuf)); +#endif /* not 32-bit AIX */ + + fflush(cusfs_dumpfp); + fprintf(cusfs_dumpfp,"PID = 0x%"PRIx64", dump sequence number = 0x%x\n", + (uint64_t)cusfs_global.caller_pid, dump_sequence_num); + fprintf(cusfs_dumpfp,"dump reason %s, filename = %s, function = %s, line_number = %d, date = %s\n", + reason,reason_filename,reason_function,reason_line_num,reason_date); + fflush(cusfs_dumpfp); + fprintf(cusfs_dumpfp,"---------------------------------------------------------------------------\n"); + + fflush(cusfs_dumpfp); + + fetch_and_add(&(dump_sequence_num),+1); + + +#ifdef BLOCK_FILEMODE_ENABLED + fprintf(cusfs_dumpfp,"FILEMODE\n"); +#endif /* BLOCK_FILEMODE_ENABLED */ + +#ifdef CFLASH_LITTLE_ENDIAN_HOST + fprintf(cusfs_dumpfp,"Little Endian\n"); +#else + fprintf(cusfs_dumpfp,"Big Endian\n"); +#endif +#if !defined(__64BIT__) && defined(_AIX) + fprintf(cusfs_dumpfp,"32-bit app support\n"); +#else + fprintf(cusfs_dumpfp,"64-bit app support\n"); + +#endif + + fprintf(cusfs_dumpfp,"\n\n"); + + fprintf(cusfs_dumpfp,"cusfs_log_verbosity 0x%x\n\n",cusfs_log_verbosity); + + fprintf(cusfs_dumpfp,"cusfs_dump_level 0x%x\n\n",cusfs_dump_level); + + fprintf(cusfs_dumpfp,"cusfs_notify_log_level 0x%x\n\n",cusfs_notify_log_level); + + fprintf(cusfs_dumpfp,"cusfs_log_lock (addr) %p\n\n",&cusfs_log_lock); + + + + fprintf(cusfs_dumpfp,"cufs = %p\n\n",&cusfs_global); + + fflush(cusfs_dumpfp); + + fprintf(cusfs_dumpfp," global_lock.file = 0x%x\n",cusfs_global.global_lock.file); + fprintf(cusfs_dumpfp," global_lock.fname = %s\n",cusfs_global.global_lock.filename); + fprintf(cusfs_dumpfp," global_lock.line = %d\n",cusfs_global.global_lock.line); + fprintf(cusfs_dumpfp," global_lock.thread &= %p\n",&(cusfs_global.global_lock.thread)); + fprintf(cusfs_dumpfp," global_lock.thread = 0x%x\n",(uint32_t)(cusfs_global.global_lock.thread)); + fprintf(cusfs_dumpfp," flags = 0x%x\n",cusfs_global.flags); + fprintf(cusfs_dumpfp," hash[] = %p\n\n",&cusfs_global.hash[0]); + + + fprintf(cusfs_dumpfp," eyec = 0x%x",cusfs_global.eyec); + if (!CFLSH_EYECATCH_GLOBAL(&cusfs_global)) { + fprintf(cusfs_dumpfp," (Valid)\n"); + } else { + fprintf(cusfs_dumpfp," (Invalid !!)\n"); + + fflush(cusfs_dumpfp); + + /* + * If global cusfs_global is corrupted then return + * now. Don't attempt to traverse other + * data structures here, because this may + * result in a segmentation fault. + */ + return; + } + + fflush(cusfs_dumpfp); + + for (i=0;ieyec); + + fflush(cusfs_dumpfp); + + fprintf(cusfs_dumpfp,"\n"); + + cufs = cufs->next; + } + + } + + fflush(cusfs_dumpfp); + + + + + fflush(cusfs_dumpfp); + + //?? loop thru all files + + + fflush(cusfs_dumpfp); + + return; +} + + +/* + * NAME: cusfs_signal_dump_handler + * + * FUNCTION: Sets up a signal handler to + * For signal to dump our internal data + * structures. + * + * RETURNS: + * + * 0 - Good completion, otherwise error + * + */ + +void cusfs_signal_dump_handler(int signum, siginfo_t *siginfo, void *uctx) +{ + char reason[100]; + + if (signum == SIGSEGV) { + + CUSFS_TRACE_LOG_FILE(1,"si_code = %d, si_addr = 0x%p", + siginfo->si_code,siginfo->si_addr); + + sprintf(reason,"SIGSEGV: si_code = %d si_addr = %p",siginfo->si_code,siginfo->si_addr); + + + } else { + + sprintf(reason,"Signal = %d",signum); + + } + + cusfs_dump_debug_data(reason,__FILE__,__FUNCTION__,__LINE__,__DATE__); + + + + /* + * If we get here then + * issue default signal. + */ + + if (signum != SIGUSR1) { + + signal(signum,SIG_DFL); + kill(getpid(),signum); + } + + return; +} + +/* + * NAME: cusfs_setup_sigsev_dump + * + * FUNCTION: Sets up a signal handler to + * For SIGSEGV to dump our internal data + * structures. + * + * RETURNS: + * + * 0 - Good completion, otherwise error + * + */ + +int cusfs_setup_sigsev_dump(void) +{ + int rc = 0; + struct sigaction action, oaction; + + + + if (cusfs_setup_dump_file()){ + + return -1; + + } + + + bzero((void *)&action,sizeof(action)); + + bzero((void *)&oaction,sizeof(oaction)); + + action.sa_sigaction = cusfs_signal_dump_handler; + action.sa_flags = SA_SIGINFO; + + + if (sigaction(SIGSEGV, &action,&oaction)) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to set up SIGSEGV handler with errno = %d\n", + errno); + + return -1; + } + + + return rc; +} + + +/* + * NAME: cusfs_setup_sigusr1_dump + * + * FUNCTION: Sets up a signal handler to + * For SIGSEGV to dump our internal data + * structures. + * + * RETURNS: + * + * 0 - Good completion, otherwise error + * + */ + +int cusfs_setup_sigusr1_dump(void) +{ + int rc = 0; + struct sigaction action, oaction; + + + + if (cusfs_setup_dump_file()){ + + return -1; + + } + + + bzero((void *)&action,sizeof(action)); + + bzero((void *)&oaction,sizeof(oaction)); + + action.sa_sigaction = cusfs_signal_dump_handler; + action.sa_flags = SA_SIGINFO; + + + if (sigaction(SIGUSR1, &action,&oaction)) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to set up SIGUSR handler with errno = %d\n", + errno); + + return -1; + } + + + return rc; +} + + + +/* + * NAME: cusfs_get_os_type + * + * FUNCTION: Get Operating System type + * + * + * + * RETURNS: + * + * os_type + * + */ + +cflsh_usfs_os_type_t cusfs_get_os_type(void) +{ + +#ifdef _AIX + + return CFLSH_OS_TYPE_AIX; + +#else + + return CFLSH_OS_TYPE_LINUX; + +#endif + +} + +#ifdef _AIX +/* ---------------------------------------------------------------------------- + * + * NAME: cusfs_get_program_name + * + * FUNCTION: Finds the name of the process associated with our PID. + * + * + * + * CALLED BY: + * + * + * INTERNAL PROCEDURES CALLED: + * + * + * + * EXTERNAL PROCEDURES CALLED: + * + * + * + * RETURNS: NULL string - No process name found. + * string - Process name found. + * + * ---------------------------------------------------------------------------- + */ + +char *cusfs_get_program_name(pid_t pid) +{ + +#define MAX_FETCH_PROCS 100 + + char *process_name = NULL; + pid_t process_index = 0; + int process_list_size; + int i; + int num_process; + + + +#if defined(__64BIT__) + + /* + * 64-bit application + */ + + /* + * NOTE: AIX does not have a mechanism to get a process + * name via getproc (the 32-bit version of the + * getprocs64 call) for 32-bit applications. This + * is due to the reduced size of the 32-bit procinfo + */ + + struct procentry64 *process_list; + + + + process_list_size = sizeof(*process_list) * MAX_FETCH_PROCS; + + + process_list = malloc(process_list_size); + + if (process_list == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to allocate process list of size = %d,with errno = %d", + process_list_size,errno); + + return NULL; + } + + + do { + + + bzero(process_list,process_list_size); + + num_process = getprocs64(process_list,sizeof(*process_list), + NULL,0,&process_index,MAX_FETCH_PROCS); + + + if (num_process == 0) { + + + CUSFS_TRACE_LOG_FILE(5,"No processes returned from getprocs64. last index = %d", + process_index); + break; + } + + for (i=0;i < num_process; i++) { + + if (pid == (pid_t)process_list[i].pi_pid) { + + /* + * We found the matching process. + * Now let's extract the process' + * name. + */ + + process_name = strdup(process_list[i].pi_comm); + break; + + } + } + + if (process_name) { + + /* + * If we found the process name, then + * break out this loop. + */ + + break; + } + + if (num_process < MAX_FETCH_PROCS) { + + + /* + * There are no more process eleents + * to fetch. + */ + + CUSFS_TRACE_LOG_FILE(5,"No more processes left to fetch. last index = %d", + process_index); + break; + } + + + } while (num_process); + + + +#endif /* 64-bit */ + + CUSFS_TRACE_LOG_FILE(5,"Our process name = %s",process_name); + + + return process_name; +} + +#endif /* _AIX */ + + +/* + * NAME: cusfs_get_fs_block_size + * + * FUNCTION: This routine determines the data block + * size used by the filesystem. This is the + * smallest data granularity allowed. + * + * + * + * INPUTS: + * + * + * RETURNS: -1 - error, otherwise success. + * + * + */ +int cusfs_get_fs_block_size(cflsh_usfs_t *cufs, int flags) +{ + + + /* + * If each bit in a 4K page represents a block, + * then such a page can represent 32K data + * blocks (8 * 4K = 32K) of the associated + * filesystem block size. + */ + + + /* + * ?? TODO: Maybe this routine should also take into account + * max transfer size too in the future, since that + * reflects how easy it is to read the whole free block + * table into host memory.. + */ + + + if (cufs->num_blocks < 0x50) { + + /* + * This disk is too small to + * fit our metadata an have any + * limited notion of data blocks. + */ + + CUSFS_TRACE_LOG_FILE(1,"This disk %s is too small for an FS, only 0x%x blocks total", + cufs->device_name,cufs->num_blocks); + errno = EINVAL; + return -1; + + } else if (cufs->num_blocks <= 0x20000) { + + + /* + * The largest disk size for this range + * will require 4 4K disk blocks + * for the free block table with a + * fs block size of CFLSH_USFS_BLOCK_SIZE_MIN + * (4K). This supports up to 128K 4K disk blocks + * (or 512 MB). + * + * NOTE: A disk max transfer size of 16K would + * allow the whole table to be read + * with one read. + */ + + cufs->fs_block_size = CFLSH_USFS_BLOCK_SIZE_MIN; + + + } else if (cufs->num_blocks <= 0x200000) { + + + + /* + * The largest disk size for this range + * will require 16 16K disk blocks + * for the free block table. This + * supportes up 512K 16K data blocks, + * (or 8GB). + * + * NOTE: A disk max transfer size of 64K would + * allow the whole table to be read + * with one read. + */ + + cufs->fs_block_size = 0x4000; + + } else if (cufs->num_blocks <= 0x2000000) { + + + + /* + * The largest disk size for this range + * will require 64 64K disk blocks to + * for the free block table. + * (or 128GB). + * + * NOTE: A disk max transfer size of 256K would + * allow the whole table to be read + * with one read. + */ + + cufs->fs_block_size = 0x10000; + + + } else if (cufs->num_blocks <= 0x20000000LL) { + + + /* + * The largest disk size for this range + * will require 256 256K disk blocks to + * for the free block table. + * (or 2TB). + * + * NOTE: A disk max transfer size of 1 MB would + * allow the whole table to be read + * with one read. + */ + + cufs->fs_block_size = 0x40000; + + + } else if (cufs->num_blocks <= 0x2000000000LL) { + + + + /* + * The largest disk size for this range + * will require 1024 1MB disk blocks to + * for the free block table. + * (or 32TB). + * + * NOTE: A disk max transfer size of 4 MB would + * allow the whole table to be read + * with one read. + */ + + cufs->fs_block_size = 0x100000LL; + + } else { + + /* + * For disks larger than 32TB, we will use + * 16 MB block sizes. + */ + + cufs->fs_block_size = 0x1000000LL; + + } + + if (cufs->fs_block_size % cufs->disk_block_size) { + + + CUSFS_TRACE_LOG_FILE(1,"The fs_block_size 0x%x is not a multiple of disk_block_size 0x%x for %", + cufs->fs_block_size,cufs->disk_block_size,cufs->device_name); + return -1; + } + + + return cufs->fs_block_size; +} + + + + +/* + * NAME: cusfs_get_inode_table_size + * + * FUNCTION: This routine determines the size of the + * inode table used by the filesystem. + * + * + * + * INPUTS: + * + * + * RETURNS: 0 - success, otherwise error. + * + * + */ +int cusfs_get_inode_table_size(cflsh_usfs_t *cufs, int flags) +{ + int rc = 0; + + + + /* + * ?? TODO: Maybe this routine should also take into account + * the size of the disk and the + * max transfer size too in the future, since that + * reflects how easy it is to read the whole inode + * table into host memory.. + */ + + /* + * NOTES: + * + * 8 inode entries fit into a 4K block. Thus if the max transfer + * size is 16MB (the maximum supported, this means that + * 32K inodes could be read into host memory at time. + * + * + */ + + + if (cufs->num_blocks < 0x50) { + + /* + * This disk is too small to + * fit our metadata an have any + * limited notion of inodes + */ + + CUSFS_TRACE_LOG_FILE(1,"This disk %s is too small for an FS, only 0x%x blocks total", + cufs->device_name,cufs->num_blocks); + errno = EINVAL; + return -1; + + } else if (cufs->num_blocks <= 0x20000) { + + /* + * If the disk soze is no + * greater than 512 MB, then limit + * the size of the inode table to + * 512K, which means only 1K inodes + * supported. + */ + + cufs->superblock.num_inodes = 0x400; + + } else if (cufs->num_blocks <= 0x2000000) { + + + + /* + * If the disk size is no greater + * 128 GB, then limit the size of the + * inode table to 16 MB, which means + * only 32K inodes supported. + */ + + + cufs->superblock.num_inodes = 0x8000; + + } else { + + /* + * For disks larger than 128 GB, + * limit the size of the inode table + * to 64MB, which means only + * 128K inodes supported. + */ + + cufs->superblock.num_inodes = 0x20000; + + } + + cufs->num_inode_data_blocks = (cufs->superblock.num_inodes * sizeof(cflsh_usfs_inode_t))/cufs->disk_block_size; + + + /* + * If we are accessing an existing filesystem, we need to ensure our calculations on + * on the size of inode table, free table etc, match those in the superblock. + */ + + return rc; +} + +/* + * NAME: cusfs_get_cufs + * + * FUNCTION: This routine gets a CAPI flash Userspace + * filesystem object. + * + * NOTE: This routine assumes the caller + * has the cusfs_global.global_lock. + * + * + * INPUTS: + * + * + * RETURNS: + * Pointer to cufs + * + */ +cflsh_usfs_t *cusfs_get_cufs(char *device_name, int flags) +{ + cflsh_usfs_t *cufs= NULL; + cflsh_usfs_t *tmp_cufs; + size_t num_blocks; + chunk_ext_arg_t ext = (chunk_ext_arg_t)NULL; + int i,j; /* General counters */ + +#define CFLSH_NUM_FREE_FAIL_CMDS 10 + + if (device_name == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Null device_name"); + return NULL; + } +#ifdef _AIX + cflsh_usfs_init(); +#endif /* AIX */ + + /* + * Align on 4K boundary, so that we can use + * the low order bits for eyecatcher ir hashing + * if we decide to pass back a modified pointer + * to the user. Currently we are not doing this, + * but depending on the efficiency of the hash table + * we may need to in the future. + */ + + if ( posix_memalign((void *)&cufs,4096, + (sizeof(*cufs)))) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed posix_memalign for cufs of %s, errno= %d", + device_name,errno); + + + return NULL; + + } + + bzero(cufs,sizeof(*cufs)); + + + cufs->max_requests = CFLSH_USFS_DISK_MAX_CMDS; + + cufs->chunk_id = cblk_open(device_name,cufs->max_requests,O_RDWR,ext,0); + + if (cufs->chunk_id == NULL_CHUNK_ID) { + + + CUSFS_TRACE_LOG_FILE(1,"cblk_open failed errno= %d, num_requests = %d, os_type = %d", + errno,cufs->max_requests,cusfs_global.os_type); + + free(cufs); + + return NULL; + } + + + strcpy(cufs->device_name,device_name); + + if (cblk_get_lun_size(cufs->chunk_id,&num_blocks,0)) { + + + CUSFS_TRACE_LOG_FILE(1,"failed to get lun size with errno = %d", + errno); + + cblk_close(cufs->chunk_id,0); + free(cufs); + + return NULL; + + } + + cufs->num_blocks = num_blocks; + + CUSFS_TRACE_LOG_FILE(5,"capacity = 0x%llx",cufs->num_blocks); + + if (cblk_get_stats(cufs->chunk_id,&(cufs->disk_stats),0)) { + + + CUSFS_TRACE_LOG_FILE(1,"failed to get stats with errno = %d", + errno); + + cblk_close(cufs->chunk_id,0); + free(cufs); + + return NULL; + + } + + cufs->max_xfer_size = cufs->disk_stats.max_transfer_size; + + + CUSFS_TRACE_LOG_FILE(5,"disk block size reported = 0x%x, max_xfer_size = 0x%x,num_paths = %d ", + cufs->disk_stats.block_size, + cufs->max_xfer_size, + cufs->disk_stats.num_paths); + + + cufs->disk_block_size = CFLSH_USFS_DEVICE_BLOCK_SIZE; + + + /* + * Set buf size to the initial max_xfer_size + */ + + cufs->buf_size = cufs->max_xfer_size * cufs->disk_block_size; + + if (cufs->buf_size < cufs->disk_block_size) { + + CUSFS_TRACE_LOG_FILE(1,"max transfer size = 0x%x is less than disk block size = 0x%x", + cufs->buf_size,cufs->disk_block_size); + + cblk_close(cufs->chunk_id,0); + free(cufs); + + return NULL; + + } + + + if ( posix_memalign((void *)&(cufs->buf),4096, + cufs->buf_size)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed posix_memalign for buf failed, errno= %d, size = 0x%x", + errno,cufs->buf_size); + + cblk_close(cufs->chunk_id,0); + free(cufs); + + return NULL; + } + + + + cufs->free_blk_tbl_buf_size = cufs->buf_size; + + if ( posix_memalign((void *)&(cufs->free_blk_tbl_buf),4096, + cufs->free_blk_tbl_buf_size)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed posix_memalign for free_blk_tbl_buf failed, errno= %d, size = 0x%x", + errno,cufs->free_blk_tbl_buf_size); + + + cblk_close(cufs->chunk_id,0); + free(cufs->buf); + free(cufs->free_blk_tbl_buf); + free(cufs); + + return NULL; + } + + cufs->inode_tbl_buf_size = cufs->buf_size; + + if ( posix_memalign((void *)&(cufs->inode_tbl_buf),4096, + cufs->inode_tbl_buf_size)) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed posix_memalign for inode_tbl_buf failed, errno= %d, size = 0x%x", + errno,cufs->inode_tbl_buf_size); + + + cblk_close(cufs->chunk_id,0); + free(cufs->buf); + free(cufs->free_blk_tbl_buf); + free(cufs); + + return NULL; + } + + + /* + * ?? TODO maybe we need to only get_fs_block_size and + * inode table size, if we are explicilty creating + * a filesystem. Otherwise we should determine this + * information (or maybe validate) it from the superblock. + */ + + if (cusfs_get_fs_block_size(cufs,0) == -1) { + + + CUSFS_TRACE_LOG_FILE(1,"failed to get fs block size for disk %s", + device_name); + + cblk_close(cufs->chunk_id,0); + free(cufs->buf); + free(cufs->free_blk_tbl_buf); + free(cufs->inode_tbl_buf); + free(cufs); + + return NULL; + + } + + if (cusfs_get_inode_table_size(cufs,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to get inode table size for disk %s", + device_name); + + cblk_close(cufs->chunk_id,0); + free(cufs->buf); + free(cufs->free_blk_tbl_buf); + free(cufs->inode_tbl_buf); + free(cufs); + + return NULL; + + } + + CUSFS_LOCK_INIT((cufs->lock)); + + CUSFS_LOCK_INIT((cufs->free_table_lock)); + + CUSFS_LOCK_INIT((cufs->inode_table_lock)); + + CUSFS_LOCK_INIT((cufs->journal_lock)); + + cusfs_global.num_active_cufs++; + cusfs_global.num_max_active_cufs = MAX(cusfs_global.num_active_cufs,cusfs_global.num_max_active_cufs); + + + cufs->index = cusfs_global.next_cusfs_id++; + + cufs->eyec = CFLSH_USFS_EYEC_FS; + + + cufs->async_pool_num_cmds = cufs->max_requests; + cufs->async_pool_size = sizeof(cflsh_usfs_aio_entry_t) * cufs->async_pool_num_cmds; + cufs->async_pool = malloc(cufs->async_pool_size); + + if (cufs->async_pool == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc async_pool disk %s", + device_name); + + cblk_close(cufs->chunk_id,0); + free(cufs->buf); + free(cufs->free_blk_tbl_buf); + free(cufs->inode_tbl_buf); + free(cufs); + + return NULL; + } + + for (i= 0; i < cufs->async_pool_num_cmds; i++) { + + + cufs->async_pool[i].eyec = CFLSH_USFS_EYEC_AIO; + cufs->async_pool[i].index = i; + cufs->async_pool[i].local_buffer_len = cufs->fs_block_size; + cufs->async_pool[i].local_buffer = malloc(cufs->async_pool[i].local_buffer_len); + if (cufs->async_pool[i].local_buffer == NULL) { + + + /* + * Just try to continue with no command pool. + */ + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc an async_pool command %d async_pool_num_cmds %d disk %s", + i,cufs->async_pool_num_cmds,device_name); + + + if (i > CFLSH_NUM_FREE_FAIL_CMDS) { + + /* + * Only free up the previous async command and try to + * continue to run. + */ + + + for (j=i-1; j >= i-CFLSH_NUM_FREE_FAIL_CMDS; j--) { + + CUSFS_DQ_NODE(cufs->head_free,cufs->tail_free, &(cufs->async_pool[j]),free_prev,free_next); + + cufs->async_pool[j].eyec = 0; + + free(cufs->async_pool[j].local_buffer); + } + + cufs->async_pool_num_cmds = i-CFLSH_NUM_FREE_FAIL_CMDS; + + } else { + + for (j=0; j < i; j++) { + + CUSFS_DQ_NODE(cufs->head_free,cufs->tail_free, &(cufs->async_pool[j]),free_prev,free_next); + + cufs->async_pool[j].eyec = 0; + + free(cufs->async_pool[j].local_buffer); + } + + cufs->async_pool_num_cmds = 0; + + } + + break; + + } else { + CUSFS_Q_NODE_TAIL(cufs->head_free,cufs->tail_free, &(cufs->async_pool[i]),free_prev,free_next); + } + } + + CUSFS_TRACE_LOG_FILE(9,"cufs->async_pool_num_cmds = 0x%x, buffer_size = 0x%llx", + cufs->async_pool_num_cmds,cufs->fs_block_size); + + if (cusfs_start_aio_thread_for_cufs(cufs,0)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to start background thread for disk %s", + device_name); + + cblk_close(cufs->chunk_id,0); + free(cufs->buf); + free(cufs->free_blk_tbl_buf); + free(cufs->inode_tbl_buf); + free(cufs); + + return NULL; + } + + + if (cflsh_usfs_master_register(device_name,&(cufs->master_handle))) { + + CUSFS_TRACE_LOG_FILE(1,"failed to register with masterfor disk %s", + device_name); + + cblk_close(cufs->chunk_id,0); + free(cufs->buf); + free(cufs->free_blk_tbl_buf); + free(cufs->inode_tbl_buf); + free(cufs); + + return NULL; + + } + + if (cusfs_global.hash[cufs->index & CUSFS_HASH_MASK] == NULL) { + + cusfs_global.hash[cufs->index & CUSFS_HASH_MASK] = cufs; + } else { + + tmp_cufs = cusfs_global.hash[cufs->index & CUSFS_HASH_MASK]; + + while (tmp_cufs) { + + if ((ulong)tmp_cufs & CUSFS_BAD_ADDR_MASK ) { + + /* + * Cufs addresses are allocated + * on certain alignment. If this + * potential cufs address does not + * have the correct alignment then fail + * this request. + */ + + cusfs_global.num_bad_cusfs_ids++; + + CUSFS_TRACE_LOG_FILE(1,"Corrupted cufs address = 0x%p, hash[] = 0x%p index = 0x%x", + tmp_cufs, cusfs_global.hash[cufs->index & CUSFS_HASH_MASK], + (cufs->index & CUSFS_HASH_MASK)); + + CUSFS_LIVE_DUMP_THRESHOLD(5,"0x200"); + + cufs->eyec = 0; + free(cufs->buf); + free(cufs->free_blk_tbl_buf); + free(cufs->inode_tbl_buf); + + if (cflsh_usfs_master_unregister(cufs->master_handle)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unregister with master for disk %s with errno %d", + cufs->device_name,errno); + } + + free(cufs); + + errno = EFAULT; + return NULL; + } + + + if (tmp_cufs->next == NULL) { + + tmp_cufs->next = cufs; + + cufs->prev = tmp_cufs; + break; + } + + tmp_cufs = tmp_cufs->next; + + } /* while */ + + } + + + + + return cufs; + + +} + +/* + * NAME: cusfs_release_cufs + * + * FUNCTION: This routine frees up all resources + * for a filesystem object. + * + * NOTE: This routine assumes the caller + * has the cusfs_global.global_lock. + * + * + * INPUTS: + * + * + * RETURNS: None + * + */ +void cusfs_release_cufs(cflsh_usfs_t *cufs, int flags) +{ + int rc; + int i; + + + if (cufs != NULL) { + + cusfs_display_stats(cufs,3); + + for (i=0; i < cufs->async_pool_num_cmds; i++) { + + free(cufs->async_pool[i].local_buffer); + } + + cusfs_stop_aio_thread_for_cufs(cufs,0); + + if (cflsh_usfs_master_unregister(cufs->master_handle)) { + + CUSFS_TRACE_LOG_FILE(1,"failed to unregister with master for disk %s with errno %d", + cufs->device_name, errno); + } + + free(cufs->buf); + free(cufs->free_blk_tbl_buf); + free(cufs->inode_tbl_buf); + + rc = cblk_close(cufs->chunk_id,0); + + if (rc) { + + CUSFS_TRACE_LOG_FILE(1,"cblk_close failed errno= %d, device_name = %s, os_type = %d", + errno,cufs->device_name,cusfs_global.os_type); + } + + if (cusfs_global.num_active_cufs > 0) { + cusfs_global.num_active_cufs--; + } + cufs->eyec = 0; + + bzero(cufs->device_name,PATH_MAX); + /* + * Remove cufs from hash list + */ + + if (((ulong)cufs->next & CUSFS_BAD_ADDR_MASK ) || + ((ulong)cufs->prev & CUSFS_BAD_ADDR_MASK )) { + + /* + * Cufs addresses are allocated + * on certain alignment. If these + * potential cufs addresses do not + * have the correct alignment then + * print an error to the trace log. + */ + + cusfs_global.num_bad_cusfs_ids++; + /* + * Try continue in this case. + */ + + + CUSFS_TRACE_LOG_FILE(1,"Corrupted cufs next address = 0x%p, prev address = 0x%p, hash[] = 0x%p", + cufs->next, cufs->prev, cusfs_global.hash[cufs->index & CUSFS_HASH_MASK]); + + } + + if (cufs->prev) { + cufs->prev->next = cufs->next; + + } else { + + cusfs_global.hash[cufs->index & CUSFS_HASH_MASK] = cufs->next; + } + + if (cufs->next) { + cufs->next->prev = cufs->prev; + } + + } + + + return; +} + + +/* + * NAME: cusfs_alloc_file + * + * FUNCTION: This routine allocates a file (data object) + * filesystem object. + * + * NOTE: This routine assumes the caller + * has the cusfs_global.global_lock. + * + * + * INPUTS: + * + * + * RETURNS: + * Pointer to file + * + */ +cflsh_usfs_data_obj_t *cusfs_alloc_file(cflsh_usfs_t *cufs, char *filename, int flags) +{ + cflsh_usfs_data_obj_t *file = NULL; + + + + if (cufs == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Null cufs"); + errno = EINVAL; + return NULL; + } + + if (filename == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Null device_name"); + errno = EINVAL; + return NULL; + } + + /* + * Align on 4K boundary, so that we can use + * the low order bits for eyecatcher ir hashing + * if we decide to pass back a modified pointer + * to the user. Currently we are not doing this, + * but depending on the efficiency of the hash table + * we may need to in the future. + */ + + if ( posix_memalign((void *)&file,4096, + (sizeof(*file)))) { + + + CUSFS_TRACE_LOG_FILE(1,"Failed posix_memalign for file of %s, errno= %d", + filename,errno); + + + return NULL; + + } + + bzero(file,sizeof(*file)); + + if (flags & CFLSH_USFS_ALLOC_READ_AHEAD) { + file->seek_ahead.local_buffer_len = cufs->fs_block_size; + file->seek_ahead.local_buffer = malloc(file->seek_ahead.local_buffer_len); + + if (file->seek_ahead.local_buffer == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to malloc read ahead buffer for filename %s", + filename); + /* + * Continue, but provide no read ahead + */ + file->seek_ahead.local_buffer_len = 0; + + } + } + + + strcpy(file->filename,filename); + + file->eyec = CFLSH_USFS_EYEC_FL; + + file->fs = cufs; + + file->index = cusfs_global.next_file_id++; + + CUSFS_LOCK_INIT((file->lock)); + + return file; + +} + + + +/* + * NAME: cusfs_free_file + * + * FUNCTION: This routine frees a file (data object) + * filesystem object from memory. + * + * NOTE: This routine assumes the caller + * has the cusfs_global.global_lock. + * + * + * INPUTS: + * + * + * RETURNS: + * Pointer to file + * + */ +void cusfs_free_file(cflsh_usfs_data_obj_t *file) +{ + if (file == NULL) { + + return; + } + + if (file->data_buf) { + free(file->data_buf); + } + + if (file->seek_ahead.local_buffer) { + free(file->seek_ahead.local_buffer); + } + free(file); + return; +} + + +/* + * NAME: cusfs_get_aio + * + * FUNCTION: Find a free async I/O entry for the specified filesystem + * + * + * NOTE: This routine assumes the caller + * has the filesystem lock. + * + * + * INPUTS: + * + * + * RETURNS: 0 - success, Otherwise error. + * + * + */ +int cusfs_get_aio(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, cflsh_usfs_aio_entry_t **aio,int flags) +{ + cflsh_usfs_aio_entry_t *aiop; + int pthread_rc = 0; + + + if (cufs == NULL) { + CUSFS_TRACE_LOG_FILE(1,"No valid cufs"); + + errno = EINVAL; + return -1; + } + + if (file == NULL) { + CUSFS_TRACE_LOG_FILE(1,"No valid file for disk %s",cufs->device_name); + + errno = EINVAL; + return -1; + } + + if (CFLSH_EYECATCH_FS(cufs)) { + CUSFS_TRACE_LOG_FILE(1,"Invalid file for disk %s",cufs->device_name); + + errno = EINVAL; + return -1; + } + + aiop = cufs->head_free; + + if (aiop == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"No free aio for disk %s, num_act_areads = 0x%x, num_act_awrites = 0x%x", + cufs->device_name,cufs->stats.num_act_areads,cufs->stats.num_act_awrites); + + + /* + * We're out of aio commands, signal back ground to process them in case + * it stopped. + */ + + pthread_rc = pthread_cond_signal(&(cufs->thread_event)); + + if (pthread_rc) { + + CUSFS_TRACE_LOG_FILE(5,"pthread_cond_signal failed rc = %d,errno = %d", + pthread_rc,errno); + } + + errno = EBUSY; + return -1; + } + + if (CFLSH_EYECATCH_AIO(aiop)) { + CUSFS_TRACE_LOG_FILE(1,"Invalid aiop for disk %s",cufs->device_name); + + errno = EINVAL; + return -1; + + } + + /* + * Remove command from free list + */ + + CUSFS_DQ_NODE(cufs->head_free,cufs->tail_free, aiop,free_prev,free_next); + + aiop->flags = 0; + + aiop->file = file; + + /* + * place command on active list for both filesystem and file. + */ + + CUSFS_Q_NODE_TAIL(cufs->head_act,cufs->tail_act, aiop,act_prev,act_next); + + CUSFS_Q_NODE_TAIL(file->head_act,file->tail_act, aiop,file_prev,file_next); + + CUSFS_TRACE_LOG_FILE(9,"Got aiop = %p, for disk %s",aiop,cufs->device_name); + + + *aio = aiop; + + return 0; +} + +/* + * NAME: cusfs_free_aio + * + * FUNCTION: Free async I/O entry for the specified filesystem + * + * + * NOTE: This routine assumes the caller + * has the file lock. + * + * + * INPUTS: + * + * + * RETURNS: 0 - success, Otherwise error. + * + * + */ +int cusfs_free_aio(cflsh_usfs_t *cufs, cflsh_usfs_aio_entry_t *aio,int flags) +{ + cflsh_usfs_data_obj_t *file; + + if (cufs == NULL) { + CUSFS_TRACE_LOG_FILE(1,"No valid cufs"); + + errno = EINVAL; + return -1; + } + + if (CFLSH_EYECATCH_FS(cufs)) { + CUSFS_TRACE_LOG_FILE(1,"Invalid file for disk %s",cufs->device_name); + + errno = EINVAL; + return -1; + } + + if (CFLSH_EYECATCH_AIO(aio)) { + CUSFS_TRACE_LOG_FILE(1,"Invalid aio for disk %s",cufs->device_name); + + errno = EINVAL; + return -1; + + } + + if (aio->op == CFLSH_USFS_AIO_E_INVAL) { + + CUSFS_TRACE_LOG_FILE(1,"This aio has already been freed for disk %s",cufs->device_name); + + errno = EINVAL; + return -1; + + } + + /* + * Remove command from active list + */ + + + CUSFS_DQ_NODE(cufs->head_act,cufs->tail_act, aio,act_prev,act_next); + + + file = aio->file; + + if (file) { + + CUSFS_DQ_NODE(file->head_act,file->tail_act, aio,file_prev,file_next); + + } else { + + CUSFS_TRACE_LOG_FILE(1,"Freeing aio = %p, but file not set for disk %s,aio->op = 0x%x, aio->flags = 0x%x", + file->head_act, cufs->device_name,aio->op,aio->flags); + } + + CUSFS_TRACE_LOG_FILE(9,"Freeing aio = %p, for disk %s, for start_lba 0x%llx aio_offset = 0x%llx aio_nbytes = 0x%llx num_act_areads = 0x%x, num_act_awrites = 0x%x", + file->head_act, cufs->device_name, + aio->aio_offset,aio->aio_nbytes, + cufs->stats.num_act_areads,cufs->stats.num_act_awrites); + + if (aio->op == CFLSH_USFS_AIO_E_WRITE) { + cufs->stats.num_act_awrites--; + } else { + + cufs->stats.num_act_areads--; + } + aio->op = CFLSH_USFS_AIO_E_INVAL; + aio->caller_buffer = NULL; + aio->caller_buffer_len = 0; + aio->local_buffer_len_used = 0; + aio->tag = 0; + aio->offset = 0; + aio->start_lba = 0; + aio->aiocbp = NULL; + + bzero(&(aio->cblk_status),sizeof(aio->cblk_status)); + + bzero(&(aio->aio_sigevent),sizeof(aio->aio_sigevent)); + + /* + * Place command on free list + */ + + CUSFS_Q_NODE_TAIL(cufs->head_free,cufs->tail_free, aio,free_prev,free_next); + + return 0; +} + + +/* + * NAME: cusfs_issue_aio + * + * FUNCTION: async I/O entry for this file + * + * + * INPUTS: + * + * + * RETURNS: 0 - success, Otherwise error. + * + * + */ +int cusfs_issue_aio(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, uint64_t start_lba, struct aiocb64 *aiocbp, + void *buffer, size_t nbytes, int flags) +{ + + cflsh_usfs_aio_entry_t *aio = NULL; +#ifdef _CFLSH_USFS_ARESULT + int arw_flags = (CBLK_ARW_USER_TAG_FLAG|CBLK_ARW_WAIT_CMD_FLAGS); +#else + int arw_flags = (CBLK_ARW_USER_STATUS_FLAG|CBLK_ARW_WAIT_CMD_FLAGS); +#endif + void *buf; + size_t nblocks; + int pthread_rc = 0; + int rc; + int retry; + + if (cufs == NULL) { + CUSFS_TRACE_LOG_FILE(1,"No valid cufs"); + + errno = EINVAL; + return -1; + } + + if (file == NULL) { + CUSFS_TRACE_LOG_FILE(1,"No valid file for disk %s",cufs->device_name); + + errno = EINVAL; + return -1; + } + + if (cufs->async_pool_num_cmds == 0) { + + CUSFS_TRACE_LOG_FILE(1,"No async commands were allocated for disk %s",cufs->device_name); + + errno = ENOMEM; + return -1; + + } + + + + if (aiocbp) { + + retry = 1; + } else { + retry = 1000; + } + + while (retry--) { + + if (cusfs_get_aio(cufs,file,&aio,0)) { + + if (retry == 0) { + errno = EAGAIN; + + return -1; + } + } else { + + break; + } + cusfs_check_active_aio_complete(cufs,0); + CUSFS_UNLOCK(file->lock); + CUSFS_UNLOCK(cufs->lock); + usleep(1000); + CUSFS_LOCK(cufs->lock); + CUSFS_LOCK(file->lock); + + } + + + nblocks = nbytes/cufs->disk_block_size; + + + + /* + * If both buffer and aiocb are set, + * then assume buffer is the intended use. + * This can happen if an aiocb needs + * to be broken into multiple requests. + */ + + if (aiocbp) { + + /* + * Since one aiocb can be broken + * into multiple requests, save off + * original aiocbp offset and size. + */ +#ifdef _AIX + aio->aiocbp = (struct aiocb64 *)aiocbp->aio_handle; + if (aiocbp->aio_word1 & CUSFS_AIOCB32_FLG) { + aio->flags |= CFLSH_USFS_AIO_E_AIOCB; + } else { + aio->flags |= CFLSH_USFS_AIO_E_AIOCB64; + } +#else + aio->aiocbp = (struct aiocb64 *)aiocbp->__next_prio; + if (aiocbp->__glibc_reserved[0] & CUSFS_AIOCB32_FLG) { + aio->flags |= CFLSH_USFS_AIO_E_AIOCB; + } else { + aio->flags |= CFLSH_USFS_AIO_E_AIOCB64; + } +#endif /* !_AIX */ + + aio->aio_offset = aiocbp->aio_offset; + aio->aio_nbytes = aiocbp->aio_nbytes; + aio->aio_sigevent = aiocbp->aio_sigevent; + + } + + if (flags & CFLSH_USFS_RDWR_DIRECT) { + + /* + * The caller is requesting that we use + * the caller's buffer and allow direct + * DMA to/from it. + */ + + aio->caller_buffer = (void *)aiocbp->aio_buf; + aio->caller_buffer_len = aiocbp->aio_nbytes; + + buf = aio->caller_buffer; + + if (aiocbp->aio_offset % cufs->disk_block_size) { + CUSFS_TRACE_LOG_FILE(1,"Request is not block aligned offset = 0x%llx for file %s for disk %s", + aiocbp->aio_offset,file->filename,cufs->device_name); + + cusfs_free_aio(cufs,aio,0); + + errno = EINVAL; + return -1; + + } + } else { + aio->flags |= CFLSH_USFS_AIO_E_LBUF; + buf = aio->local_buffer; + + if (nbytes > aio->local_buffer_len) { + CUSFS_TRACE_LOG_FILE(1,"Request transfer size of 0x%llx is larger than local_buffer_len of 0x%llx, for file %s for disk %s", + nbytes,aio->local_buffer_len,file->filename,cufs->device_name); + + cusfs_free_aio(cufs,aio,0); + + errno = EINVAL; + return -1; + } + aio->local_buffer_len_used = nbytes; + bcopy(buffer,aio->local_buffer,aio->local_buffer_len_used); + } + aio->start_lba = start_lba; + + + + aio->flags |= CFLSH_USFS_AIO_E_ISSUED; + + /* + * Signal background thread to look for completion even + * though we have not yet sent it. This is because it + * will take a while to wake up and we currently have the + * same lock. Thus it can not start to process until this + * thread releases the lock. + */ + + pthread_rc = pthread_cond_signal(&(cufs->thread_event)); + + if (pthread_rc) { + + CUSFS_TRACE_LOG_FILE(5,"pthread_cond_signal failed rc = %d,errno = %d", + pthread_rc,errno); + } + + + + CUSFS_TRACE_LOG_FILE(9,"Issuing aio %p aio_offset = 0x%llx, start_lba 0x%llx with status 0x%x of on aio->aiocbp = %p", + aio,aio->aio_offset,aio->start_lba,aio->cblk_status.status,aio->aiocbp); + +#ifdef _CFLSH_USFS_ARESULT + + aio->tag = aio->index; +#endif /* _CFLSH_USFS_ARESULT */ + + if (flags & CFLSH_USFS_RDWR_WRITE) { + + aio->op = CFLSH_USFS_AIO_E_WRITE; + rc = cblk_awrite(cufs->chunk_id,buf,aio->start_lba,nblocks,&(aio->tag),&(aio->cblk_status),arw_flags); + } else { + + aio->op = CFLSH_USFS_AIO_E_READ; + rc = cblk_aread(cufs->chunk_id,buf,aio->start_lba,nblocks,&(aio->tag),&(aio->cblk_status),arw_flags); + + } + + if (rc < 0) { + + CUSFS_TRACE_LOG_FILE(1,"Failed to issue aio %p with ,errno = %d", + aio,errno); + + + if (flags & CFLSH_USFS_RDWR_WRITE) { + + cufs->stats.num_error_awrites++; + + } else { + + cufs->stats.num_error_areads++; + + } + + cusfs_free_aio(cufs,aio,0); + + + + } else { + + if (flags & CFLSH_USFS_RDWR_WRITE) { + + cufs->stats.num_awrites++; + cufs->stats.num_act_awrites++; + + cufs->stats.max_num_act_awrites = MAX(cufs->stats.max_num_act_awrites,cufs->stats.num_act_awrites); + + } else { + + cufs->stats.num_areads++; + cufs->stats.num_act_areads++; + + cufs->stats.max_num_act_areads = MAX(cufs->stats.max_num_act_areads,cufs->stats.num_act_areads); + + } + + rc = nblocks; + } + return rc; + +} + +/* + * NAME: cusfs_check_active_aio_complete + * + * FUNCTION: Check if active async I/O is complete + * then process them accordingly. + * + * NOTES: This routine assumes the caller has the cufs->lock + * + * + * INPUTS: + * + * + * RETURNS: 0 - success, Otherwise error. + * + * + */ +int cusfs_check_active_aio_complete(cflsh_usfs_t *cufs, int flags) +{ + int rc = 0; + cflsh_usfs_aio_entry_t *aio; + int pthread_rc = 0; + pthread_t thread_id; + pthread_attr_t *thread_attr = NULL; + struct aiocb *aiocbp; + struct aiocb64 *aiocbp64; + int error_code; + ssize_t ret_code; + cflsh_usfs_data_obj_t *file; +#ifndef _CFLSH_USFS_ARESULT + cflsh_usfs_aio_entry_t *next_aio; +#else + int tag; + uint64_t status; +#endif /* _CFLSH_USFS_ARESULT */ + + + aio = cufs->head_act; + + CUSFS_TRACE_LOG_FILE(9,"aio = %p, for disk %s, num_act_areads = 0x%x, num_act_awrites = 0x%x", + aio, cufs->device_name,cufs->stats.num_act_areads,cufs->stats.num_act_awrites); + if ((aio) && + (flags & CFLSH_USFS_PTHREAD_SIG)) { + + pthread_rc = pthread_cond_signal(&(cufs->thread_event)); + + if (pthread_rc) { + + CUSFS_TRACE_LOG_FILE(5,"pthread_cond_signal failed rc = %d,errno = %d", + pthread_rc,errno); + } + + } + + + while (aio) { + + + CUSFS_TRACE_LOG_FILE(6,"Found aio %p for start_lba 0x%llx with status 0x%x for aio_offset = 0x%llx", + aio,aio->start_lba,aio->cblk_status.status,aio->aio_offset); + + +#ifndef _CFLSH_USFS_ARESULT + + + + next_aio = aio->act_next; + + if ((aio->flags & CFLSH_USFS_AIO_E_ISSUED) && + ((aio->cblk_status.status != CBLK_ARW_STATUS_PENDING))) { + + /* + * AIO completed with success/error. Lets process it + * now. First dequue it from active list. + */ + + aio->flags &= ~CFLSH_USFS_AIO_E_ISSUED; + + + + + CUSFS_TRACE_LOG_FILE(6,"aio %p completed for start_lba 0x%llx with status 0x%x for aio_offset = 0x%llx, aio->flags = 0x%x", + aio,aio->start_lba,aio->cblk_status.status,aio->aio_offset, aio->flags); + + + CUSFS_DQ_NODE(cufs->head_act,cufs->tail_act, aio,act_prev,act_next); + + if ((aio->flags & CFLSH_USFS_AIO_E_AIOCB) || + (aio->flags & CFLSH_USFS_AIO_E_AIOCB64)) { + + /* + * This aio is associated with a POSIX AIO + * request. So queue it to the completion + * queue. + */ + + error_code = 0; + ret_code = -1; + + switch(aio->cblk_status.status) { + case CBLK_ARW_STATUS_PENDING: + error_code = EINPROGRESS; + ret_code = -1; + break; + case CBLK_ARW_STATUS_SUCCESS: + + //?? This does not handle situation where one aiocb is split across multiple AIOs. + error_code = 0; + ret_code = aio->aio_nbytes; + + break; + case CBLK_ARW_STATUS_INVALID: + + error_code = EINVAL; + ret_code = -1; + break; + case CBLK_ARW_STATUS_FAIL: + + error_code = aio->cblk_status.fail_errno; + ret_code = -1; + break; + + } + + + if (aio->flags & CFLSH_USFS_AIO_E_AIOCB64) { + + aiocbp64 = aio->aiocbp; + + + /* + * Set errno and return code + * in aiocb here. + */ + + + CUSFS_TRACE_LOG_FILE(9,"aio %p completed for start_lba 0x%llx with status 0x%x for aio_offset = 0x%llx, aio->aiocbp = %p", + aio,aio->start_lba,aio->cblk_status.status,aio->aio_offset, aio->aiocbp); + + + CUSFS_RET_AIOCB(aiocbp64,ret_code,error_code); + + } else { + + + aiocbp = (struct aiocb *)aio->aiocbp; + + + /* + * Set errno and return code + * in aiocb here. + */ + + + CUSFS_TRACE_LOG_FILE(9,"aio %p completed for start_lba 0x%llx with status 0x%x for aio_offset = 0x%llx, aio->aiocbp = %p", + aio,aio->start_lba,aio->cblk_status.status,aio->aio_offset, aio->aiocbp); + + + CUSFS_RET_AIOCB(aiocbp,ret_code,error_code); + + } + + if (aio->aio_sigevent.sigev_notify == SIGEV_NONE) { + + /* + * Do not send any signal. + */ + + CUSFS_TRACE_LOG_FILE(6,"d_queue aio %p for start_lba 0x%llx with status 0x%x of on file = %s", + aio,aio->start_lba,aio->cblk_status.status,aio->file->filename); + + } else if (aio->aio_sigevent.sigev_notify == SIGEV_SIGNAL) { + + /* + * Send signal to process, which is using this + * library, which is this process. + */ + + + if (aio->aio_sigevent.sigev_signo) { + rc = sigqueue(getpid(),aio->aio_sigevent.sigev_signo, + aio->aio_sigevent.sigev_value); + + if (rc) { + + CUSFS_TRACE_LOG_FILE(1,"signal 0x%x failed sig_value = 0x%x, rc %d, errno = %d", + aio->aio_sigevent.sigev_signo, + aio->aio_sigevent.sigev_value, + rc,errno); + + CUSFS_TRACE_LOG_FILE(1,"signal failed for aio %p completed for start_lba 0x%llx with status 0x%x of on file = %s", + aio,aio->start_lba,aio->cblk_status.status,aio->file->filename); + + + } + + rc = 0; + + } else { + + CUSFS_TRACE_LOG_FILE(1,"signal is 0 for filename %s", + aio->file->filename); + } + + + } else if (aio->aio_sigevent.sigev_notify == SIGEV_THREAD) { + + /* + * Send signal to thread, which is using this + * library, which is this process. + */ + + if (aio->aio_sigevent.sigev_notify_attributes) { + + thread_attr = (pthread_attr_t *)aio->aio_sigevent.sigev_notify_attributes; + } + + if (aio->aio_sigevent.sigev_notify_function) { + pthread_rc = pthread_create(&(thread_id),thread_attr, + (void *(*)(void *))(aio->aio_sigevent.sigev_notify_function), + (void *)&(aio->aio_sigevent.sigev_value)); + + + if (pthread_rc) { + + cufs->stats.num_failed_threads++; + + CUSFS_TRACE_LOG_FILE(5,"pthread_create failed rc = %d,errno = %d, thread_attr = %p", + pthread_rc,errno, thread_attr); + } + + } else { + + + CUSFS_TRACE_LOG_FILE(1,"Null sig_notify_function for filename %s", + aio->file->filename); + } + + + } + + } + + file = aio->file; + + if (file) { + CUSFS_LOCK(file->lock); + } + + cusfs_free_aio(cufs,aio,0); + + if (file) { + CUSFS_UNLOCK(file->lock); + } + + } + + aio = next_aio; + + + +#else + + CUSFS_UNLOCK(cufs->lock); + + ret_code = cblk_aresult(cufs->chunk_id,&tag,&status, + (CBLK_ARESULT_BLOCKING|CBLK_ARESULT_NEXT_TAG|CBLK_ARESULT_USER_TAG)); + + error_code = errno; + + CUSFS_LOCK(cufs->lock); + + + if (ret_code < 1) { + /* + * This should not happen + */ + + CUSFS_TRACE_LOG_FILE(1,"cblk_aresult failed rc = %d,errno = %d", + ret_code,errno); + + return -1; + + } + + aio = &cufs->async_pool[tag]; + + if ((aio == NULL) || + !(aio->flags & CFLSH_USFS_AIO_E_ISSUED)) { + + /* + * This should not happen + */ + + CUSFS_TRACE_LOG_FILE(1,"invalid aio command returned tag = %d, aio = %p", + tag,aio); + + return -1; + + } + + /* + * AIO completed with success/error. Lets process it + * now. First dequue it from active list. + */ + + aio->flags &= ~CFLSH_USFS_AIO_E_ISSUED; + + + + + CUSFS_TRACE_LOG_FILE(6,"aio %p completed for start_lba 0x%llx with status 0x%x for aio_offset = 0x%llx, aio->flags = 0x%x", + aio,aio->start_lba,aio->cblk_status.status,aio->aio_offset, aio->flags); + + + CUSFS_DQ_NODE(cufs->head_act,cufs->tail_act, aio,act_prev,act_next); + + if ((aio->flags & CFLSH_USFS_AIO_E_AIOCB) || + (aio->flags & CFLSH_USFS_AIO_E_AIOCB64)) { + + /* + * This aio is associated with a POSIX AIO + * request. So queue it to the completion + * queue. + */ + + if (aio->flags & CFLSH_USFS_AIO_E_AIOCB64) { + + aiocbp64 = aio->aiocbp; + + + /* + * Set errno and return code + * in aiocb here. + */ + + + CUSFS_TRACE_LOG_FILE(9,"aio %p completed for start_lba 0x%llx with status 0x%x for aio_offset = 0x%llx, aio->aiocbp = %p", + aio,aio->start_lba,aio->cblk_status.status,aio->aio_offset, aio->aiocbp); + + + CUSFS_RET_AIOCB(aiocbp64,ret_code,error_code); + + } else { + + + aiocbp = (struct aiocb *)aio->aiocbp; + + + /* + * Set errno and return code + * in aiocb here. + */ + + + CUSFS_TRACE_LOG_FILE(9,"aio %p completed for start_lba 0x%llx with status 0x%x for aio_offset = 0x%llx, aio->aiocbp = %p", + aio,aio->start_lba,aio->cblk_status.status,aio->aio_offset, aio->aiocbp); + + + CUSFS_RET_AIOCB(aiocbp,ret_code,error_code); + + } + + if (aio->aio_sigevent.sigev_notify == SIGEV_NONE) { + + /* + * Do not send any signal. + */ + + CUSFS_TRACE_LOG_FILE(6,"d_queue aio %p for start_lba 0x%llx with status 0x%x of on file = %s", + aio,aio->start_lba,aio->cblk_status.status,aio->file->filename); + + } else if (aio->aio_sigevent.sigev_notify == SIGEV_SIGNAL) { + + /* + * Send signal to process, which is using this + * library, which is this process. + */ + + + if (aio->aio_sigevent.sigev_signo) { + rc = sigqueue(getpid(),aio->aio_sigevent.sigev_signo, + aio->aio_sigevent.sigev_value); + + if (rc) { + + CUSFS_TRACE_LOG_FILE(1,"signal 0x%x failed sig_value = 0x%x, rc %d, errno = %d", + aio->aio_sigevent.sigev_signo, + aio->aio_sigevent.sigev_value, + rc,errno); + + CUSFS_TRACE_LOG_FILE(1,"signal failed for aio %p completed for start_lba 0x%llx with status 0x%x of on file = %s", + aio,aio->start_lba,aio->cblk_status.status,aio->file->filename); + + + } + + rc = 0; + + } else { + + CUSFS_TRACE_LOG_FILE(1,"signal is 0 for filename %s", + aio->file->filename); + } + + + } else if (aio->aio_sigevent.sigev_notify == SIGEV_THREAD) { + + /* + * Send signal to thread, which is using this + * library, which is this process. + */ + + if (aio->aio_sigevent.sigev_notify_attributes) { + + thread_attr = (pthread_attr_t *)aio->aio_sigevent.sigev_notify_attributes; + } + + if (aio->aio_sigevent.sigev_notify_function) { + pthread_rc = pthread_create(&(thread_id),thread_attr, + (void *(*)(void *))(aio->aio_sigevent.sigev_notify_function), + (void *)&(aio->aio_sigevent.sigev_value)); + + + if (pthread_rc) { + + cufs->stats.num_failed_threads++; + + CUSFS_TRACE_LOG_FILE(5,"pthread_create failed rc = %d,errno = %d, thread_attr = %p", + pthread_rc,errno, thread_attr); + } + + } else { + + + CUSFS_TRACE_LOG_FILE(1,"Null sig_notify_function for filename %s", + aio->file->filename); + } + + + } + + } + + file = aio->file; + + if (file) { + CUSFS_LOCK(file->lock); + } + + cusfs_free_aio(cufs,aio,0); + + if (file) { + CUSFS_UNLOCK(file->lock); + } + + + aio = cufs->head_act; + +#endif /* _CFLSH_USFS_ARESULT */ + + } /* while */ + + + return rc; +} + +/* + * NAME: cusfs_cleanup_all_completed_aios_for_file. + * + * FUNCTION: Clean up all completed aios for file on + * assumption no one will call aio_return for them + * (such as close time). + * + * NOTES: This routine assumes the caller has the file->lock. + * + * + * INPUTS: + * + * + * RETURNS: None + * + * + */ +void cusfs_cleanup_all_completed_aios_for_file(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, int flags) +{ + cflsh_usfs_aio_entry_t *aio; + cflsh_usfs_aio_entry_t *next_aio; + + + + aio = file->head_act; + + while (aio) { + + + next_aio = aio->file_next; + + CUSFS_TRACE_LOG_FILE(6,"aio %p for start_lba 0x%llx with status 0x%x of on file = %s", + aio,aio->start_lba,aio->cblk_status.status,aio->file->filename); + + + + + + if (aio->cblk_status.status != CBLK_ARW_STATUS_PENDING) { + + /* + * AIO completed with success/error. Lets process it + * now. First dequue it from active list. + */ + + aio->flags &= ~CFLSH_USFS_AIO_E_ISSUED; + + CUSFS_TRACE_LOG_FILE(6,"aio %p completed for start_lba 0x%llx with status 0x%x of on file = %s", + aio,aio->start_lba,aio->cblk_status.status,aio->file->filename); + + + CUSFS_DQ_NODE(cufs->head_act,cufs->tail_act, aio,act_prev,act_next); + + + /* + * free_aio removes it from file->head_act list and cufs->head_complt list. + */ + + cusfs_free_aio(cufs,aio,0); + } + + + aio = next_aio; + + } /* while */ + + return; +} + + +/* + * NAME: cusfs_wait_for_aio_for_file + * + * FUNCTION: Wait for all Async I/O for this file. + * + * NOTES: This routine assumes the caller has the file->lock. + * + * + * INPUTS: + * + * + * RETURNS: 0 - success, Otherwise error. + * + * + */ +int cusfs_wait_for_aio_for_file(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, int flags) +{ + int rc = 0; + int retry = 0; + cflsh_usfs_aio_entry_t *aio; + int found_issued_aio; + + if (flags & CFLSH_USFS_ONLY_ASYNC_CMPLT) { + + CUSFS_TRACE_LOG_FILE(9,"aio = %p, for disk %s, num_act_areads = 0x%x, num_act_awrites = 0x%x", + file->head_act, cufs->device_name,cufs->stats.num_act_areads,cufs->stats.num_act_awrites); + + while ((file->head_act) && + (retry < CFLSH_USFS_WAIT_AIO_RETRY)) { + + aio = file->head_act; + found_issued_aio = FALSE; + while (aio) { + + if (aio->flags & CFLSH_USFS_AIO_E_ISSUED) { + CUSFS_TRACE_LOG_FILE(6,"Found aio %p for completionfor start_lba 0x%llx with status 0x%x of on file = %s", + aio,aio->start_lba,aio->cblk_status.status,aio->file->filename); + found_issued_aio = TRUE; + break; + } + + aio = aio->file_next; + + } /* while */ + + + if (!found_issued_aio) { + + break; + } else { + + CUSFS_UNLOCK(file->lock); + usleep(CFLSH_USFS_WAIT_AIO_DELAY); + CUSFS_LOCK(file->lock); + retry++; + } + } /* outer while */ + + if (retry == CFLSH_USFS_WAIT_AIO_RETRY) { + + CUSFS_TRACE_LOG_FILE(1,"exceeded wait time for disk %s, num_act_areads = 0x%x, num_act_awrites = 0x%x", + cufs->device_name,cufs->stats.num_act_areads,cufs->stats.num_act_awrites); + errno = EBUSY; + rc = -1; + } + + } else { + + CUSFS_TRACE_LOG_FILE(9,"aio = %p, for disk %s, num_act_areads = 0x%x, num_act_awrites = 0x%x", + file->head_act, cufs->device_name,cufs->stats.num_act_areads,cufs->stats.num_act_awrites); + + while ((file->head_act) && + (retry < CFLSH_USFS_WAIT_AIO_RETRY)) { + + + CUSFS_UNLOCK(file->lock); + usleep(CFLSH_USFS_WAIT_AIO_DELAY); + CUSFS_LOCK(file->lock); + cusfs_cleanup_all_completed_aios_for_file(cufs,file,0); + + retry++; + } + + if (file->head_act) { + + CUSFS_TRACE_LOG_FILE(1,"exceeded wait time for disk %s, num_act_areads = 0x%x, num_act_awrites = 0x%x", + cufs->device_name,cufs->stats.num_act_areads,cufs->stats.num_act_awrites); + + errno = EBUSY; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: cusfs_get_status_all_aio_for_aiocb + * + * FUNCTION: For the specified aiocb64, find all asociated + * aio I/O requests's status. + * + * NOTES: This routine assumes the caller has the file->lock and + * the cufs->lock. + * + * + * INPUTS: + * + * + * RETURNS: Return status of aiocb + * + * + */ +int cusfs_get_status_all_aio_for_aiocb(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, struct aiocb64 *aiocbp, int flags) +{ + int rc = 0; + cflsh_usfs_aio_entry_t *aio; + int num_found_aio = 0; + struct aiocb64 *aio_handle; + + + + + if (aiocbp == NULL) { + + + return EINVAL; + } + +#ifdef _AIX + aio_handle = (struct aiocb64 *)aiocbp->aio_handle; +#else + aio_handle = (struct aiocb64 *)aiocbp->__next_prio; +#endif /* !_AIX */ + + aio = file->head_act; + + while (aio) { + + + if ((aio_handle == aio->aiocbp) && + (aiocbp->aio_offset == aio->aio_offset) && + (aiocbp->aio_nbytes == aio->aio_nbytes)) { + + CUSFS_TRACE_LOG_FILE(6,"Found aio %p for aio_handle %p, start_lba 0x%llx with status 0x%x of on file = %s", + aio,aio_handle,aio->start_lba,aio->cblk_status.status,aio->file->filename); + + num_found_aio++; + + + if ((aio->flags & CFLSH_USFS_AIO_E_ISSUED) && + ((aio->cblk_status.status != CBLK_ARW_STATUS_PENDING))) { + + /* + * AIO completed with success/error. Lets process it + * now. First dequue it from active list. + */ + + aio->flags &= ~CFLSH_USFS_AIO_E_ISSUED; + + CUSFS_TRACE_LOG_FILE(6,"aio %p completed for start_lba 0x%llx with status 0x%x of on file = %s", + aio,aio->start_lba,aio->cblk_status.status,aio->file->filename); + + + CUSFS_DQ_NODE(cufs->head_act,cufs->tail_act, aio,act_prev,act_next); + + /* + * Since this aio is associated with a POSIX AIO + * request, queue it to the completion + * queue. + */ + + + CUSFS_Q_NODE_TAIL(cufs->head_cmplt,cufs->tail_cmplt, + aio,cmplt_prev,cmplt_next); + + } + + /* + * Give EINPROGRESS priority over all + * errors here, since we do not want + * caller to call aio_return until + * everything is done. + */ + + switch(aio->cblk_status.status) { + case CBLK_ARW_STATUS_PENDING: + rc = EINPROGRESS; + break; + case CBLK_ARW_STATUS_SUCCESS: + if (!rc) { + rc = 0; + } + break; + case CBLK_ARW_STATUS_INVALID: + CUSFS_TRACE_LOG_FILE(1,"Invalid status aio %p for start_lba 0x%llx with status 0x%x fail_errno = %d of on file = %s", + aio,aio->start_lba,aio->cblk_status.status, aio->cblk_status.fail_errno,aio->file->filename); + if (rc != EINPROGRESS) { + rc = EINVAL; + } + break; + case CBLK_ARW_STATUS_FAIL: + if (rc != EINPROGRESS) { + rc = aio->cblk_status.fail_errno; + } + break; + + } + } + + aio = aio->file_next; + } + + + + + if (num_found_aio == 0) { + + CUSFS_TRACE_LOG_FILE(1,"no file->head_act found %p aio_handle %p, aiocbp->aio_offset = 0x%llx aiocbp->aio_nbytes = 0x%llx", + file->head_act,aio_handle,aiocbp->aio_offset,aiocbp->aio_nbytes); + + aio = file->head_act; + + while (aio) { + + CUSFS_TRACE_LOG_FILE(1,"file act aio %p for start_lba 0x%llx aio_offset = 0x%llx aio_nbytes = 0x%llx with status 0x%x, aiocbp = %p", + aio,aio->start_lba, + aio->aio_offset, + aio->aio_nbytes, + aio->cblk_status.status, + aio->aiocbp); + aio = aio->file_next; + } + + + aio = cufs->head_act; + + while (aio) { + + CUSFS_TRACE_LOG_FILE(1,"cufs act aio %p for start_lba 0x%llx aio_offset = 0x%llx aio_nbytes = 0x%llx with status 0x%x, aiocbp = %p", + aio,aio->start_lba, + aio->aio_offset, + aio->aio_nbytes, + aio->cblk_status.status, + aio->aiocbp); + aio = aio->act_next; + } + + + aio = cufs->head_cmplt; + + + while (aio) { + + CUSFS_TRACE_LOG_FILE(1,"complete act aio %p for start_lba 0x%llx aio_offset = 0x%llx aio_nbytes = 0x%llx with status 0x%x, aiocbp = %p", + aio,aio->start_lba, + aio->aio_offset, + aio->aio_nbytes, + aio->cblk_status.status, + aio->aiocbp); + aio = aio->cmplt_next; + } + + + rc = EINVAL; + } + + CUSFS_TRACE_LOG_FILE(6,"num_found_aio = %d, rc = %d", + num_found_aio,rc); + return rc; +} + + +/* + * NAME: cusfs_return_status_all_aio_for_aiocb + * + * FUNCTION: For the specified aiocb64, find all asociated + * aio I/O requests's completed status. + * + * NOTES: This routine assumes the caller has the file->lock. + * + * + * INPUTS: + * + * + * RETURNS: Return status of aiocb + * + * + */ +int cusfs_return_status_all_aio_for_aiocb(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, struct aiocb64 *aiocbp, int flags) +{ + int rc = 0; + cflsh_usfs_aio_entry_t *aio; + cflsh_usfs_aio_entry_t *next_aio; + int num_found_aio = 0; + struct aiocb64 *aio_handle; + + +#ifdef _AIX + aio_handle = (struct aiocb64 *)aiocbp->aio_handle; +#else + aio_handle = (struct aiocb64 *)aiocbp->__next_prio; +#endif /* !_AIX */ + + aio = file->head_act; + + while (aio) { + + if ((aio_handle == aio->aiocbp) && + (aiocbp->aio_offset == aio->aio_offset) && + (aiocbp->aio_nbytes == aio->aio_nbytes)) { + + CUSFS_TRACE_LOG_FILE(6,"Found aio %p for aio_handle %p start_lba 0x%llx with status 0x%x of on file = %s", + aio,aio_handle,aio->start_lba,aio->cblk_status.status,aio->file->filename); + + num_found_aio++; + + + + if ((aio->flags & CFLSH_USFS_AIO_E_ISSUED) && + ((aio->cblk_status.status != CBLK_ARW_STATUS_PENDING))) { + + /* + * AIO completed with success/error. Lets process it + * now. First dequue it from active list. + */ + + aio->flags &= ~CFLSH_USFS_AIO_E_ISSUED; + + CUSFS_TRACE_LOG_FILE(6,"aio %p completed for start_lba 0x%llx with status 0x%x of on file = %s", + aio,aio->start_lba,aio->cblk_status.status,aio->file->filename); + + + CUSFS_DQ_NODE(cufs->head_act,cufs->tail_act, aio,act_prev,act_next); + + /* + * Since this aio is associated with a POSIX AIO + * request, queue it to the completion + * queue. + */ + + + CUSFS_Q_NODE_TAIL(cufs->head_cmplt,cufs->tail_cmplt, + aio,cmplt_prev,cmplt_next); + } + + /* + * Give EINPROGRESS priority over all + * errors here, since we do not want + * caller to call aio_return until + * everything is done. + */ + + switch(aio->cblk_status.status) { + case CBLK_ARW_STATUS_PENDING: + rc = -EINPROGRESS; + break; + case CBLK_ARW_STATUS_SUCCESS: + if (!rc) { + rc = aiocbp->aio_nbytes; + } + break; + case CBLK_ARW_STATUS_INVALID: + if (rc != -EINPROGRESS) { + rc = -EINVAL; + } + break; + case CBLK_ARW_STATUS_FAIL: + if (rc != -EINPROGRESS) { + rc = -aio->cblk_status.fail_errno; + } + break; + + } + + + /* + * Set errno and return code + * in aiocb here. + */ + + if (rc < 0) { + + CUSFS_RET_AIOCB(aiocbp,-1,-rc); + } else { + + CUSFS_RET_AIOCB(aiocbp,rc,0); + + } + + } + + aio = aio->file_next; + } /* while */ + + + if (num_found_aio == 0) { + + rc = EINVAL; + } else if (rc != EINPROGRESS) { + /* + * if there are one or more + * aio's associated with this aiocb + * and they are all completed. + * Then free each of them up now. + */ + + aio = file->head_act; + + while (aio) { + + next_aio = aio->file_next; + + if ((aio_handle == aio->aiocbp) && + (aiocbp->aio_offset == aio->aio_offset) && + (aiocbp->aio_nbytes == aio->aio_nbytes)) { + + CUSFS_TRACE_LOG_FILE(6,"Found aio %p aio_handle %p for completionfor start_lba 0x%llx with status 0x%x aio->aiocbp %p", + aio,aio_handle,aio->start_lba,aio->cblk_status.status,aio->aiocbp); + + + + + if (!(aio->flags & CFLSH_USFS_AIO_E_ISSUED) && + ((aio->cblk_status.status != CBLK_ARW_STATUS_PENDING))) { + + /* + * Command completed, process it. + */ + CUSFS_DQ_NODE(cufs->head_cmplt,cufs->tail_cmplt, aio,cmplt_prev,cmplt_next); + + cusfs_free_aio(cufs,aio,0); + } + + } + + aio = next_aio; + + } /* while */ + } + + CUSFS_TRACE_LOG_FILE(6,"num_found_aio = %d, rc = %d", + num_found_aio,rc); + return rc; +} + + +/* + * NAME: cusfs_start_aio_thread_for_cufs + * + * FUNCTION: Start a background thread for this filesystem + * to process async I/O completions. + * + * NOTES: This routine assumes the caller + * is holding both the cufs lock. + * + * + * INPUTS: + * + * + * RETURNS: 0 - success, Otherwise error. + * + * + */ +int cusfs_start_aio_thread_for_cufs(cflsh_usfs_t *cufs, int flags) +{ + int rc = 0; + int pthread_rc; + cflsh_usfs_async_thread_t *async_data; + + + pthread_rc = pthread_cond_init(&(cufs->thread_event),NULL); + + if (pthread_rc) { + + CUSFS_TRACE_LOG_FILE(1,"pthread_cond_init failed for thread_event rc = %d errno= %d", + pthread_rc,errno); + + + errno = EAGAIN; + return -1; + + } + + async_data = &(cufs->async_data); + async_data->cufs = cufs; + + pthread_rc = pthread_create(&(cufs->thread_id),NULL,cusfs_aio_complete_thread,async_data); + + if (pthread_rc) { + + cufs->stats.num_failed_threads++; + + CUSFS_TRACE_LOG_FILE(5,"pthread_create failed rc = %d,errno = %d", + pthread_rc,errno); + + errno = EAGAIN; + return -1; + } else { + cufs->stats.num_success_threads++; + + cufs->stats.num_active_threads++; + } + + + return rc; +} + +/* + * NAME: cusfs_stop_aio_thread_for_cufs + * + * FUNCTION: Stop the background thread for this file + * system. + * + * NOTES: This routine assumes the caller + * is holding both the cufs lock. + * + * + * INPUTS: + * + * + * RETURNS: None + * + * + */ +void cusfs_stop_aio_thread_for_cufs(cflsh_usfs_t *cufs, int flags) +{ + int pthread_rc; + + + + cufs->thread_flags |= CFLSH_CUFS_EXIT_ASYNC; + + pthread_rc = pthread_cond_signal(&(cufs->thread_event)); + + if (pthread_rc) { + + CUSFS_TRACE_LOG_FILE(5,"pthread_cond_signal failed rc = %d,errno = %d", + pthread_rc,errno); + } + + /* + * Since we are going to do pthread_join we need to unlock here. + */ + + CUSFS_UNLOCK(cufs->lock); + + pthread_join(cufs->thread_id,NULL); + + CUSFS_LOCK(cufs->lock); + + cufs->stats.num_active_threads--; + + cufs->thread_flags &= ~CFLSH_CUFS_EXIT_ASYNC; + + + return; +} + +/* + * NAME: cusfs_aio_complete_thread + * + * FUNCTION: This routine is invoked as background + * thread to process async I/O events + * for this cufs. + * + * + * Environment: This routine assumes the cufs mutex + * lock is held by the caller. + * + * INPUTS: + * data - of type cflsh_usfs_async_thread_t + * + * RETURNS: + * + */ +void *cusfs_aio_complete_thread(void *data) +{ + void *ret_code = NULL; + cflsh_usfs_async_thread_t *async_data = data; + int pthread_rc = 0; + cflsh_usfs_t *cufs; + + + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,NULL); + + cufs = async_data->cufs; + + if (cufs == NULL) { + + CUSFS_TRACE_LOG_FILE(5,"cufs filename is NULL"); + + return (ret_code); + } + + if (CFLSH_EYECATCH_FS(cufs)) { + /* + * Invalid cufs. Exit now. + */ + + //??cflsh_blk.num_bad_cufs_ids++; + CUSFS_TRACE_LOG_FILE(1,"cufs filename = %s pthread_cond_wait failed rc = %d errno = %d", + async_data->cufs->device_name,pthread_rc,errno); + + return (ret_code); + } + + CUSFS_TRACE_LOG_FILE(5,"start of thread cufs->device_name = %d",cufs->device_name); + + while (TRUE) { + + CUSFS_LOCK(cufs->lock); + + if (CFLSH_EYECATCH_FS(cufs)) { + /* + * Invalid cufs. Exit now. + */ + + //??cflsh_blk.num_bad_cufs_ids++; + CUSFS_TRACE_LOG_FILE(1,"cufs filename = %s invalid cufs eye catcher 0x%x", + async_data->cufs->device_name,cufs->eyec); + + CUSFS_LIVE_DUMP_THRESHOLD(9,"0x20d"); + CUSFS_UNLOCK(cufs->lock); + return (ret_code); + } + + if (!(cufs->thread_flags) && + (cufs->head_act == NULL)) { + + /* + * Only wait if the thread_flags + * has not been set and there are no active commands + */ + pthread_rc = pthread_cond_wait(&(cufs->thread_event),&(cufs->lock.plock)); + + if (pthread_rc) { + + CUSFS_TRACE_LOG_FILE(5,"cufs filename = %s, pthread_cond_wait failed rc = %d errno = %d", + cufs->device_name,pthread_rc,errno); + CUSFS_UNLOCK(cufs->lock); + return (ret_code); + } + + } + + + CUSFS_TRACE_LOG_FILE(9,"cufs filename = %s thread_flags = %d ", + cufs->device_name,cufs->thread_flags); + + if (cufs->thread_flags & CFLSH_CUFS_EXIT_ASYNC) { + + + + CUSFS_TRACE_LOG_FILE(5,"exiting thread: cufs->device_name = %s thread_flags = %d", + cufs->device_name,cufs->thread_flags); + + CUSFS_UNLOCK(cufs->lock); + break; + } + + + cusfs_check_active_aio_complete(cufs,0); + + //?? Should this sleep and process peridically if commands found, but not complete. + + CUSFS_UNLOCK(cufs->lock); + +#ifdef _REMOVE + if (cufs->head_act) { + usleep(CFLSH_USFS_WAIT_AIO_BG_DLY); + } +#endif + } /* while */ + + return ret_code; + +} + + + +/* + * NAME: cusfs_access_fs + * + * FUNCTION: Access an existing filesystem + * + * + * NOTE: This routine assumes the caller + * has the cusfs_global.global_lock. + + * + * INPUTS: + * NONE + * + * RETURNS: Null failure, otherwise success + * + * + */ + +cflsh_usfs_t *cusfs_access_fs(char *device_name, int flags) +{ + cflsh_usfs_t *cufs; + + + if (device_name == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"Null device_name"); + + errno = EINVAL; + return NULL; + } + + cufs = cusfs_get_cufs(device_name,0); + + if (cufs == NULL) { + + return NULL; + } + + if (cusfs_validate_superblock(cufs,0)) { + + + cusfs_release_cufs(cufs,0); + return NULL; + } + + + + /* + * Get root directory + * + * NOTE: cusfs_lookup_data_obj, queues + * root directory file into cufs' + * filelist and set the root_directory + * pointer to it. + */ + + if (cusfs_lookup_data_obj(cufs,"/",0) == NULL) { + + cusfs_release_cufs(cufs,0); + return NULL; + } + + + + return cufs; +} + + +/* + * NAME: cusfs_add_file_to_hash + * + * FUNCTION: Add file to global hash table + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure + * + * + */ +void cusfs_add_file_to_hash(cflsh_usfs_data_obj_t *file, int flags) +{ + cflsh_usfs_data_obj_t *tmp_file = NULL; + + if (file == NULL) { + + return; + } + + + if (file->flags & CFLSH_USFS_IN_HASH) { + + /* + * Already in hash table + */ + + return; + } + + + if (cusfs_global.file_hash[file->index & (MAX_NUM_CUSFS_FILE_HASH-1)] == NULL) { + + cusfs_global.file_hash[file->index & (MAX_NUM_CUSFS_FILE_HASH-1)] = file; + + file->flags |= CFLSH_USFS_IN_HASH; + + } else { + + tmp_file = cusfs_global.file_hash[file->index & (MAX_NUM_CUSFS_FILE_HASH-1)]; + + while (tmp_file) { + + CUSFS_TRACE_LOG_FILE(9,"tmp_file address = 0x%p, file address = 0x%p, hash[] = 0x%p index = 0x%x", + tmp_file, file,cusfs_global.hash[file->index & (MAX_NUM_CUSFS_FILE_HASH-1)], + (file->index & (MAX_NUM_CUSFS_FILE_HASH-1))); + + + if ((ulong)tmp_file & CUSFS_BAD_ADDR_MASK) { + + + cusfs_global.num_bad_file_ids++; + + + + CUSFS_TRACE_LOG_FILE(1,"Corrupted file address = 0x%p, hash[] = 0x%p index = 0x%x", + tmp_file, cusfs_global.hash[file->index & (MAX_NUM_CUSFS_FILE_HASH-1)], + (file->index & MAX_NUM_CUSFS_FILE_HASH)); + } + + + if (tmp_file == tmp_file->fnext) { + CUSFS_TRACE_LOG_FILE(1,"tmp_file = 0x%p points to itself, hash[] = 0x%p index = 0x%x", + tmp_file, cusfs_global.hash[file->index & (MAX_NUM_CUSFS_FILE_HASH-1)], + (file->index & MAX_NUM_CUSFS_FILE_HASH)); + + break; + } + + + + if (tmp_file->fnext == NULL) { + + tmp_file->fnext = file; + + file->fprev = tmp_file; + + file->flags |= CFLSH_USFS_IN_HASH; + + break; + + } + + tmp_file = tmp_file->fnext; + } + } + return; + +} + +/* + * NAME: cusfs_remove_file_from_hash + * + * FUNCTION: Remove file to global hash table + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure + * + * + */ +void cusfs_remove_file_from_hash(cflsh_usfs_data_obj_t *file, int flags) +{ + if (file == NULL) { + + return; + } + + if (file->fprev) { + file->fprev->fnext = file->fnext; + + } else { + + cusfs_global.file_hash[file->index & (MAX_NUM_CUSFS_FILE_HASH-1)] = file->fnext; + } + + if (file->fnext) { + file->fnext->fprev = file->fprev; + } + + + file->flags &= ~CFLSH_USFS_IN_HASH; + + return; + +} + +/* + * NAME: cusfs_find_data_obj_from_inode + * + * FUNCTION: This routine finds existing + * data objects for the specified filesystem + * from inode number. + * + * + * NOTE: + * + * + * INPUTS: + * + * + * RETURNS: None + * + */ +cflsh_usfs_data_obj_t *cusfs_find_data_obj_from_inode(cflsh_usfs_t *cufs, uint64_t inode_num, int flags) +{ + cflsh_usfs_data_obj_t *file = NULL; + + + file = cufs->filelist_head; + + while (file) { + + if (file->inode->index == inode_num) { + + break; + } + + file = file->next; + } + + + return file; +} + +/* + * NAME: cusfs_find_data_obj_from_filename + * + * FUNCTION: This routine finds existing + * data objects for the specified filesystem + * from filename. + * + * + * NOTE: + * + * + * INPUTS: + * + * + * RETURNS: None + * + */ +cflsh_usfs_data_obj_t *cusfs_find_data_obj_from_filename(cflsh_usfs_t *cufs, char *filename, int flags) +{ + cflsh_usfs_data_obj_t *file = NULL; + + + + CUSFS_TRACE_LOG_FILE(9,"filename = %s",filename); + + file = cufs->filelist_head; + + while (file) { + + + CUSFS_TRACE_LOG_FILE(9,"file->filename = %s",file->filename); + if (!(strcmp(file->filename,filename))) { + + break; + } + + file = file->next; + } + + CUSFS_TRACE_LOG_FILE(9,"file->filename = %s", + ((file != NULL) ? file->filename:NULL)); + return file; +} + +/* + * NAME: cusfs_rdwr_data_object + * + * FUNCTION: This routine will read/write the data blocks + * for the specified inode into the + * supplied buffer + * + * + * NOTE: + * + * + * INPUTS: + * + * + * RETURNS: 0 success, + * + */ +int cusfs_rdwr_data_object(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, void *buf, int *buf_size, int flags) +{ + int rc = 0; + cflsh_usfs_disk_lba_e_t *disk_lbas; + uint64_t num_disk_lbas; + uint64_t offset = 0; + size_t nblocks; + size_t bytes_transferred = 0; + size_t ret_code; + char *bptr; + int i; + int exit_loop = FALSE; + + + if (cufs == NULL) { + + + errno = EINVAL; + return -1; + } + + if (inode == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Inode pointer is null for device %s", + cufs->device_name); + errno = EINVAL; + return -1; + } + +#ifdef _REMOVE + + //?? Allow one to specify a buffer smaller than the file size? + if (inode->file_size > *buf_size) { + + CUSFS_TRACE_LOG_FILE(1,"Inode file_size 0x%llx is larger then buf_size 0x%llxfor device %s", + inode->file_size,buf_size,cufs->device_name); + errno = EINVAL; + return -1; + } + +#endif /* _REMOVE */ + + + if ((*buf_size) % cufs->disk_block_size) { + + /* + * ?? TODO: Don't support internally allocating a buffer + * to the end of the sector and returning data back to the + * caller. + */ + + CUSFS_TRACE_LOG_FILE(1,"Data buffer of size 0x%llx does not end on sector boundary for device %s", + *buf_size,cufs->device_name); + + return -1; + + } + + + if (cusfs_get_disk_lbas_from_inode_ptr_for_transfer(cufs, inode, offset, + *buf_size, &disk_lbas,&num_disk_lbas,0)) { + + CUSFS_TRACE_LOG_FILE(1,"For inode of type %d on disk %s failed to get inode pointers offset = 0x%llx", + inode->type,cufs->device_name,offset); + + + return -1; + + } + + bptr = buf; + + bytes_transferred = 0; + + for (i = 0; i < num_disk_lbas; i++) { + + nblocks = MIN(disk_lbas[i].num_lbas,cufs->max_xfer_size); + + nblocks = MIN(nblocks,((*buf_size - bytes_transferred) / cufs->disk_block_size)); + + + + while (nblocks) { + + if (flags & CFLSH_USFS_RDWR_WRITE) { + ret_code = cblk_write(cufs->chunk_id,bptr,disk_lbas[i].start_lba,nblocks,0); + } else { + ret_code = cblk_read(cufs->chunk_id,bptr,disk_lbas[i].start_lba,nblocks,0); + } + + if (ret_code < 0) { + + exit_loop = TRUE; + rc = -1; + break; + } + + bptr += ret_code * cufs->disk_block_size; + + nblocks = MIN((disk_lbas[i].num_lbas - ret_code),cufs->max_xfer_size); + + bytes_transferred += ret_code * cufs->disk_block_size; + if (ret_code < nblocks) { + + /* + * Give up on future reads, + * but return good status with + * the number of bytes read. + */ + + exit_loop = TRUE; + break; + } + + } /* while */ + + + if (bytes_transferred == *buf_size) { + + break; + } + + if (exit_loop) { + + break; + } + + } /* for */ + + *buf_size = bytes_transferred; + + return rc; +} + +/* + * NAME: cusfs_read_data_object + * + * FUNCTION: This routine will read the data blocks + * for the specified inode into the + * supplied buffer + * + * + * NOTE: + * + * + * INPUTS: + * + * + * RETURNS: 0 success, + * + */ +int cusfs_read_data_object(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, void *buf, int *buf_size, int flags) +{ + + return (cusfs_rdwr_data_object(cufs,inode,buf,buf_size,flags)); +} + +/* + * NAME: cusfs_read_data_obj_for_inode_no + * + * FUNCTION: This routine will read the data blocks + * for the specified inode number into the + * supplied buffer + * + * + * NOTE: + * + * + * INPUTS: + * + * + * RETURNS: 0 success, + * + */ +int cusfs_read_data_object_for_inode_no(cflsh_usfs_t *cufs, uint64_t inode_num, + void **buf, int *buf_size, int flags) +{ + int rc = 0; + cflsh_usfs_inode_t *inode; + void *lbuf; + int lbuf_size; + + if (cufs == NULL) { + + + errno = EINVAL; + return -1; + } + + + inode = cusfs_get_inode_from_index(cufs,inode_num,0); + + + if (inode == NULL) { + + return -1; + } + + + lbuf_size = inode->file_size; + + lbuf = malloc(lbuf_size); + + if (lbuf == NULL) { + + CUSFS_TRACE_LOG_FILE(1,"malloc of buf of size 0x%x for failed for disk %s", + lbuf_size,cufs->device_name); + + return -1; + } + + rc = cusfs_read_data_object(cufs, inode, lbuf,&lbuf_size, flags); + + free(inode); + + *buf = lbuf; + + *buf_size = lbuf_size; + + return rc; +} + +/* + * NAME: cusfs_write_data_obj + * + * FUNCTION: This routine will write the data blocks + * for the specified inode into the + * supplied buffer + * + * + * NOTE: + * + * + * INPUTS: + * + * + * RETURNS: 0 success, + * + */ +int cusfs_write_data_object(cflsh_usfs_t *cufs, cflsh_usfs_inode_t *inode, void *buf, int *buf_size, int flags) +{ + + return (cusfs_rdwr_data_object(cufs,inode,buf,buf_size,(flags | CFLSH_USFS_RDWR_WRITE))); +} + + +/* + * NAME: cusfs_lookup_data_obj + * + * FUNCTION: Look up a filesystem data object.. + * + * + * NOTES: This code assumes the pathname specified + * starts with the disk special filename + * followed a ":" and then the + * filename. + * + * INPUTS: + * NONE + * + * RETURNS: Null failure, Otherwise success + * + * + */ + +cflsh_usfs_data_obj_t *cusfs_lookup_data_obj(cflsh_usfs_t *cufs, + char *pathname, int flags) +{ + cflsh_usfs_data_obj_t *file = NULL; + char filename2[PATH_MAX]; + char *local_pathname; + char *filename; + char *name; + uint64_t directory_inode_no; + void *buf; + int buf_size; + + + if (cufs == NULL) { + + + errno = EINVAL; + return NULL; + } + + if (pathname == NULL) { + + + errno = EINVAL; + return NULL; + } + + local_pathname = cusfs_get_only_filename_from_diskpath(pathname,0); + + + + CUSFS_TRACE_LOG_FILE(9,"local_pathname = %s, pathname = %s", + local_pathname,pathname); + + /* + * See if we already have a data object for this + * filename. + */ + + file = cusfs_find_data_obj_from_filename(cufs,local_pathname,0); + + if (file) { + + return file; + } + + /* + * If no data object was found, then search + * through the directories on the disk + * to find one that matches this pathname. + */ + + file = cusfs_alloc_file(cufs,local_pathname,CFLSH_USFS_ALLOC_READ_AHEAD); + + if (file == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"malloc of file %s for failed for disk %s", + local_pathname,cufs->device_name); + return NULL; + } + + if (!strcmp(local_pathname,"/")) { + + /* + * This is the root directory + */ + + + file->flags |= CFLSH_USFS_ROOT_DIR; + + file->inode = cusfs_find_root_inode(cufs,0); + + if (file->inode == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Root directory inode is NULL for %s on disk %s%s", + local_pathname,cufs->device_name); + cusfs_free_file(file); + return NULL; + } + + cufs->root_directory = file; + + } else { + + + strcpy(filename2,local_pathname); + + /* + * Skip past the first slash + */ + filename = filename2; + filename++; + + + + name = strtok(filename,"/"); + + directory_inode_no = cufs->root_directory->inode->index; + + while (name) { + + + buf = NULL; + + if (cusfs_read_data_object_for_inode_no(cufs, directory_inode_no, + &buf, &buf_size,0)) { + + + CUSFS_TRACE_LOG_FILE(5,"Failed to read directory inode = 0x%llx of type %d on disk %s, ", + directory_inode_no, cufs->device_name); + + if (buf) { + free(buf); + } + + free(file->inode); + + cusfs_free_file(file); + + return NULL; + } + + + file->inode = cusfs_find_inode_num_from_directory_by_name(cufs,name,buf,buf_size,0); + + + if (file->inode == NULL) { + + CUSFS_TRACE_LOG_FILE(5,"Failed to find %s in directory inode = 0x%llx on disk %s", + name,directory_inode_no, cufs->device_name); + + free(buf); + cusfs_free_file(file); + + return NULL; + } + + if (file->inode->type == CFLSH_USFS_INODE_FILE_DIRECTORY) { + + + directory_inode_no = file->inode->index; + + name = strtok(NULL,"/"); + + if (name) { + free(file->inode); + } else { + file->file_type = file->inode->type; + + } + } else { + + + file->file_type = file->inode->type; + name = NULL; + } + + + + free(buf); + + } /* while */ + + } + + + + file->data_buf_size = cufs->fs_block_size; + + file->data_buf = malloc(file->data_buf_size); + + + if (file->data_buf == NULL) { + + + if (file->flags & CFLSH_USFS_ROOT_DIR) { + + cufs->root_directory = NULL; + } + + + + cusfs_free_file(file); + + return NULL; + } + + //?? what about cufs->locking here? + + if (file) { + + CUSFS_Q_NODE_TAIL(cufs->filelist_head,cufs->filelist_tail,file,prev,next); + + } + + CUSFS_TRACE_LOG_FILE(5,"Found file = %s of type %d on disk %s, ", + pathname,file->inode->type, cufs->device_name); + + + + return file; +} + + +/* + * NAME: cusfs_find_fs_for_disk + * + * FUNCTION: This routine looks for a cufs entry + * for this process that has the specified + * device_name. If that is not found, then + * it attempts to read the filesystem from the specified + * disk + * + * + * ?? TODO: This code does not handle the linux case + * where different disk special names can + * be paths to the same disk. Thus this code + * could incorrectly have multiple cufs entries + * for a process pointing to the same disk. + * + * For linux maybe this is solved via the + * WWID/udid + * + * + * NOTE: This routine assumes the caller + * has the cusfs_global.global_lock. + * INPUTS: + * NONE + * + * RETURNS: Null failure, otherwise success + * + * + */ +cflsh_usfs_t *cusfs_find_fs_for_disk(char *device_name, int flags) +{ + cflsh_usfs_t *cufs; + int i; + int found = FALSE; + + + for (i=0; i < MAX_NUM_CUSFS_HASH; i++) { + + cufs = cusfs_global.hash[i]; + + while (cufs) { + + if (!strcmp(cufs->device_name,device_name)) { + + found = TRUE; + + break; + } + + cufs = cufs->next; + } + + if (found) { + break; + } + } + + if (cufs == NULL) { + + + cufs = cusfs_access_fs(device_name,0); + + } + + return cufs; +} +/* + * NAME: cusfs_get_only_filename_from_diskpath + * + * FUNCTION: This routine from a concatenated + * disk filename will extract only the + * file path inside filesystem without the disk + * name. If there is no disk path then the full + * disk pathname is returned. + * + * + * NOTES: This code assumes the pathname specified + * starts with the disk special filename + * followed a ":" and then the + * filename. + * + * INPUTS: + * NONE + * + * RETURNS: Null failure, Otherwise success. + * + * + */ +char *cusfs_get_only_filename_from_diskpath(char *disk_pathname, int flags) +{ + char *filename; + + + + if (disk_pathname == NULL) { + + + errno = EINVAL; + return NULL; + } + + + + + filename = strchr(disk_pathname,CFLSH_USFS_DISK_DELIMINATOR); + + if (filename == NULL) { + + return disk_pathname; + } + + filename++; + + + + return filename; + +} + + +/* + * NAME: cusfs_find_fs_from_path + * + * FUNCTION: This routine from a concatenated + * disk filename will get the associated + * filesystem data structure. + * + * + * NOTES: This code assumes the pathname specified + * starts with the disk special filename + * followed a ":" and then the + * filename. + * + * INPUTS: + * NONE + * + * RETURNS: Null failure, Otherwise success. + * + * + */ +cflsh_usfs_t *cusfs_find_fs_from_path(char *disk_pathname, int flags) +{ + char device_name[PATH_MAX]; + char disk_pathname_copy[PATH_MAX]; + char *filename; + cflsh_usfs_t *cufs = NULL; + + + + if (disk_pathname == NULL) { + + + errno = EINVAL; + return NULL; + } + + + /* + * Copy disk_pathname, since we're + * going to modify it. + */ + + strcpy(disk_pathname_copy,disk_pathname); + + filename = strchr(disk_pathname_copy,CFLSH_USFS_DISK_DELIMINATOR); + + if (filename == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Invalid disk_pathname %s", + disk_pathname); + return NULL; + } + + filename[0] = '\0'; + + + + /* + * Extract disk special filename + */ + + strcpy(device_name,disk_pathname_copy); + + cufs = cusfs_find_fs_for_disk(device_name,0); + + return cufs; + +} + +/* + * NAME: cusfs_open_disk_filename + * + * FUNCTION: This routine from a concatenated + * disk filename will open file on + * the specified disk. + * + * + * NOTES: This code assumes the pathname specified + * starts with the disk special filename + * followed a ":" and then the + * filename. + * + * INPUTS: + * NONE + * + * RETURNS: Null failure, Otherwise success. + * + * + */ +cflsh_usfs_data_obj_t *cusfs_open_disk_filename(char *disk_pathname, int flags) +{ + char device_name[PATH_MAX]; + char *filename; + char pathname[PATH_MAX]; + char disk_pathname_copy[PATH_MAX]; + cflsh_usfs_t *cufs = NULL; + cflsh_usfs_data_obj_t *file = NULL; + + + if (disk_pathname == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"disk_pathname is null"); + + errno = EINVAL; + return NULL; + } + + CUSFS_TRACE_LOG_FILE(9,"disk_pathname = %s", + disk_pathname); + + + /* + * Copy disk_pathname, since we're + * going to modify it. + */ + + strcpy(disk_pathname_copy,disk_pathname); + + filename = strchr(disk_pathname_copy,CFLSH_USFS_DISK_DELIMINATOR); + + if (filename == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Invalid disk_pathname %s", + disk_pathname); + return NULL; + } + + filename[0] = '\0'; + + filename++; + + + + /* + * Extract disk special filename + */ + + strcpy(device_name,disk_pathname_copy); + + /* + * Extract file pathname for this filesystem + * contained on this disk. + */ + + strcpy(pathname,filename); + + /* + * ?? TODO: This code does not handle the linux case + * where different disk special names can + * be paths to the same disk. Thus this code + * could incorrectly have multiple cufs entries + * for a process pointing to the same disk. + * + * For linux maybe this is solved via the + * WWID/udid + */ + + + CUSFS_TRACE_LOG_FILE(9,"device_name = %s, filename = %s", + device_name,filename); + + cufs = cusfs_find_fs_for_disk(device_name,0); + + if (cufs == NULL) { + + /* + * If we did not yet access that filesystem, + * then do that now. + */ + + cufs = cusfs_access_fs(device_name,0); + + if (cufs == NULL) { + errno = ENOENT; + + } + + } + + + file = cusfs_lookup_data_obj(cufs,filename,0); + + return file; + +} + + +/* + * NAME: cusfs_flush_inode_timestamps + * + * FUNCTION: Flush any pending inode time stamp + * updates to the disk. + * + * NOTE: This routine assumes one is called under a lock + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 sucess, Otherwise failure + * + * + */ +int cusfs_flush_inode_timestamps(cflsh_usfs_t *cusfs, cflsh_usfs_data_obj_t *file) +{ + + if (file->inode_updates.flags & (CFLSH_USFS_D_INODE_MTIME|CFLSH_USFS_D_INODE_ATIME)) { + + if ((file->inode_updates.flags & CFLSH_USFS_D_INODE_MTIME) && + (file->inode_updates.mtime > file->inode->mtime)) { + + file->inode->mtime = file->inode_updates.mtime; + } + + if ((file->inode_updates.flags & CFLSH_USFS_D_INODE_ATIME) && + (file->inode_updates.atime > file->inode->atime)) { + + file->inode->atime = file->inode_updates.atime; + } + + file->inode_updates.flags &= ~(CFLSH_USFS_D_INODE_MTIME|CFLSH_USFS_D_INODE_ATIME); + cusfs_update_inode(cusfs,file->inode,0); + } + + return 0; + +} + +/* + * NAME: cusfs_get_disk_lbas_for_transfer + * + * FUNCTION: For the specified transfer, get a list of + * disk LBA ranges that are associated. + * + * + * INPUTS: + * + * + * RETURNS: 0 success, otherwise failure + * + * + */ + +int cusfs_get_disk_lbas_for_transfer(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + uint64_t offset, size_t num_bytes, + cflsh_usfs_disk_lba_e_t **disk_lbas, uint64_t *num_disk_lbas, + int flags) +{ + int rc = 0; + int i; + uint64_t start_disk_lba_index; + uint64_t start_lba = 0; + uint64_t local_num_disk_lbas; + uint64_t offset_in_this_fs_lba; + cflsh_usfs_disk_lba_e_t *lba_list; + uint64_t num_lbas=0, num_lbas2=0,orig_num_lbas; + + + if (cufs == NULL) { + + errno = EINVAL; + return -1; + + } + + if (file == NULL) { + + errno = EINVAL; + return -1; + } + + + + start_disk_lba_index = offset/cufs->fs_block_size; + + + + local_num_disk_lbas = num_bytes/cufs->fs_block_size; + + if (num_bytes % cufs->fs_block_size) { + + local_num_disk_lbas++; + } + + if ((offset % cufs->fs_block_size) && + (offset/cufs->fs_block_size != (offset+num_bytes-1)/cufs->fs_block_size)) { + + /* + * If the offset is not aligned on a filesystem block and + * the start and end of the transfer are not on the same + * filesystem block, then we need to span an additional + * filesystem block. + */ + + local_num_disk_lbas++; + } + + if (local_num_disk_lbas > file->num_disk_lbas) { + + errno = EINVAL; + return -1; + } + + lba_list = malloc(sizeof(cflsh_usfs_disk_lba_e_t) * local_num_disk_lbas); + + if (lba_list == NULL) { + + + CUSFS_TRACE_LOG_FILE(1,"Malloc of disk_lbas on disk %s failed with errno %d ", + cufs->device_name,errno); + + return -1; + } + + + CUSFS_TRACE_LOG_FILE(9,"offset = 0x%llx, num_bytes = 0x%llx", + offset,num_bytes); + + /* + * Find start lba entry + */ + + for (i=0; i < local_num_disk_lbas;i++) { + + lba_list[i].num_lbas = file->disk_lbas[start_disk_lba_index + i].num_lbas; + lba_list[i].start_lba = file->disk_lbas[start_disk_lba_index + i].start_lba; + + orig_num_lbas = lba_list[i].num_lbas; + + CUSFS_TRACE_LOG_FILE(9,"lba_list[%d].num_lbas = 0x%llx, start_lba = 0x%llx", + i,lba_list[i].num_lbas,lba_list[i].start_lba); + if (i == 0) { + + offset_in_this_fs_lba = offset % cufs->fs_block_size; + + /* + * Adjust the transfer's start lba to the closest disk lba in this + * fs lba. + */ + + if (offset_in_this_fs_lba) { + + start_lba = lba_list[i].start_lba; + + start_lba += offset_in_this_fs_lba/cufs->disk_block_size; + + lba_list[i].num_lbas -= (start_lba - lba_list[i].start_lba); + + lba_list[i].start_lba = start_lba; + } + + CUSFS_TRACE_LOG_FILE(9,"offset = 0x%llx, offset_in_this_fs_lba = 0x%llx, start_lba = 0x%llx", + offset,offset_in_this_fs_lba,start_lba); + } + + if (i == (local_num_disk_lbas - 1)) { + + offset_in_this_fs_lba = (offset + num_bytes) % cufs->fs_block_size; + + + /* + * Adjust the transfer's end lba to the closest disk lba in this + * fs lba + */ + + if (offset_in_this_fs_lba) { + + num_lbas = orig_num_lbas; + + num_lbas2 = offset_in_this_fs_lba/cufs->disk_block_size; + + if (offset_in_this_fs_lba % cufs->disk_block_size) { + num_lbas2++; + } + + if (num_lbas2 < num_lbas) { + + lba_list[i].num_lbas -= (num_lbas - num_lbas2); + } + } + + CUSFS_TRACE_LOG_FILE(9,"offset_in_this_fs_lba = 0x%llx, num_lbas = 0x%llx, num_lbas2 = 0x%llx", + offset_in_this_fs_lba,num_lbas,num_lbas2); + + + } + + CUSFS_TRACE_LOG_FILE(9,"lba_list[%d].num_lbas = 0x%llx, start_lba = 0x%llx", + i,lba_list[i].num_lbas,lba_list[i].start_lba); + } + + *num_disk_lbas = local_num_disk_lbas; + + *disk_lbas = lba_list; + + CUSFS_TRACE_LOG_FILE(9,"offset = 0x%llx, num_bytes = 0x%llx, *num_disk_lbas = 0x%llx, rc = %d", + offset,num_bytes,*num_disk_lbas,rc); + + return rc; +} + +/* + * NAME: cusfs_seek_read_ahead + * + * FUNCTION: When a seek request is done to the filesystem + * This routine will do a read starting at that sector + * in case the next request is a read. + * + * NOTE: This routine assumes the caller has the file->lock. + * + * INPUTS: + * + * + * RETURNS: None + * + * + */ + +void cusfs_seek_read_ahead(cflsh_usfs_t *cufs, cflsh_usfs_data_obj_t *file, + uint64_t offset, int flags) +{ +#ifdef EXPERIMENTAL + int rc = 0; + int arw_flags = (CBLK_ARW_USER_STATUS_FLAG|CBLK_ARW_WAIT_CMD_FLAGS); + size_t size_transfer; + cflsh_usfs_disk_lba_e_t *disk_lbas; + uint64_t num_disk_lbas; + uint64_t start_offset = offset; + + if ((file->seek_ahead.flags & CFLSH_USFS_D_RH_ACTIVE) && + (file->seek_ahead.cblk_status.status == CBLK_ARW_STATUS_PENDING)) { + + /* + * If the seek read head buffer is in use, then + * give up. + */ + + CUSFS_TRACE_LOG_FILE(1,"Read ahead in use for lba = 0x%llx and nblocks 0x%llx", + file->seek_ahead.start_lba,file->seek_ahead.nblocks); + return; + } + + if ((file->seek_ahead.local_buffer == NULL) || + (file->seek_ahead.local_buffer_len == 0x0LL)) { + + CUSFS_TRACE_LOG_FILE(1,"Read ahead buffer %p is either null or size 0 (0x%llx)", + file->seek_ahead.local_buffer,file->seek_ahead.local_buffer_len); + return; + } + + size_transfer = MIN(cufs->disk_block_size,file->seek_ahead.local_buffer_len); + + + if (offset + size_transfer > file->inode->file_size) { + + + /* + * If doing a read ahead here goes beyond the end of the current + * file, then don't attempt a read ahead. + */ + + CUSFS_TRACE_LOG_FILE(1,"Read ahead is beyond end of disk offset = 0x%llx transfer size = 0x%llx file size = 0x%llx", + offset,size_transfer,file->inode->file_size); + return; + } + + /* + * If we got here, then the seek read ahead is not in use + * so use it now. Get start_offset that is fs block aligned. + */ + + + start_offset &= ~(size_transfer - 1); + + + if (cusfs_get_disk_lbas_for_transfer(cufs, file, start_offset, + size_transfer, &disk_lbas,&num_disk_lbas,0)) { + + CUSFS_TRACE_LOG_FILE(1,"For file = %s on disk %s failed to get inode pointers offset = 0x%llx", + file->filename,cufs->device_name,start_offset); + + + + + return; + + } + + if (num_disk_lbas > 1) { + + /* + * If read ahead spans multiple block, + * then do not do it. + */ + + CUSFS_TRACE_LOG_FILE(6,"Read ahead not attempted file = %s on disk %s num_disk_lbas = 0x%llx", + file->filename,cufs->device_name,num_disk_lbas); + return; + + } + + if (disk_lbas[0].num_lbas < 1) { + + /* + * If read ahead is not at least one sector + */ + + CUSFS_TRACE_LOG_FILE(6,"Read ahead not attempted file = %s on disk %s disk_lbas[0].num_lbas = 0x%llx", + file->filename,cufs->device_name,disk_lbas[0].num_lbas); + return; + + } + + + file->seek_ahead.flags &= ~CFLSH_USFS_D_RH_ACTIVE; + + file->seek_ahead.start_lba = disk_lbas[0].start_lba; + file->seek_ahead.nblocks = MIN(disk_lbas[0].num_lbas,cufs->max_xfer_size);; + bzero(file->seek_ahead.local_buffer,file->seek_ahead.local_buffer_len); + bzero(&(file->seek_ahead.cblk_status),sizeof(file->seek_ahead.cblk_status)); + + /* + * Issue async read operation + */ + + rc = cblk_aread(cufs->chunk_id,file->seek_ahead.local_buffer, + file->seek_ahead.start_lba,file->seek_ahead.nblocks, + &(file->seek_ahead.tag), + &(file->seek_ahead.cblk_status), + arw_flags); + + if (rc >= 0) { + file->seek_ahead.flags |= CFLSH_USFS_D_RH_ACTIVE; + + file->seek_ahead.start_offset = start_offset; + + CUSFS_TRACE_LOG_FILE(9,"Successful issue aread for lba = 0x%llx and nblocks 0x%llx, with ,rc = %d", + file->seek_ahead.start_lba,file->seek_ahead.nblocks,rc); + } else { + + + CUSFS_TRACE_LOG_FILE(1,"Failed to issue aread for lba = 0x%llx and nblocks 0x%llx, with ,errno = %d", + file->seek_ahead.start_lba,file->seek_ahead.nblocks,errno); + } +#endif /* EXPERIMENTAL */ + return; +} + + + + diff --git a/src/usfs/cflsh_usfs_wrapper.c b/src/usfs/cflsh_usfs_wrapper.c new file mode 100644 index 00000000..9ae47ce0 --- /dev/null +++ b/src/usfs/cflsh_usfs_wrapper.c @@ -0,0 +1,4454 @@ +// IBM_PROLOG_BEGIN_TAG +// This is an automatically generated prolog. +// +// $Source: src/usfs/cflsh_usfs_wrapper.c $ +// +// IBM CONFIDENTIAL +// +// COPYRIGHT International Business Machines Corp. 2016 +// +// p1 +// +// Object Code Only (OCO) source materials +// Licensed Internal Code Source Materials +// IBM Surelock Licensed Internal Code +// +// The source code for this program is not published or other- +// wise divested of its trade secrets, irrespective of what has +// been deposited with the U.S. Copyright Office. +// +// Origin: 30 +// +// IBM_PROLOG_END_TAG */ +#ifdef _AIX +static char sccsid[] = "%Z%%M% %I% %W% %G% %U%"; +#endif + +/* + * COMPONENT_NAME: (sysxcflashusfs) CAPI Flash user space filesystem library + * + * FUNCTIONS: + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +/* + * This source file provides shim where all user space common requests + * typically for files or directories are intercepted and then either: + * + * - Passed through to the original interface + * + * - Routed to this library for a user space filesystem. These items + * are determined if a path is provided, since it must have the + * disk special filename prepended onto the filesystem path with a ":" + * separating them. File descriptors returned for these operations (open, + * creat) are saved in a hash table to that subsequent calls with the file + * descriptor can be intercepted and routed to this library. + * + * To leverage this shim one must use an environment variable + * to preload this library first. This environment variable specifies + * the absolute path of this library. For Linux the environment variable + * is LD_PRELOAD. For AIX 32-bit is LDR_PRELOAD and AIX 64-bit LDR_PRELOAD64. + * For AIX the library member functiones of shr.0 or shr_64.o must be specified + * for LDR_PRELOAD or LDR_PROLOAD64, respectively. Thus one one would have + * something like LDR_PRELOAD="/lib/libcflsh_usfs.a(shr.o)" .... The double quotes + * must be used along with the parentheses. + * + * Also for AIX, one must run the following command: chdev -l sys0 -a enhanced_RBAC=false + * and reboot. + * + * NOTES; To determine what wrapper functions may be required for an application one + * can use the ltrace call on linux. For AIX, one should use the truss -u libc.a + * + * + */ + +#define CFLSH_USFS_FILENUM 0x0600 + +#ifndef _AIX +#define _GNU_SOURCE +#endif /* !_AIX */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _AIX +#include +#else +#include +#endif +#if !defined(_AIX) && !defined(_MACOSX) +#include +#include +#include /* For SYS_xxx definitions */ +#endif /* !_AIX && !_MACOSX */ + + +#include "cflsh_usfs_wrapper.h" + +#ifdef _AIX +#include "cflsh_usfs.h" +#include "cflsh_usfs_admin.h" +#else +#include +#include +#endif + +#ifdef _AIX +#define CUSFS_RET_AIOCB(_aiocbp_,_rc_,_errno_) \ +do { \ + \ + (_aiocbp_)->aio_errno = (_errno_); \ + \ + (_aiocbp_)->aio_return = (_rc_); \ + \ +} while (0) + +#else + +#define CUSFS_RET_AIOCB(_aiocbp_,_rc_,_errno_) \ +do { \ + \ + (_aiocbp_)->__error_code = (_errno_); \ + \ + (_aiocbp_)->__return_value = (_rc_); \ + \ +} while (0) + + +#endif /* !_AIX */ + + +cusfs_gwrapper_t cusfs_gwrapper; + +pthread_mutex_t cusfs_gwrapper_lock = PTHREAD_MUTEX_INITIALIZER; + +static int (*real_fcntl)(int fd,int cmd,...) = NULL; + +static int (*real_dup)(int fd) = NULL; + +static int (*real_dup2)(int old_fd, int new_fd) = NULL; + +static int (*real_posix_fadvise)(int fd, off_t offset, off_t len, int advice) = NULL; + +static int (*real_posix_fallocate)(int fd, off_t offset, off_t len) = NULL; + +static int (*real_posix_fallocate64)(int fd, off64_t offset, off64_t len) = NULL; + +#ifndef _AIX +static int (*real_posix_fadvise64)(int fd, off64_t offset, off64_t len, int advice) = NULL; +#endif /* !_AIX */ + +static int (*real_statfs)(char *pathname, struct statfs *statfs) = NULL; + + +static int (*real_statfs64)(char *pathname, struct statfs64 *statfs) = NULL; + + + +static int (*real_mkdir)(char *path,mode_t mode_flags) = NULL; + +static int (*real_rmdir)(char *pathname) = NULL; + +#ifdef _AIX +static DIR64 *(*real_opendir64)(char *directory_name) = NULL; +#else +static DIR *(*real_opendir)(char *directory_name) = NULL; +#endif /* !_AIX */ + +#ifdef _AIX +static int (*real_closedir64)(DIR64 *directory) = NULL; +#else +static int (*real_closedir)(DIR *directory_linux) = NULL; +#endif /* !_AIX */ + +#ifdef _AIX +static int (*real_readdir64_r)(DIR64 *directory, struct dirent64 *entry,struct dirent64 **result) = NULL; +static struct dirent64 *(*real_readdir64)(DIR64 *directory) = NULL; +#else +static int (*real_readdir_r)(DIR *directory_linux, struct dirent *entry,struct dirent **result) = NULL; + +#endif +static struct dirent *(*real_readdir)(DIR *directory) = NULL; + +#ifdef _NOT_YET +#ifdef _AIX +static offset_t (*real_telldir64)(DIR64 *directory) = NULL; +#else +static off_t (*real_telldir)(DIR *directory_linux) = NULL; +#endif + + +#ifdef _AIX +static void (*real_seekdir64)(DIR64 *directory, offset_t location) = NULL; +#else +static void (*real_seekdir)(DIR *directory_linux, off_t location) = NULL; +#endif +#endif /* NOT_YET */ + +static int (*real_rename)(char *from_pathname,char *to_pathname) = NULL; + +static int (*real_creat)(char *path,mode_t mode) = NULL; + +static int (*real_open)(char *path,int flags,...) = NULL; + +static int (*real_close)(int fd) = NULL; + +static ssize_t (*real_read)(int fd, void *buffer,size_t nbytes) = NULL; + +static ssize_t (*real_write)(int fd, void *buffer,size_t nbytes) = NULL; + +static int (*real_fsync)(int fd) = NULL; + +static off_t (*real_lseek)(int fd, off_t Offset, int Whence) = NULL; + +static off64_t (*real_lseek64)(int fd, off64_t Offset, int Whence) = NULL; + +#ifdef _AIX +static offset_t (*real_llseek)(int fd, offset_t Offset, int Whence) = NULL; + +#endif /* AIX */ + + +static int (*real_fstat)(int fd,struct stat *buffer) = NULL; + +#ifdef _AIX +static int (*real_fstatx)(int fd, struct stat *buffer, int length, int command) = NULL; +static int (*real_fstat64)(int fd,struct stat64 *buffer) = NULL; + +#endif /* AIX */ + +#ifdef _AIX +static int (*real_statx)(char *path, struct stat *buffer, int length, int command) = NULL; +static int (*real_stat64)(char *path, struct stat64 *buffer) = NULL; +#else +static int (*real_stat)(char *path, struct stat *buffer) = NULL; +#endif /* !AIX */ + +#ifdef _AIX +static int (*real_fstat64x)(int fd,struct stat64x *buffer) = NULL; +static int (*real_stat64x)(char *path, struct stat64x *buffer) = NULL; + + +#else +static int (*real_xstat)(int version, const char *path, struct stat *buffer) = NULL; +static int (*real_fxstat)(int version, int fd, struct stat *buffer) = NULL; + +static int (*real_xstat64)(int version, const char *path, struct stat64 *buffer) = NULL; +static int (*real_fxstat64)(int version, int fd, struct stat64 *buffer) = NULL; + +#endif /* !_AIX */ + +#ifdef _AIX +static int (*real_readlink)(char *path, char *buffer, size_t buffer_size) = NULL; + +#else +static ssize_t (*real_readlink)(char *path, char *buffer, size_t buffer_size) = NULL; +#endif /* !_AIX */ + +static int (*real_access)(char *path, int mode) = NULL; + +static int (*real_fchmod)(int fd,mode_t mode) = NULL; + +static int (*real_fchown)(int fd, uid_t uid, gid_t gid) = NULL; + +static int (*real_ftruncate)(int fd, off_t length) = NULL; + +static int (*real_ftruncate64)(int fd, off64_t length) = NULL; + +static int (*real_link)(char *orig_path, char *path) = NULL; + +static int (*real_unlink)(char *path) = NULL; + +static int (*real_utime)(char *path,struct utimbuf *times) = NULL; + +//??__futimens + +static int (*real_aio_read64)(struct aiocb64 *aiocbp) = NULL; +static int (*real_aio_read)(struct aiocb *aiocbp) = NULL; + +static int (*real_aio_write64)(struct aiocb64 *aiocbp) = NULL; +static int (*real_aio_write)(struct aiocb *aiocbp) = NULL; + +static int (*real_aio_error64)(struct aiocb64 *aiocbp) = NULL; +static int (*real_aio_error)(struct aiocb *aiocbp) = NULL; + +static ssize_t (*real_aio_return64)(struct aiocb64 *aiocbp) = NULL; +static size_t (*real_aio_return)(struct aiocb *aiocbp) = NULL; + + +static int (*real_aio_fsync64)(int op, struct aiocb64 *aiocbp) = NULL; +static int (*real_aio_fsync)(int op, struct aiocb *aiocbp) = NULL; + +static int (*real_aio_cancel64)(int fd, struct aiocb64 *aiocbp) = NULL; +static int (*real_aio_cancel)(int fd, struct aiocb *aiocbp) = NULL; + +static int (*real_aio_suspend64)(const struct aiocb64 *const list[], int nent, + const struct timespec *timeout) = NULL; + +static int (*real_aio_suspend)(const struct aiocb *const list[], int nent, + const struct timespec *timeout) = NULL; + +// #include +//??int statvfs(const char *path, struct statvfs *buf); + +/* + * NAME: CUSFS_GET_WRAPPER_FD_HASH + * + * FUNCTION: Find wrapper fd from file descriptor in the hash table + * + * + * INPUTS: + * file index - file identifier + * + * RETURNS: + * NULL = No chunk found. + * pointer = chunk found. + * + */ + +inline cflsh_usfs_wrap_fd_t *CUSFS_GET_WRAPPER_FD_HASH(int fd) +{ + cflsh_usfs_wrap_fd_t *fd_wrapper = NULL; + + + pthread_mutex_lock(&cusfs_gwrapper_lock); + fd_wrapper = cusfs_gwrapper.fd_hash[fd & MAX_NUM_CUSFS_WRAP_FDS_HASH]; + + + while (fd_wrapper) { + + if (fd_wrapper->fd == fd) { + + + break; + } + + fd_wrapper = fd_wrapper->next; + + } + + pthread_mutex_unlock(&cusfs_gwrapper_lock); + return fd_wrapper; + +} + + + + +/* + * NAME: CUSFS_IS_USFS_PATH + * + * FUNCTION: Determine if the path specified is for a + * userspace filesystem. + * + * + * INPUTS: + * file index - file identifier + * + * RETURNS: + * NULL = No chunk found. + * pointer = chunk found. + * + */ + +inline int CUSFS_IS_USFS_PATH(char *path) +{ + int rc = 0; + + + if ((strchr(path,CFLSH_USFS_DISK_DELIMINATOR)) +#ifndef BLOCK_FILEMODE_ENABLED +#ifdef _AIX + && (!strncmp(path,"/dev/hdisk",10)) +#else + && ((!strncmp(path,"/dev/sd",7)) || (!strncmp(path,"/dev/sg",7))) +#endif +#endif /* !BLOCK_FILEMODE_ENABLED */ + ) + { + + /* + * This path starts with "/dev/" and + * has a ":" in it. + */ + + //?? TODO: Should we do anything special here for filemode. + rc = 1; + + + } + + + return rc; +} + + +/* +` * NAME: cusfs_add_fd_wrapper_to_hash + * + * FUNCTION: Add fd_wrapper to global wrapper hash table + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure + * + * + */ +void cusfs_add_fd_wrapper_to_hash(cflsh_usfs_wrap_fd_t *fd_wrapper, int flags) +{ + cflsh_usfs_wrap_fd_t *tmp_fd_wrapper = NULL; + + + + pthread_mutex_lock(&cusfs_gwrapper_lock); + + if (cusfs_gwrapper.fd_hash[fd_wrapper->fd & MAX_NUM_CUSFS_WRAP_FDS_HASH] == NULL) { + + cusfs_gwrapper.fd_hash[fd_wrapper->fd & MAX_NUM_CUSFS_WRAP_FDS_HASH] = fd_wrapper; + + } else { + + tmp_fd_wrapper = cusfs_gwrapper.fd_hash[fd_wrapper->fd & MAX_NUM_CUSFS_WRAP_FDS_HASH]; + + while (tmp_fd_wrapper) { + + + if (tmp_fd_wrapper->next == NULL) { + + tmp_fd_wrapper->next = fd_wrapper; + + fd_wrapper->prev = tmp_fd_wrapper; + + break; + + } + + tmp_fd_wrapper = tmp_fd_wrapper->next; + } + } + + pthread_mutex_unlock(&cusfs_gwrapper_lock); + + return; + +} + + +/* + * NAME: cusfs_remove_fd_wrapper_from_hash + * + * FUNCTION: Remove fd_wrapper from hash + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure + * + * + */ +void cusfs_remove_fd_wrapper_from_hash(cflsh_usfs_wrap_fd_t *fd_wrapper, int flags) +{ + if (fd_wrapper == NULL) { + + return; + } + + pthread_mutex_lock(&cusfs_gwrapper_lock); + + if (fd_wrapper->prev) { + fd_wrapper->prev->next = fd_wrapper->next; + + } else { + + cusfs_gwrapper.fd_hash[fd_wrapper->fd & MAX_NUM_CUSFS_WRAP_FDS_HASH] = fd_wrapper->next; + } + + if (fd_wrapper->next) { + fd_wrapper->next->prev = fd_wrapper->prev; + } + + pthread_mutex_unlock(&cusfs_gwrapper_lock); + + return; + +} + +/* + * NAME: cufs_alloc_fd_wrapper + * + * FUNCTION: Allocate an fd_wrapper structure + * + * + * + * INPUTS: + * file index - file identifier + * + * RETURNS: + * 0 success, otherwise error. + * + * + */ + +int cufs_alloc_fd_wrapper(int *fd) +{ + cflsh_usfs_wrap_fd_t *fd_wrapper; + + + if (CUSFS_GET_WRAPPER_FD_HASH(*fd+CFLSH_USFS_WRAPPER_INITIAL_FD) != NULL) { + + /* + * if this fd is already in the hash list, then don't + * add it again. + */ + + return 0; + + } + + + if (fd == NULL) { + + return -1; + } + + fd_wrapper = malloc(sizeof(*fd_wrapper)); + + if (fd_wrapper == NULL) { + + return -1; + + } + + bzero(fd_wrapper,sizeof(*fd_wrapper)); + + + /* + * ??TODO: There is an issue that this code is not taking into account + * that this library could generate the same file descriptors as other + * + * The current work around for this now is to add an offset CFLSH_USFS_WRAPPER_INITIAL_FD + * to all file descriptors returned by the library. + */ + + fd_wrapper->usfs_fd = *fd; + + *fd += CFLSH_USFS_WRAPPER_INITIAL_FD; + fd_wrapper->fd = *fd; + + cusfs_add_fd_wrapper_to_hash(fd_wrapper,0); + + return 0; +} + + +/* + * NAME: cufs_free_fd_wrapper + * + * FUNCTION: Free an fd_wrapper structure + * + * + * + * INPUTS: + * file index - file identifier + * + * RETURNS: + * 0 success, otherwise error. + * + * + */ + + +int cufs_free_fd_wrapper(cflsh_usfs_wrap_fd_t *fd_wrapper) +{ + + + cusfs_remove_fd_wrapper_from_hash(fd_wrapper,0); + + free(fd_wrapper); + + + return 0; +} + + +/* + * NAME: statfs + * + * FUNCTION: Get statistics on filesystems + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +int statfs(const char *path, struct statfs *statfs) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_statfs((char *)path,statfs); + + + } else { + + real_statfs = (int(*)(char*,struct statfs *))dlsym(RTLD_NEXT,"statfs"); + + if (real_statfs) { + + rc = real_statfs((char *)path,statfs); + + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + + + +/* + * NAME: statfs64 + * + * FUNCTION: Get statistics on filesystems + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +int statfs64(const char *path, struct statfs64 *statfs) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_statfs64((char *)path,statfs); + + + } else { + + real_statfs64 = (int(*)(char*,struct statfs64 *))dlsym(RTLD_NEXT,"statfs64"); + + if (real_statfs64) { + + rc = real_statfs64((char *)path,statfs); + + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + + + +/* + * NAME: mkdir + * + * FUNCTION: Make a directory + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int mkdir(const char *path,mode_t mode_flags) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_mkdir((char *)path,mode_flags); + + + } else { + + real_mkdir = (int(*)(char*,unsigned int))dlsym(RTLD_NEXT,"mkdir"); + + if (real_mkdir) { + + rc = real_mkdir((char *)path,mode_flags); + + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: rmdir + * + * FUNCTION: Remove a directory + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int rmdir(const char *path) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_rmdir((char *)path); + + + } else { + + real_rmdir = (int(*)(char*))dlsym(RTLD_NEXT,"rmdir"); + + if (real_rmdir) { + + rc = real_rmdir((char *)path); + + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: fcntl + * + * FUNCTION: performs controlling function + * + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 on error. Otherwise success + * + * + */ + +int fcntl(int fd,int cmd,...) +{ + int rc = 0; + cflsh_usfs_wrap_fd_t *fd_wrapper; + long arg; + va_list ap; + + /* + * fcntl has a most 3 arguments. + * So we only need to try to extract + * one argument here. We chose type long + * which varies from 32-bit to 64-bit to + * consistent with possible pointer here. + */ + + va_start(ap,cmd); + arg = va_arg(ap,long); + va_end(ap); + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + fprintf(stderr,"fcntl called with cmd = %d",cmd); + + } else { + + real_fcntl = (int(*)(int, int, ...))dlsym(RTLD_NEXT,"fcntl"); + + if (real_fcntl) { + + rc = real_fcntl(fd,cmd,arg); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: dup2 + * + * FUNCTION: Duplicating file descriptor + * + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 on error. Otherwise success + * + * + */ + +int dup2(int old_fd,int new_fd) +{ + int rc = 0; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(old_fd); + + if (fd_wrapper) { + + fprintf(stderr,"dup2 called with old_fd = %d, and new_fd = %d",old_fd,new_fd); + + } else { + + real_dup2 = (int(*)(int, int))dlsym(RTLD_NEXT,"dup2"); + + if (real_dup2) { + + rc = real_dup2(old_fd,new_fd); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: posix_fallocate + * + * FUNCTION: Provides advisory information about a file. + * Currently this call ignores the advice. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 on error. Otherwise success + * + * + */ + +int posix_fallocate(int fd,off_t offset, off_t len) +{ + int rc = 0; + cflsh_usfs_wrap_fd_t *fd_wrapper; + off_t truncate_len = offset + len; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + fprintf(stderr,"posix_fallocated called with fd = %d",fd); + /* + * Map this into a ftruncate request. + */ + + rc = cusfs_ftruncate(fd_wrapper->usfs_fd,truncate_len); + + } else { + + real_posix_fallocate = (int(*)(int, off_t, off_t))dlsym(RTLD_NEXT,"posix_fallocate"); + + if (real_posix_fallocate) { + + rc = real_posix_fallocate(fd,offset, len); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: posix_fallocate64 + * + * FUNCTION: Provides advisory information about a file. + * Currently this call ignores the advice. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 on error. Otherwise success + * + * + */ + +int posix_fallocate64(int fd,off64_t offset, off64_t len) +{ + int rc = 0; + cflsh_usfs_wrap_fd_t *fd_wrapper; + off_t truncate_len = offset + len; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + fprintf(stderr,"posix_fallocated called with fd = %d",fd); + /* + * Map this into a ftruncate request. + */ + + rc = cusfs_ftruncate64(fd_wrapper->usfs_fd,truncate_len); + + } else { + + real_posix_fallocate64 = (int(*)(int, off64_t, off64_t))dlsym(RTLD_NEXT,"posix_fallocate64"); + + if (real_posix_fallocate64) { + + rc = real_posix_fallocate64(fd,offset, len); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: posix_fadvise + * + * FUNCTION: Provides advisory information about a file. + * Currently this call ignores the advice. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 on error. Otherwise success + * + * + */ + +int posix_fadvise(int fd,off_t offset, off_t len, int advice) +{ + int rc = 0; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_posix_fadvise(fd_wrapper->usfs_fd,offset,len,advice); + + } else { + + real_posix_fadvise = (int(*)(int, off_t, off_t, int))dlsym(RTLD_NEXT,"posix_fadvise"); + + if (real_posix_fadvise) { + + rc = real_posix_fadvise(fd,offset, len, advice); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +#ifndef _AIX + + +/* + * NAME: posix_fadvise64 + * + * FUNCTION: Provides advisory information about a file. + * Currently this call ignores the advice. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 on error. Otherwise success + * + * + */ + +int posix_fadvise64(int fd,off64_t offset, off64_t len, int advice) +{ + int rc = 0; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + + rc = cusfs_posix_fadvise(fd_wrapper->usfs_fd,offset,len,advice); + + } else { + + real_posix_fadvise64 = (int(*)(int, off64_t, off64_t, int))dlsym(RTLD_NEXT,"posix_fadvise64"); + + if (real_posix_fadvise64) { + + rc = real_posix_fadvise64(fd,offset, len, advice); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +#endif /* !_AIX */ + +/* + * NAME: dup + * + * FUNCTION: Duplicating file descriptor + * + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 on error. Otherwise success + * + * + */ + +int dup(int fd) +{ + int rc = 0; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + fprintf(stderr,"dup called with fd = %d",fd); + + } else { + + real_dup = (int(*)(int))dlsym(RTLD_NEXT,"dup"); + + if (real_dup) { + + rc = real_dup(fd); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +#ifdef _AIX +/* + * NAME: opendir64 + * + * FUNCTION: Opens a a directory + * on a disk. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +DIR64 *opendir64(const char *directory_name) +{ + DIR64 *rc; +#ifdef _AIX + DIR64 *directory; +#else + cflsh_usfs_DIR_t *directory; +#endif /* _AIX */ + + if (CUSFS_IS_USFS_PATH((char *)directory_name)) { + + rc = cusfs_opendir64((char *)directory_name); + + if (rc) { + +#ifdef _AIX + directory = rc; +#else + directory = (cflsh_usfs_DIR_t *)rc; +#endif + + if (cufs_alloc_fd_wrapper(&(directory->dd_fd))) { + + rc = NULL; + } + + } + + } else { + + real_opendir64 = (DIR64 *(*)(char *))dlsym(RTLD_NEXT,"opendir64"); + + if (real_opendir64) { + + rc = real_opendir64((char *)directory_name); + + } else { + + errno = EBADF; + rc = NULL; + } + } + + return rc; + +} +#else +/* + * NAME: opendir + * + * FUNCTION: Opens a a directory + * on a disk. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +DIR *opendir(const char *directory_name) +{ + DIR *rc = NULL; +#ifdef _AIX + DIR *directory; +#else + cflsh_usfs_DIR_t *directory; +#endif /* _AIX */ + + + if (CUSFS_IS_USFS_PATH((char *)directory_name)) { + + directory = (cflsh_usfs_DIR_t *)cusfs_opendir((char *)directory_name); + + if (directory) { + + rc = (DIR *) directory; + + if (cufs_alloc_fd_wrapper(&(directory->dd_fd))) { + + rc = NULL; + } + + } + + } else { + + real_opendir = (DIR *(*)(char *))dlsym(RTLD_NEXT,"opendir"); + + if (real_opendir) { + + rc = real_opendir((char *)directory_name); + + } else { + + errno = EBADF; + rc = NULL; + } + } + + return rc; + +} +#endif /* !_AIX */ + +#ifdef _AIX +/* + * NAME: closedir64 + * + * FUNCTION: Opens a a directory + * on a disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int closedir64(DIR64 *directory) +{ + int rc; +#ifndef _AIX + cflsh_usfs_DIR_t *usfs_directory = (cflsh_usfs_DIR_t *)directory; +#else + DIR64 *usfs_directory = directory; +#endif + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(usfs_directory->dd_fd); + + if (fd_wrapper) { + + usfs_directory->dd_fd = fd_wrapper->usfs_fd; + + rc = cusfs_closedir64((DIR64 *)usfs_directory); + + if (!rc) { + + cufs_free_fd_wrapper(fd_wrapper); + } + } else { + + + real_closedir64 = (int(*)(struct _dirdesc64*))dlsym(RTLD_NEXT,"closedir64"); + + if (real_closedir64) { + + rc = real_closedir64(directory); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +#else + +/* + * NAME: closedir + * + * FUNCTION: Opens a a directory + * on a disk. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +int closedir(DIR *directory) +{ + int rc; +#ifndef _AIX + cflsh_usfs_DIR_t *usfs_directory = (cflsh_usfs_DIR_t *)directory; +#else + DIR *usfs_directory = directory; +#endif + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(usfs_directory->dd_fd); + + if (fd_wrapper) { + + usfs_directory->dd_fd = fd_wrapper->usfs_fd; + + rc = cusfs_closedir((DIR *)usfs_directory); + + if (!rc) { + + cufs_free_fd_wrapper(fd_wrapper); + } + } else { + + + real_closedir = dlsym(RTLD_NEXT,"closedir"); + + if (real_closedir) { + + rc = real_closedir(directory); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; + +} +#endif /* !_AIX */ + + +#ifdef _AIX +/* + * NAME: readdir64_r + * + * FUNCTION: Returns a pointer to the next directory entry. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +int readdir64_r(DIR64 *directory, struct dirent64 *entry,struct dirent64 **result) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + DIR64 usfs_directory; + + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(directory->dd_fd); + + if (fd_wrapper) { + + usfs_directory = *directory; + + usfs_directory.dd_fd = fd_wrapper->usfs_fd; + rc = cusfs_readdir64_r(&usfs_directory,entry,result); + *directory = usfs_directory; + directory->dd_fd = fd_wrapper->fd; + } else { + + + real_readdir64_r = (int(*)(struct _dirdesc64*,struct dirent64*,struct dirent64**))dlsym(RTLD_NEXT,"readdir64_r"); + + if (real_readdir64_r) { + + rc = real_readdir64_r(directory,entry,result); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: readdir64 + * + * FUNCTION: Returns a pointer to the next directory entry. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +struct dirent64 *readdir64(DIR64 *directory) +{ + struct dirent64 *result = NULL; + cflsh_usfs_wrap_fd_t *fd_wrapper; + DIR64 usfs_directory; + + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(directory->dd_fd); + + if (fd_wrapper) { + + + usfs_directory = *directory; + + usfs_directory.dd_fd = fd_wrapper->usfs_fd; + result = cusfs_readdir64(&usfs_directory); + usfs_directory.dd_fd = fd_wrapper->fd; + *directory = usfs_directory; + + } else { + + + real_readdir64 = (struct dirent64 *(*)(DIR64 *))dlsym(RTLD_NEXT,"readdir64"); + + if (real_readdir64) { + + result = real_readdir64(directory); + } else { + + errno = EBADF; + result = NULL; + } + } + + return result; +} + +#else +/* + * NAME: readdir_r + * + * FUNCTION: Returns a pointer to the next directory entry. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +int readdir_r(DIR *directory, struct dirent *entry,struct dirent **result) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; +#ifndef _AIX + cflsh_usfs_DIR_t usfs_directory; + cflsh_usfs_DIR_t *usfs_directory2 = (cflsh_usfs_DIR_t *)directory; +#else + DIR usfs_directory; +#endif + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(usfs_directory2->dd_fd); + + if (fd_wrapper) { + + +#ifndef _AIX + usfs_directory = *usfs_directory2; +#else + usfs_directory = *directory; +#endif + + + usfs_directory.dd_fd = fd_wrapper->usfs_fd; + rc = cusfs_readdir_r((DIR *)&usfs_directory,entry,result); + + usfs_directory.dd_fd = fd_wrapper->fd; +#ifndef _AIX + *usfs_directory2 = usfs_directory; +#else + *directory = usfs_directory; +#endif + + } else { + + + real_readdir_r = dlsym(RTLD_NEXT,"readdir_r"); + + if (real_readdir_r) { + + rc = real_readdir_r(directory,entry,result); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + +#endif + +/* + * NAME: readdir + * + * FUNCTION: Returns a pointer to the next directory entry. + * + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +struct dirent *readdir(DIR *directory) +{ + struct dirent *result = NULL; + cflsh_usfs_wrap_fd_t *fd_wrapper; +#ifndef _AIX + cflsh_usfs_DIR_t usfs_directory; + cflsh_usfs_DIR_t *usfs_directory2 = (cflsh_usfs_DIR_t *)directory; +#else + DIR usfs_directory; + DIR *usfs_directory2 = directory; +#endif + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(usfs_directory2->dd_fd); + + if (fd_wrapper) { + + + usfs_directory = *usfs_directory2; + + + usfs_directory.dd_fd = fd_wrapper->usfs_fd; + + result = cusfs_readdir((DIR *)&usfs_directory); + + usfs_directory.dd_fd = fd_wrapper->fd; +#ifndef _AIX + *usfs_directory2 = usfs_directory; +#else + *directory = usfs_directory; +#endif + + } else { + + + real_readdir = (struct dirent *(*)(DIR *))dlsym(RTLD_NEXT,"readdir"); + + if (real_readdir) { + + result = real_readdir(directory); + } else { + + errno = EBADF; + result = NULL; + } + } + + return result; +} + + +/* + * NAME: rename + * + * FUNCTION: Rename a file/directory to another name + * in the same filesystem. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int rename(const char *from_pathname, + const char *to_pathname) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)from_pathname)) { + + rc = cusfs_rename((char *)from_pathname,(char *)to_pathname); + + + } else { + + real_rename = (int(*)(char*,char*))dlsym(RTLD_NEXT,"rename"); + + if (real_rename) { + + rc = real_rename((char *)from_pathname,(char *)to_pathname); + + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: creat + * + * FUNCTION: Wrapper for creat + * + * + * + * INPUTS: + * + * + * RETURNS: + * 0 success, otherwise error. + * + * + */ + +int creat(const char *path, mode_t mode) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_creat((char *)path,mode); + + if (rc >= 0) { + + + if (cufs_alloc_fd_wrapper(&rc)) { + + rc = -1; + } + } + + } else { + + real_creat = (int(*)(char*,unsigned int))dlsym(RTLD_NEXT,"creat"); + + if (real_creat) { + + rc = real_creat((char *)path,mode); + } else { + + errno = EBADF; + rc = -1; + } + + } + + return rc; + +} + +/* + * NAME: creat64 + * + * FUNCTION: Wrapper for creat64 + * + * + * + * INPUTS: + * + * + * RETURNS: + * 0 success, otherwise error. + * + * + */ + +int creat64(const char *path, mode_t mode) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_creat((char *)path,mode); + + if (rc >= 0) { + + + if (cufs_alloc_fd_wrapper(&rc)) { + + rc = -1; + } + } + + } else { + + real_creat = (int(*)(char*,unsigned int))dlsym(RTLD_NEXT,"creat64"); + + if (real_creat) { + + rc = real_creat((char *)path,mode); + } else { + + errno = EBADF; + rc = -1; + } + + } + + return rc; + +} + +/* + * NAME: open + * + * FUNCTION: Wrapper for open + * + * + * + * INPUTS: + * + * + * RETURNS: + * 0 success, otherwise error. + * + * + */ + +int open(const char *path, int flags, ...) +{ + int rc; + mode_t mode; + va_list ap; + + /* + * open has a most 3 arguments. + * So we only need to try to extract + * one argument here. If that argument + * exists it will be of type mode_t. + */ + + va_start(ap,flags); + mode = va_arg(ap,mode_t); + va_end(ap); + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_open((char *)path,flags,mode); + + if (rc >= 0) { + + if (cufs_alloc_fd_wrapper(&rc)) { + + rc = -1; + } + } + + } else { + + real_open = (int(*)(char*,int,...))dlsym(RTLD_NEXT,"open"); + + if (real_open) { + + rc = real_open((char *)path,flags,mode); + } else { + + errno = EBADF; + rc = -1; + } + + } + + return rc; + +} + +/* + * NAME: open64 + * + * FUNCTION: Wrapper for open64 + * + * + * + * INPUTS: + * + * + * RETURNS: + * 0 success, otherwise error. + * + * + */ + +int open64(const char *path, int flags, ...) +{ + int rc; + mode_t mode; + va_list ap; + + /* + * open has a most 3 arguments. + * So we only need to try to extract + * one argument here. If that argument + * exists it will be of type mode_t. + */ + + va_start(ap,flags); + mode = va_arg(ap,mode_t); + va_end(ap); + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_open((char *)path,flags,mode); + + if (rc >= 0) { + + if (cufs_alloc_fd_wrapper(&rc)) { + + rc = -1; + } + } + + } else { + + real_open = (int(*)(char*,int,...))dlsym(RTLD_NEXT,"open64"); + + if (real_open) { + + rc = real_open((char *)path,flags,mode); + } else { + + errno = EBADF; + rc = -1; + } + + } + + return rc; + +} + +/* + * NAME: close + * + * FUNCTION: Wrapper for close + * + * + * + * INPUTS: + * + * + * RETURNS: + * 0 success, otherwise error. + * + * + */ + +int close(int fd) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_close(fd_wrapper->usfs_fd); + + if (!rc) { + + cufs_free_fd_wrapper(fd_wrapper); + } + } else { + + + real_close = (int(*)(int))dlsym(RTLD_NEXT,"close"); + + if (real_close) { + + rc = real_close(fd); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; + +} + + +/* + * NAME: read + * + * FUNCTION: Read a file of the specified size. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +ssize_t read(int fd, void *buffer,size_t nbytes) +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_read(fd_wrapper->usfs_fd,buffer,nbytes); + + } else { + + real_read = (ssize_t(*)(int, void *, size_t))dlsym(RTLD_NEXT,"read"); + + if (real_read) { + + rc = real_read(fd,buffer,nbytes); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: write + * + * FUNCTION: Write a file of the specified size. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +ssize_t write(int fd, const void *buffer,size_t nbytes) +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_write(fd_wrapper->usfs_fd,(void *)buffer,nbytes); + + } else { + + + real_write = (ssize_t(*)(int, void *, size_t))dlsym(RTLD_NEXT,"write"); + + if (real_write) { + + rc = real_write(fd,(void *)buffer,nbytes); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + + +/* + * NAME: fsync + * + * FUNCTION: sync a file + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int fsync(int fd) +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_fsync(fd_wrapper->usfs_fd); + + } else { + + + real_fsync = (int(*)(int))dlsym(RTLD_NEXT,"fsync"); + + if (real_fsync) { + + rc = real_fsync(fd); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: lseek + * + * FUNCTION: Seek to a specified offset in the file + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +off_t lseek(int fd, off_t offset, int whence) +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_lseek(fd_wrapper->usfs_fd,offset,whence); + + } else { + + + real_lseek = (off_t(*)(int,off_t,int))dlsym(RTLD_NEXT,"lseek"); + + if (real_lseek) { + + rc = real_lseek(fd,offset,whence); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +#ifdef _AIX +/* + * NAME: llseek + * + * FUNCTION: Seek to a specified offset in the file + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +offset_t llseek(int fd, offset_t offset, int whence) +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_llseek(fd_wrapper->usfs_fd,offset,whence); + + } else { + + + real_llseek = (offset_t(*)(int,offset_t,int))dlsym(RTLD_NEXT,"llseek"); + + if (real_llseek) { + + rc = real_llseek(fd,offset,whence); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +#endif /* AIX */ + +/* + * NAME: lseek64 + * + * FUNCTION: Seek to a specified offset in the file + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +off64_t lseek64(int fd, off64_t offset, int whence) +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_lseek64(fd_wrapper->usfs_fd,offset,whence); + + } else { + + + real_lseek64 = (off64_t(*)(int,off64_t,int))dlsym(RTLD_NEXT,"lseek64"); + + if (real_lseek64) { + + rc = real_lseek64(fd,offset,whence); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + + +/* + * NAME: fstat + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int fstat(int fd,struct stat *buffer) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_fstat(fd_wrapper->usfs_fd,buffer); + + } else { + + + real_fstat = (int(*)(int,struct stat *))dlsym(RTLD_NEXT,"fstat"); + + if (real_fstat) { + + rc = real_fstat(fd,buffer); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} +#ifdef _AIX + +/* + * NAME: fstatx + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int fstatx(int fd,struct stat *buffer, int length, int command) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_fstat(fd_wrapper->usfs_fd,buffer); + + } else { + + + real_fstatx = (int(*)(int,struct stat *, int, int))dlsym(RTLD_NEXT,"fstatx"); + + if (real_fstatx) { + + rc = real_fstatx(fd,buffer,length,command); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} +/* + * NAME: fstat64 + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int fstat64(int fd,struct stat64 *buffer) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_fstat64(fd_wrapper->usfs_fd,buffer); + + } else { + + + real_fstat64 = (int(*)(int,struct stat64 *))dlsym(RTLD_NEXT,"fstat64"); + + if (real_fstat64) { + + rc = real_fstat64(fd,buffer); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: fstat64x + * + * FUNCTION: Returns fstat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int fstat64x(int fd,struct stat64x *buffer) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_fstat64x(fd_wrapper->usfs_fd,buffer); + + } else { + + + real_fstat64x = (int(*)(int,struct stat64x *))dlsym(RTLD_NEXT,"fstat64x"); + + if (real_fstat64x) { + + rc = real_fstat64x(fd,buffer); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +#endif /* _AIX */ + +#ifdef _AIX +/* + * NAME: stat64 + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int stat64(const char *path, struct stat64 *buffer) +#else +/* + * NAME: stat + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int stat(const char *path, struct stat *buffer) +#endif +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { +#ifdef _AIX + rc = cusfs_stat64((char *)path,buffer); +#else + rc = cusfs_stat((char *)path,buffer); +#endif + + + } else { +#ifdef _AIX + + real_stat64 = (int(*)(char *,struct stat64 *))dlsym(RTLD_NEXT,"stat64"); + + if (real_stat64) { + + rc = real_stat64((char *)path,buffer); + + } else { + + errno = EBADF; + rc = -1; + } +#else + + real_stat = (int(*)(char *,struct stat *))dlsym(RTLD_NEXT,"stat"); + + if (real_stat) { + + rc = real_stat((char *)path,buffer); + + } else { + + errno = EBADF; + rc = -1; + } + +#endif + + } + + return rc; +} + + +#ifdef _AIX + + +/* + * NAME: statx + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int statx(const char *path, struct stat *buffer, int length, int command) + +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_stat((char *)path,buffer); + + + + } else { + + real_statx = (int(*)(char *,struct stat *,int, int))dlsym(RTLD_NEXT,"statx"); + + if (real_statx) { + + rc = real_statx((char *)path,buffer,length,command); + + } else { + + errno = EBADF; + rc = -1; + } + + } + + return rc; +} +/* + * NAME: stat64x + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int stat64x(const char *path, struct stat64x *buffer) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_stat64x((char *)path,buffer); + + + + } else { + + + /* + * Reuse same real_stat64x for both stat64x and lstat64x + */ + + real_stat64x = (int(*)(char *,struct stat64x *))dlsym(RTLD_NEXT,"stat64x"); + + if (real_stat64x) { + + rc = real_stat64x((char *)path,buffer); + + } else { + + errno = EBADF; + rc = -1; + } + + } + + return rc; + + +} + +/* + * NAME: lstat64x + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int lstat64x(const char *path, struct stat64x *buffer) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_lstat64x((char *)path,buffer); + + + + } else { + + + /* + * Reuse same real_stat64x for both stat64x and lstat64x + */ + + real_stat64x = (int(*)(char *,struct stat64x *))dlsym(RTLD_NEXT,"lstat64x"); + + if (real_stat64x) { + + rc = real_stat64x((char *)path,buffer); + + } else { + + errno = EBADF; + rc = -1; + } + + } + + return rc; + + +} + +#endif /* _AIX */ + + +#ifdef _AIX +/* + * NAME: lstat64 + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int lstat64(const char *path, struct stat64 *buffer) +#else +/* + * NAME: lstat + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int lstat(const char *path, struct stat *buffer) +#endif +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { +#ifdef _AIX + rc = cusfs_lstat64((char *)path,buffer); +#else + rc = cusfs_lstat((char *)path,buffer); +#endif + + + } else { +#ifdef _AIX + + /* + * Reuse same real_stat64 for both stat64 and lstat64 + */ + + real_stat64 = (int(*)(char *,struct stat64 *))dlsym(RTLD_NEXT,"lstat64"); + + if (real_stat64) { + + rc = real_stat64((char *)path,buffer); + + } else { + + errno = EBADF; + rc = -1; + } +#else + + /* + * Reuse same real_stat for both stat and lstat + */ + + real_stat = (int(*)(char *,struct stat *))dlsym(RTLD_NEXT,"lstat"); + + if (real_stat) { + + rc = real_stat((char *)path,buffer); + + } else { + + errno = EBADF; + rc = -1; + } + +#endif + + } + + return rc; +} + +#ifndef _AIX +/* + * NAME: __xstat + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int __xstat(int version, const char *path, struct stat *buffer) + +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_stat((char *)path,buffer); + + + } else { + + real_xstat = (int(*)(int, const char *,struct stat *))dlsym(RTLD_NEXT,"__xstat"); + + if (real_xstat) { + + rc = real_xstat(version,path,buffer); + + } else { + + errno = EBADF; + rc = -1; + } + + } + + return rc; +} + +/* + * NAME: __lxstat + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int __lxstat(int version, const char *path, struct stat *buffer) + +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_lstat((char *)path,buffer); + + + } else { + + real_xstat = (int(*)(int, const char *,struct stat *))dlsym(RTLD_NEXT,"__lxstat"); + + if (real_xstat) { + + rc = real_xstat(version,path,buffer); + + } else { + + errno = EBADF; + rc = -1; + } + + } + + return rc; +} + + +/* + * NAME: __fxstat + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int __fxstat(int version,int fd,struct stat *buffer) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_fstat(fd_wrapper->usfs_fd,buffer); + + } else { + + + real_fxstat = (int(*)(int,int,struct stat *))dlsym(RTLD_NEXT,"__fxstat"); + + if (real_fxstat) { + + rc = real_fxstat(version,fd,buffer); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + +/* + * NAME: __xstat64 + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int __xstat64(int version, const char *path, struct stat64 *buffer) + +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_stat64((char *)path,buffer); + + + } else { + + real_xstat64 = (int(*)(int, const char *,struct stat64 *))dlsym(RTLD_NEXT,"__xstat64"); + + if (real_xstat64) { + + rc = real_xstat64(version,path,buffer); + + } else { + + errno = EBADF; + rc = -1; + } + + } + + return rc; +} + +/* + * NAME: __lxstat64 + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ +int __lxstat64(int version, const char *path, struct stat64 *buffer) + +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_lstat64((char *)path,buffer); + + + } else { + + real_xstat64 = (int(*)(int, const char *,struct stat64 *))dlsym(RTLD_NEXT,"__lxstat64"); + + if (real_xstat64) { + + rc = real_xstat64(version,path,buffer); + + } else { + + errno = EBADF; + rc = -1; + } + + } + + return rc; +} + + +/* + * NAME: __fxstat64 + * + * FUNCTION: Returns stat + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int __fxstat64(int version,int fd,struct stat64 *buffer) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_fstat64(fd_wrapper->usfs_fd,buffer); + + } else { + + + real_fxstat64 = (int(*)(int,int,struct stat64 *))dlsym(RTLD_NEXT,"__fxstat64"); + + if (real_fxstat64) { + + rc = real_fxstat64(version,fd,buffer); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +#endif /* !_AIX */ + + +/* + * NAME: readlink + * + * FUNCTION: Read the symlink filename + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +#ifdef _AIX +int +#else +ssize_t +#endif +readlink(const char *path, char *buffer, size_t buffer_size) + +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_readlink((char *)path,buffer,buffer_size); + + } else { + + + /* + * Reuse same real_stat for both stat and lstat + */ + +#ifdef _AIX + real_readlink = (int(*)(char *,char *,size_t))dlsym(RTLD_NEXT,"readlink"); +#else + real_readlink = (ssize_t(*)(char *,char *,size_t))dlsym(RTLD_NEXT,"readlink"); +#endif + + if (real_readlink) { + + rc = real_readlink((char *)path,buffer, buffer_size); + + } else { + + errno = EBADF; + rc = -1; + } + + + + } + + return rc; +} + + +/* + * NAME: access + * + * FUNCTION: Validate access to file + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int access(const char *path, int mode) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_access((char *)path,mode); + + + } else { + + real_access = (int(*)(char *,int))dlsym(RTLD_NEXT,"access"); + + if (real_access) { + + rc = real_access((char *)path,mode); + + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: fchmod + * + * FUNCTION: Change mode of file + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int fchmod(int fd,mode_t mode) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_fchmod(fd_wrapper->usfs_fd,mode); + + } else { + + + real_fchmod = (int(*)(int,mode_t))dlsym(RTLD_NEXT,"fchmod"); + + if (real_fchmod) { + + rc = real_fchmod(fd,mode); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: fchown + * + * FUNCTION: Change owner/group + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int fchown(int fd, uid_t uid, gid_t gid) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_fchown(fd_wrapper->usfs_fd,uid,gid); + + } else { + + + real_fchown = (int(*)(int,uid_t,gid_t))dlsym(RTLD_NEXT,"fchown"); + + if (real_fchown) { + + rc = real_fchown(fd,uid,gid); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: ftruncate + * + * FUNCTION: Change the length of a regular file + * to the length specified by the caller. + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int ftruncate(int fd, off_t length) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_ftruncate(fd_wrapper->usfs_fd,length); + + } else { + + + real_ftruncate = (int(*)(int,off_t))dlsym(RTLD_NEXT,"ftruncate"); + + if (real_ftruncate) { + + rc = real_ftruncate(fd,length); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + +/* + * NAME: ftruncate64 + * + * FUNCTION: Change the length of a regular file + * to the length specified by the caller. + * + * + * INPUTS: + * NONE + * + * RETURNS: -1 failure, Otherwise success + * + * + */ + +int ftruncate64(int fd, off64_t length) +{ + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(fd); + + if (fd_wrapper) { + + rc = cusfs_ftruncate64(fd_wrapper->usfs_fd,length); + + } else { + + + real_ftruncate64 = (int(*)(int,off64_t))dlsym(RTLD_NEXT,"ftruncate64"); + + if (real_ftruncate64) { + + rc = real_ftruncate64(fd,length); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + +/* + * NAME: link + * + * FUNCTION: link a file + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int link(const char *orig_path, const char *path) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_link((char *)orig_path,(char *)path); + + + } else { + + real_link = (int(*)(char*,char *))dlsym(RTLD_NEXT,"link"); + + if (real_link) { + + rc = real_link((char *)orig_path,(char *)path); + + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: unlink + * + * FUNCTION: Unlink a file + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int unlink(const char *path) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_unlink((char *)path); + + + } else { + + real_unlink = (int(*)(char*))dlsym(RTLD_NEXT,"unlink"); + + if (real_unlink) { + + rc = real_unlink((char *)path); + + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: utime + * + * FUNCTION: set acces and modification time of a file + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +int utime(const char *path,const struct utimbuf *times) +{ + int rc; + + if (CUSFS_IS_USFS_PATH((char *)path)) { + + rc = cusfs_utime((char *)path,(struct utimbuf*)times); + + + } else { + + real_utime = (int(*)(char*,struct utimbuf*))dlsym(RTLD_NEXT,"utime"); + + if (real_utime) { + + rc = real_utime((char *)path,(struct utimbuf*)times); + + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NOTE: For AIO interfaces below, AIX has two interfaces: + * Legacy and POSIX. For Linux there is just posix. + * For AIX the POSIX has function names with the + * prefix "_posix", with legacy without them. + * Linux's POSIX AIO does not have this posix + * prefix naming convention. We are only supporting + * POSIX AIO here for both AIX and Linux. + */ + + +/* + * NAME: aio_read64 + * + * FUNCTION: Issue an asynchronous read + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int _posix_aio_read64(struct aiocb64 *aiocbp) +#else +int aio_read64(struct aiocb64 *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb64 aiocb_arg; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + + /* + * Set initial state of AIOCB + */ + + CUSFS_RET_AIOCB(aiocbp,-1,EINPROGRESS); + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = (struct aiocb *)aiocbp; +#endif + + rc = cusfs_aio_read64(&aiocb_arg); + + } else { + +#ifdef _AIX + real_aio_read64 = (int(*)(struct aiocb64 *))dlsym(RTLD_NEXT,"_posix_aio_read64"); +#else + real_aio_read64 = (int(*)(struct aiocb64 *))dlsym(RTLD_NEXT,"aio_read64"); +#endif + + if (real_aio_read64) { + + rc = real_aio_read64(aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: aio_read + * + * FUNCTION: Issue an asynchronous read + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ + +#ifdef _AIX +int _posix_aio_read(struct aiocb *aiocbp) +#else +int aio_read(struct aiocb *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb aiocb_arg; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + + /* + * Set initial state of AIOCB + */ + + CUSFS_RET_AIOCB(aiocbp,-1,EINPROGRESS); + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = aiocbp; +#endif + + rc = cusfs_aio_read(&aiocb_arg); + + } else { + +#ifdef _AIX + real_aio_read = (int(*)(struct aiocb *))dlsym(RTLD_NEXT,"_posix_aio_read"); +#else + real_aio_read = (int(*)(struct aiocb *))dlsym(RTLD_NEXT,"aio_read"); +#endif + + if (real_aio_read) { + + rc = real_aio_read(aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: aio_write64 + * + * FUNCTION: Issue an asynchronous write + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int _posix_aio_write64(struct aiocb64 *aiocbp) +#else +int aio_write64(struct aiocb64 *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb64 aiocb_arg; + + + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + + /* + * Set initial state of AIOCB + */ + + CUSFS_RET_AIOCB(aiocbp,-1,EINPROGRESS); + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = (struct aiocb *)aiocbp; +#endif + rc = cusfs_aio_write64(&aiocb_arg); + + } else { + +#ifdef _AIX + real_aio_write64 = (int(*)(struct aiocb64 *))dlsym(RTLD_NEXT,"_posix_aio_write64"); +#else + real_aio_write64 = (int(*)(struct aiocb64 *))dlsym(RTLD_NEXT,"aio_write64"); +#endif + + if (real_aio_write64) { + + rc = real_aio_write64(aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: aio_write + * + * FUNCTION: Issue an asynchronous write + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int _posix_aio_write(struct aiocb *aiocbp) +#else +int aio_write(struct aiocb *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb aiocb_arg; + + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + + /* + * Set initial state of AIOCB + */ + + CUSFS_RET_AIOCB(aiocbp,-1,EINPROGRESS); + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = aiocbp; +#endif + + rc = cusfs_aio_write(&aiocb_arg); + + } else { + +#ifdef _AIX + real_aio_write = (int(*)(struct aiocb *))dlsym(RTLD_NEXT,"_posix_aio_write"); +#else + real_aio_write = (int(*)(struct aiocb *))dlsym(RTLD_NEXT,"aio_write"); +#endif + + if (real_aio_write) { + + rc = real_aio_write(aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + +/* + * NAME: aio_error64 + * + * FUNCTION: Check status on an aiocb + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int _posix_aio_error64(struct aiocb64 *aiocbp) +#else +int aio_error64(const struct aiocb64 *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb64 aiocb_arg; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = (struct aiocb *)aiocbp; +#endif + + rc = cusfs_aio_error64(&aiocb_arg); + + } else { + +#ifdef _AIX + real_aio_error64 = (int(*)(struct aiocb64 *))dlsym(RTLD_NEXT,"_posix_aio_error64"); +#else + real_aio_error64 = (int(*)(struct aiocb64 *))dlsym(RTLD_NEXT,"aio_error64"); +#endif + + if (real_aio_error64) { + + rc = real_aio_error64((struct aiocb64 *)aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + +/* + * NAME: aio_error + * + * FUNCTION: Check status on an aiocb + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int _posix_aio_error(struct aiocb *aiocbp) +#else +int aio_error(const struct aiocb *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb aiocb_arg; + + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = (struct aiocb *)aiocbp; +#endif + + rc = cusfs_aio_error(&aiocb_arg); + + } else { + +#ifdef _AIX + real_aio_error = (int(*)(struct aiocb *))dlsym(RTLD_NEXT,"_posix_aio_error"); +#else + real_aio_error = (int(*)(struct aiocb *))dlsym(RTLD_NEXT,"aio_error"); +#endif + + if (real_aio_error) { + + rc = real_aio_error((struct aiocb *)aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + +/* + * NAME: aio_return64 + * + * FUNCTION: Return complete aiocb + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +ssize_t _posix_aio_return64(struct aiocb64 *aiocbp) +#else +ssize_t aio_return64(struct aiocb64 *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb64 aiocb_arg; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = (struct aiocb *)aiocbp; +#endif + + rc = cusfs_aio_return64(&aiocb_arg); + + /* + * Set errno and return code + * in aiocb here. + */ + +#ifdef _AIX + CUSFS_RET_AIOCB(aiocbp,aiocb_arg.aio_return,aiocb_arg.aio_errno); + +#else + CUSFS_RET_AIOCB(aiocbp,aiocb_arg.__return_value,aiocb_arg.__error_code); + +#endif /* _AIX */ + + } else { + +#ifdef _AIX + real_aio_return64 = (ssize_t(*)(struct aiocb64 *))dlsym(RTLD_NEXT,"_posix_aio_return64"); +#else + real_aio_return64 = (ssize_t(*)(struct aiocb64 *))dlsym(RTLD_NEXT,"aio_return64"); +#endif + + if (real_aio_return64) { + + rc = real_aio_return64(aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: aio_return + * + * FUNCTION: Return complete aiocb + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +ssize_t _posix_aio_return(struct aiocb *aiocbp) +#else +ssize_t aio_return(struct aiocb *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb aiocb_arg; + + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = aiocbp; +#endif + + rc = cusfs_aio_return(&aiocb_arg); + +#ifdef _AIX + CUSFS_RET_AIOCB(aiocbp,aiocb_arg.aio_return,aiocb_arg.aio_errno); + +#else + CUSFS_RET_AIOCB(aiocbp,aiocb_arg.__return_value,aiocb_arg.__error_code); + +#endif /* _AIX */ + } else { + +#ifdef _AIX + real_aio_return = (size_t(*)(struct aiocb *))dlsym(RTLD_NEXT,"_posix_aio_return"); +#else + real_aio_return = (size_t(*)(struct aiocb *))dlsym(RTLD_NEXT,"aio_return"); +#endif + + if (real_aio_return) { + + rc = real_aio_return(aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + +/* + * NAME: aio_fsync64 + * + * FUNCTION: Sync aio + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int _posix_aio_fsync64(int op,struct aiocb64 *aiocbp) +#else +int aio_fsync64(int op,struct aiocb64 *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb64 aiocb_arg; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = (struct aiocb *)aiocbp; +#endif + + rc = cusfs_aio_fsync64(op,&aiocb_arg); + + } else { + +#ifdef _AIX + real_aio_fsync64 = (int(*)(int, struct aiocb64 *))dlsym(RTLD_NEXT,"_posix_aio_fsync64"); +#else + real_aio_fsync64 = (int(*)(int, struct aiocb64 *))dlsym(RTLD_NEXT,"aio_fsync64"); +#endif + + if (real_aio_fsync64) { + + rc = real_aio_fsync64(op,aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: aio_fsync + * + * FUNCTION: Sync aio + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int _posix_aio_fsync(int op,struct aiocb *aiocbp) +#else +int aio_fsync(int op,struct aiocb *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb aiocb_arg; + + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = aiocbp; +#endif + + rc = cusfs_aio_fsync(op,&aiocb_arg); + + } else { + +#ifdef _AIX + real_aio_fsync = (int(*)(int,struct aiocb *))dlsym(RTLD_NEXT,"_posix_aio_fsync"); +#else + real_aio_fsync = (int(*)(int,struct aiocb *))dlsym(RTLD_NEXT,"aio_fsync"); +#endif + + if (real_aio_fsync) { + + rc = real_aio_fsync(op,aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: aio_cancel64 + * + * FUNCTION: Cancel an iocb + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int _posix_aio_cancel64(int fd, struct aiocb64 *aiocbp) +#else +int aio_cancel64(int fd, struct aiocb64 *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb64 aiocb_arg; + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = (struct aiocb *)aiocbp; +#endif + + rc = cusfs_aio_cancel64(fd,&aiocb_arg); + + } else { + +#ifdef _AIX + real_aio_cancel64 = (int(*)(int, struct aiocb64 *))dlsym(RTLD_NEXT,"_posix_aio_cancel64"); +#else + real_aio_cancel64 = (int(*)(int, struct aiocb64 *))dlsym(RTLD_NEXT,"aio_cancel64"); +#endif + + if (real_aio_cancel64) { + + rc = real_aio_cancel64(fd,aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + + +/* + * NAME: aio_cancel + * + * FUNCTION: Cancel an aiocb. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int _posix_aio_cancel(int fd, struct aiocb *aiocbp) +#else +int aio_cancel(int fd, struct aiocb *aiocbp) +#endif +{ + + int rc; + cflsh_usfs_wrap_fd_t *fd_wrapper; + struct aiocb aiocb_arg; + + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(aiocbp->aio_fildes); + + if (fd_wrapper) { + + aiocb_arg = *aiocbp; + aiocb_arg.aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_arg.aio_handle = (aio_handle_t)aiocbp; +#else + aiocb_arg.__next_prio = aiocbp; +#endif + + rc = cusfs_aio_cancel(fd,&aiocb_arg); + + } else { + +#ifdef _AIX + real_aio_cancel = (int(*)(int, struct aiocb *))dlsym(RTLD_NEXT,"_posix_aio_cancel"); +#else + real_aio_cancel = (int(*)(int, struct aiocb *))dlsym(RTLD_NEXT,"aio_cancel"); +#endif + + if (real_aio_cancel) { + + rc = real_aio_cancel(fd,aiocbp); + } else { + + errno = EBADF; + rc = -1; + } + } + + return rc; +} + +/* + * NAME: aio_suspend64 + * + * FUNCTION: Wait for an aiocb to complete. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int _posix_aio_suspend64(const struct aiocb64 *const list[], int nent, + const struct timespec *timeout) +#else +int aio_suspend64(const struct aiocb64 *const list[], int nent, + const struct timespec *timeout) +#endif +{ + int rc; + int i,j; + cflsh_usfs_wrap_fd_t *fd_wrapper; + int num_usfs_aiocbs = 0; + int num_other_aiocbs = 0; + struct aiocb64 **usfs_list = NULL; + struct aiocb64 *aiocb_list; + + if (nent == 0) { + + return 0; + } + +#ifdef _AIX + real_aio_suspend64 = (int(*)(const struct aiocb64 *const list[], int, + const struct timespec *))dlsym(RTLD_NEXT,"_posix_aio_suspend64"); +#else + real_aio_suspend64 = (int(*)(const struct aiocb64 *const list[], int, + const struct timespec *))dlsym(RTLD_NEXT,"aio_suspend64"); +#endif + + for (i = 0; i < nent; i++) { + + if (list[i] != NULL) { + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(list[i]->aio_fildes); + + if (fd_wrapper) { + num_usfs_aiocbs++; + } else { + num_other_aiocbs++; + } + } + + } + + if (num_other_aiocbs == 0) { + + /* + * If everything is for USFS, + * then handle it. Thus we need + * to create a duplicate list of aiocbs, but + * each with the correct file descriptor for + * this library. + */ + + usfs_list = malloc(sizeof(struct aiocb64*) * num_usfs_aiocbs); + + if (usfs_list == NULL) { + + errno = ENOMEM; + return -1; + } + + aiocb_list = malloc(sizeof(struct aiocb64) * num_usfs_aiocbs); + + if (aiocb_list == NULL) { + + errno = ENOMEM; + + free(usfs_list); + return -1; + } + + + for (i = 0; i < num_usfs_aiocbs; i++) { + + if (list[i] != NULL) { + + usfs_list[i] = &(aiocb_list[i]); + + aiocb_list[i] = *(list[i]); + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(list[i]->aio_fildes); + + if (fd_wrapper == NULL) { + + + free(usfs_list); + free(aiocb_list); + return -1; + } + + aiocb_list[i].aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_list[i].aio_handle = (aio_handle_t)(list[i]); +#else + aiocb_list[i].__next_prio = (struct aiocb *)(list[i]); +#endif + + } + } + + + rc = cusfs_aio_suspend64((const struct aiocb64 * const*)usfs_list,num_usfs_aiocbs,timeout); + + + free(usfs_list); + free(aiocb_list); + + } else if (num_usfs_aiocbs == 0) { + + /* + * If everything is non-USFS aiocbs, we route + * everything through to the non-USFS AIO. + */ + if (real_aio_suspend64) { + + rc = real_aio_suspend64(list,nent,timeout); + } else { + + errno = EBADF; + rc = -1; + } + + } else { + + struct aiocb64 *non_usfs_list[num_other_aiocbs]; + /* + * If there is a mixture of both, then we only + * pass thru the non-USFS ones. This is not + * ideal, but the hope is that waiting on these + * will be sufficient for some USFS ones to complete. + */ + + for (i = 0, j = 0; i < nent; i++) { + + if (list[i] != NULL) { + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(list[i]->aio_fildes); + + if (fd_wrapper == NULL) { + non_usfs_list[j++] = (struct aiocb64 *)list[i]; + } + } + } + + if (real_aio_suspend64) { + + rc = real_aio_suspend64((const struct aiocb64 * const*)non_usfs_list,num_other_aiocbs,timeout); + } else { + + errno = EBADF; + rc = -1; + } + + + } + + return rc; +} + +/* + * NAME: aio_suspend + * + * FUNCTION: Wait for an aiocb to complete. + * + * + * INPUTS: + * NONE + * + * RETURNS: 0 success, otherwise failure. + * + * + */ +#ifdef _AIX +int _posix_aio_suspend(const struct aiocb *const list[], int nent, + const struct timespec *timeout) +#else +int aio_suspend(const struct aiocb *const list[], int nent, + const struct timespec *timeout) +#endif +{ + + int rc; + int i,j; + cflsh_usfs_wrap_fd_t *fd_wrapper; + int num_usfs_aiocbs = 0; + int num_other_aiocbs = 0; + struct aiocb **usfs_list = NULL; + struct aiocb *aiocb_list; + + if (nent == 0) { + + return 0; + } + + +#ifdef _AIX + real_aio_suspend = (int(*)(const struct aiocb *const list[], int, + const struct timespec *))dlsym(RTLD_NEXT,"_posix_aio_suspend"); +#else + real_aio_suspend = (int(*)(const struct aiocb *const list[], int, + const struct timespec *))dlsym(RTLD_NEXT,"aio_suspend"); +#endif + + for (i = 0; i < nent; i++) { + + if (list[i] != NULL) { + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(list[i]->aio_fildes); + + if (fd_wrapper) { + num_usfs_aiocbs++; + } else { + num_other_aiocbs++; + } + } + } + + if (num_other_aiocbs == 0) { + + /* + * If everything is for USFS, + * then handle it. Thus we need + * to create a duplicate list of aiocbs, but + * each with the correct file descriptor for + * this library. + */ + + usfs_list = malloc(sizeof(struct aiocb*) * num_usfs_aiocbs); + + if (usfs_list == NULL) { + + errno = ENOMEM; + return -1; + } + + aiocb_list = malloc(sizeof(struct aiocb) * num_usfs_aiocbs); + + if (aiocb_list == NULL) { + + errno = ENOMEM; + + free(usfs_list); + return -1; + } + + + for (i = 0; i < num_usfs_aiocbs; i++) { + + if (list[i] != NULL) { + + usfs_list[i] = &(aiocb_list[i]); + + aiocb_list[i] = *(list[i]); + + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(list[i]->aio_fildes); + + if (fd_wrapper == NULL) { + + + free(usfs_list); + free(aiocb_list); + return -1; + } + + aiocb_list[i].aio_fildes = fd_wrapper->usfs_fd; +#ifdef _AIX + aiocb_list[i].aio_handle = (aio_handle_t)(list[i]); +#else + aiocb_list[i].__next_prio = (struct aiocb *)(list[i]); +#endif + + + } + } + + + rc = cusfs_aio_suspend((const struct aiocb * const*)usfs_list,num_usfs_aiocbs,timeout); + + + free(usfs_list); + free(aiocb_list); + + } else if (num_usfs_aiocbs == 0) { + + /* + * If everything is non-USFS aiocbs, we route + * everything through to the non-USFS AIO. + */ + if (real_aio_suspend) { + + rc = real_aio_suspend(list,nent,timeout); + } else { + + errno = EBADF; + rc = -1; + } + + } else { + + struct aiocb *non_usfs_list[num_other_aiocbs]; + /* + * If there is a mixture of both, then we only + * pass thru the non-USFS ones. This is not + * ideal, but the hope is that waiting on these + * will be sufficient for some USFS ones to complete. + */ + + for (i = 0, j = 0; i < nent; i++) { + + if (list[i] != NULL) { + fd_wrapper = CUSFS_GET_WRAPPER_FD_HASH(list[i]->aio_fildes); + + if (fd_wrapper == NULL) { + non_usfs_list[j++] = (struct aiocb *)list[i]; + } + } + } + + if (real_aio_suspend) { + + rc = real_aio_suspend((const struct aiocb * const*)non_usfs_list,num_other_aiocbs,timeout); + } else { + + errno = EBADF; + rc = -1; + } + + + } + + return rc; +} + diff --git a/src/usfs/cflsh_usfs_wrapper.h b/src/usfs/cflsh_usfs_wrapper.h new file mode 100644 index 00000000..e1e03014 --- /dev/null +++ b/src/usfs/cflsh_usfs_wrapper.h @@ -0,0 +1,110 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* bos720 src/bos/usr/ccs/lib/libcflsh_block/cflsh_usfs_wrapper.h 1.1 */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +/* %Z%%M% %I% %W% %G% %U% */ + + +/* + * COMPONENT_NAME: (sysxcflashusfs) CAPI Flash user space file system library + * + * FUNCTIONS: Data structure used by library for its various internal + * operations. + * + * ORIGINS: 27 + * + * -- ( when + * combined with the aggregated modules for this product) + * OBJECT CODE ONLY SOURCE MATERIALS + * (C) COPYRIGHT International Business Machines Corp. 2015 + * All Rights Reserved + * + * US Government Users Restricted Rights - Use, duplication or + * disclosure restricted by GSA ADP Schedule Contract with IBM Corp. + */ + +#ifndef _H_CFLASH_USFS_WRAPPER +#define _H_CFLASH_USFS_WRAPPER + +#ifndef _AIX +/************************************************************************/ +/* Linux special internal DIR structure implementation. */ +/* Linux hides the internals of this structure */ +/* and wants it opaque. So for linux we will use */ +/* our own internal implementation. There are only */ +/* two key aspects we need to contain in this */ +/* structure, the file descriptor and the offset */ +/* in the directory. */ +/************************************************************************/ +typedef struct cflsh_usfs_DIR_s { + int dd_fd; /* file descriptor of directory */ + uint64_t dd_blksize; /* This is the filesystems block size */ + uint64_t dd_size; + uint64_t dd_loc; /* logical offset in directory */ + uint64_t dd_curoff; /* real offset in directory */ + +} cflsh_usfs_DIR_t; + + +#endif /* !_AIX */ + +#define CFLSH_USFS_DISK_DELIMINATOR ':' + +#define MAX_NUM_CUSFS_WRAP_FDS 16 + +#define MAX_NUM_CUSFS_WRAP_FDS_HASH (MAX_NUM_CUSFS_WRAP_FDS-1) + +#define CFLSH_USFS_WRAPPER_INITIAL_FD 512 /* Intial filedescriptor value */ + /* used by this library */ + + + +/************************************************************************/ +/* cflsh_usfs_wrap_fd - The data structure file descriptors associated */ +/* with this userspace filesystem. */ +/* . */ +/************************************************************************/ +typedef struct cflsh_usfs_wrap_fd_s { + struct cflsh_usfs_wrap_fd_s *prev; /* Previous filesystem in list */ + struct cflsh_usfs_wrap_fd_s *next; /* Next filesystem in list */ + int fd; /* file descriptor; */ + int usfs_fd; /* File descriptor used by USFS*/ +} cflsh_usfs_wrap_fd_t; + +/************************************************************************/ +/* cusfs_gwrapper - Global wrapper library data structure per process */ +/************************************************************************/ + +typedef struct cusfs_gwrapper_s { + int flags; /* Global flags for this chunk */ + + cflsh_usfs_wrap_fd_t *fd_hash[MAX_NUM_CUSFS_WRAP_FDS]; + +//?? Not yet cusfs_lock_t lock; +} cusfs_gwrapper_t; + +extern cusfs_gwrapper_t cusfs_gwrapper; + + + +#endif /* _H_CFLASH_USFS_WRAPPER */ diff --git a/src/usfs/cusfs_serv.c b/src/usfs/cusfs_serv.c new file mode 100644 index 00000000..e66d9e1e --- /dev/null +++ b/src/usfs/cusfs_serv.c @@ -0,0 +1,1171 @@ +// IBM_PROLOG_BEGIN_TAG +// This is an automatically generated prolog. +// +// $Source: src/master/cusfs_serv.c $ +// +// IBM CONFIDENTIAL +// +// COPYRIGHT International Business Machines Corp. 2014 +// +// p1 +// +// Object Code Only (OCO) source materials +// Licensed Internal Code Source Materials +// IBM Surelock Licensed Internal Code +// +// The source code for this program is not published or other- +// wise divested of its trade secrets, irrespective of what has +// been deposited with the U.S. Copyright Office. +// +// Origin: 30 +// +// IBM_PROLOG_END +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef _AIX +#include +#else +#include +#endif /* !_AIX */ +#include +#include +#include + +#include +#include "cflsh_usfs_client.h" +#include + +#ifndef _AIX +#include +REVISION_TAGS(cusfs_serv); +#endif + +// LOG_ERR(3), LOG_WARNING(4), LOG_NOTICE(5), LOG_INFO(6) & LOG_DEBUG(7) +// +// TRACE_x versions: fmt must be of the form: "%s: xxxx" - the +// leading %s is the disk name +// +// trace_x versions: used when the log is not specific to +// a particular disk or the disk name is not available +// +#define TRACE_0(lvl, p_disk, fmt) \ + if (trc_lvl > lvl) { syslog(lvl, fmt, (p_disk)->name); } + +#define TRACE_1(lvl, p_disk, fmt, A) \ + if (trc_lvl > lvl) { syslog(lvl, fmt, (p_disk)->name, A); } + +#define TRACE_2(lvl, p_disk, fmt, A, B) \ + if (trc_lvl > lvl) { syslog(lvl, fmt, (p_disk)->name, A, B); } + +#define TRACE_3(lvl, p_disk, fmt, A, B, C) \ + if (trc_lvl > lvl) { syslog(lvl, fmt, (p_disk)->name, A, B, C); } + +#define TRACE_4(lvl, p_disk, fmt, A, B, C, D)\ + if (trc_lvl > lvl) { syslog(lvl, fmt, (p_disk)->name, A, B, C, D); } + +#define TRACE_5(lvl, p_disk, fmt, A, B, C, D, E)\ + if (trc_lvl > lvl) { syslog(lvl, fmt, (p_disk)->name, A, B, C, D, E); } + +// these do not have a disk name +#define trace_0(lvl, fmt) \ + if (trc_lvl > lvl) { syslog(lvl, fmt); } + +#define trace_1(lvl, fmt, A) \ + if (trc_lvl > lvl) { syslog(lvl, fmt, A); } + +#define trace_2(lvl, fmt, A, B) \ + if (trc_lvl > lvl) { syslog(lvl, fmt, A, B); } + +#define trace_3(lvl, fmt, A, B, C) \ + if (trc_lvl > lvl) { syslog(lvl, fmt, A, B, C); } + +unsigned int trc_lvl = LOG_INFO; // by default, log NOTICE and lower levels. + // lower level is higher severity + +/*************************************************************************** + * Master Context Server + * + * The functions of the master context are: + * + * 1. virtualize physical LUN(s) for user contexts. Users send IO to a + * resource handle that identifies a virtual LUN. Resource handles + * are mapped to physical LBAs by the master. + * 2. control what a user context is allowed or not allowed to do. + * + * There are several key functions that are not provided by the master: + * + * 1. Does not handle different linux disks (paths) with same filesystem. + * + * + * A single instance of the master serves a host. + * + * + ***************************************************************************/ + +typedef struct { + struct capikv_ini *p_ini; + struct afu_alloc *p_disk_a; //?? +} global_t; + +global_t gb; /* cusfs_serv globals */ + + +conn_info_t* +alloc_connection(cusfs_serv_disk_t *p_disk, int fd) +{ + int i; + + TRACE_0(LOG_INFO, p_disk, + "%s: allocating a new connection\n"); + + for (i = 0; i < MAX_CONNS; i++) { + if (p_disk->conn_tbl[i].fd == -1) { /* free entry */ + p_disk->conn_tbl[i].fd = fd; + p_disk->conn_tbl[i].rx = rx_mcreg; + +#ifdef _AIX + /* + * AIX uses the same index into both the + * conn_tbl and events table. + */ + p_disk->events[i].fd = fd; + p_disk->events[i].events = POLLIN|POLLMSG; + p_disk->num_poll_events++; +#endif /* _AIX */ + break; + } + } + + return (i == MAX_CONNS) ? NULL : &p_disk->conn_tbl[i]; +} + +void +free_connection(cusfs_serv_disk_t *p_disk, conn_info_t *p_conn_info) +{ + TRACE_1(LOG_INFO, p_disk, "%s: freeing connection %p\n", + p_conn_info); + +#ifdef _AIX + int i; + + + for (i = 0; i < MAX_CONNS; i++) { + if (&(p_disk->conn_tbl[i]) == p_conn_info) { /* matching entry */ + + /* + * AIX uses the same index into both the + * conn_tbl and events table. + */ + p_disk->events[i].fd = 0; + p_disk->events[i].events = 0; + p_disk->num_poll_events--; + break; + } + } + +#endif /* _AIX */ + + p_conn_info->fd = -1; +} + + +void undo_master_init(cusfs_serv_disk_t *p_disk, enum undo_level level) +{ + + switch(level) + { + case UNDO_AFU_ALL: + case UNDO_EPOLL_ADD: + case UNDO_EPOLL_CREATE: +#ifndef _AIX + close(p_disk->epfd); +#endif + + case UNDO_BIND_SOCK: + unlink(p_disk->svr_addr.sun_path); + case UNDO_OPEN_SOCK: + close(p_disk->listen_fd); + default: + break; + } +} + + +int master_init(cusfs_serv_disk_t *p_disk, struct capikv_ini_elm *p_elm) +{ + int i; + int rc; + mode_t oldmask; + pthread_mutexattr_t mattr; + pthread_condattr_t cattr; + + char *env_master_socket = getenv("CFLSH_USFS_MASTER_SOCKET"); +#ifndef _AIX + struct epoll_event epoll_event; +#endif /* !_AIX */ + enum undo_level level = UNDO_NONE; + + pthread_mutexattr_init(&mattr); + pthread_condattr_init(&cattr); + + memset(p_disk, 0, sizeof(*p_disk)); + + strncpy(p_disk->master_dev_path, p_elm->afu_dev_pathm, MC_PATHLEN - 1); + p_disk->name = strrchr(p_disk->master_dev_path, '/') + 1; + pthread_mutex_init(&p_disk->mutex, &mattr); + pthread_mutex_init(&p_disk->err_mutex, &mattr); + pthread_cond_init(&p_disk->err_cv, &cattr); + + for (i = 0; i < MAX_CONNS; i++) { + p_disk->conn_tbl[i].fd = -1; + } + + // keep AFU accessed data in RAM as much as possible +// mlock(p_disk->rht, sizeof(p_disk->rht)); + level = UNDO_MLOCK; + + pthread_condattr_destroy(&cattr); + pthread_mutexattr_destroy(&mattr); + + // open master device ?? + p_disk->afu_fd = open(p_disk->master_dev_path, O_RDWR); + if (p_disk->afu_fd < 0) { + TRACE_1(LOG_ERR, p_disk, "%s: open failed, errno %d\n", errno); + undo_master_init(p_disk, level); + return -1; + } + level = UNDO_AFU_OPEN; + + // enable the AFU. This must be done before mmap. + + + // Create a socket to be used for listening to connections + p_disk->listen_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (p_disk->listen_fd < 0) { + TRACE_1(LOG_ERR, p_disk, "%s: socket failed, errno %d\n", errno); + undo_master_init(p_disk, level); + return -1; + } + level = UNDO_OPEN_SOCK; + + // Bind the socket to the file + bzero(&p_disk->svr_addr, sizeof(struct sockaddr_un)); + p_disk->svr_addr.sun_family = AF_UNIX; + + if (env_master_socket) { + strcpy(p_disk->svr_addr.sun_path, env_master_socket); + } else { + strcpy(p_disk->svr_addr.sun_path, CFLSH_USFS_MASTER_SOCKET_DIR); + } + strcat(p_disk->svr_addr.sun_path, p_disk->master_dev_path); + mkdir_p(p_disk->svr_addr.sun_path); // make intermediate directories + unlink(p_disk->svr_addr.sun_path); + // create socket with rwx for group but no perms for others + oldmask = umask(007); + rc = bind(p_disk->listen_fd, (struct sockaddr *)&p_disk->svr_addr, + sizeof(p_disk->svr_addr)); + umask(oldmask); // set umask back to default + if (rc) { + TRACE_2(LOG_ERR, p_disk, "%s: bind failed, rc %d, errno %d\n", rc, errno); + undo_master_init(p_disk, level); + return -1; + } + level = UNDO_BIND_SOCK; + + // do not listen on the socket at this point since we do not + // want clients to be able to connect yet. + +#ifndef _AIX + /* + * NOTE: This functionality is currently broken in AIX, which + * does not support epoll. So if this is needed in AIX + * some analog must be added. + */ + + // Create the epoll array to be used for waiting on events. + p_disk->epfd = epoll_create(MAX_CONN_TO_POLL); + if (p_disk->epfd == -1) { + TRACE_1(LOG_ERR, p_disk, "%s: epoll_create failed, errno %d\n", errno); + undo_master_init(p_disk, level); + return -1; + } + level = UNDO_EPOLL_CREATE; + + // Add the listening file descriptor to the epoll array + p_disk->conn_tbl[0].fd = p_disk->listen_fd; + memset(&epoll_event, 0, sizeof(struct epoll_event)); + epoll_event.events = EPOLLIN; + epoll_event.data.ptr = &p_disk->conn_tbl[0]; + rc = epoll_ctl(p_disk->epfd, EPOLL_CTL_ADD, p_disk->listen_fd, &epoll_event); + if (rc) { + TRACE_2(LOG_ERR, p_disk, "%s: epoll_ctl failed for ADD, rc %d, errno %d\n", + rc, errno); + undo_master_init(p_disk, level); + return -1; + } + level = UNDO_EPOLL_ADD; +#else + + memset(p_disk->events,0,sizeof(struct pollfd) * MAX_CONN_TO_POLL); + + // Add the listening file descriptor to the epoll array + p_disk->conn_tbl[0].fd = p_disk->listen_fd; + p_disk->events[0].fd = p_disk->listen_fd; + p_disk->events[0].events = POLLIN|POLLMSG; + p_disk->num_poll_events++; +#endif /* !_AIX */ + + return 0; +} + + +int cusfs_serv_disk_term(cusfs_serv_disk_t *p_disk) +{ + undo_master_init(p_disk, UNDO_AFU_ALL); + + return 0; +} + +// all do_xxx functions return 0 or an errno value +// + +/* + * NAME: do_master_register + * + * FUNCTION: Register a user AFU context with master. + * + * INPUTS: + * p_disk - Pointer to afu struct + * p_conn_info - Pointer to connection the request came in + * challenge - Challenge to validate if requester owns + * the context it is trying to register. + * OUTPUTS: + * none + * + * RETURNS: + * 0 - Success + * errno - Failure + * + * NOTES: + * When successful: + * a. Sets CTX_CAP + * c. Clears all RHT entries effectively making + * all resource handles invalid. + * d. goes to rx_ready state + * + * If registation fails, stay in rx_mcreg state to allow client retries. + */ +int +do_master_register(cusfs_serv_disk_t *p_disk, + conn_info_t *p_conn_info, + uint64_t challenge) +{ + + + TRACE_3(LOG_INFO, p_disk, "%s: %s, client_pid=%d client_fd=%d\n", + __func__, + p_conn_info->client_pid, + p_conn_info->client_fd); + + + + /* a fresh registration will cause all previous registrations, + if any, to be forcefully canceled. This is important since + a client can close the context (AFU) but not unregister the + mc_handle. A new owner of the same context must be able to + mc_register by forcefully unregistering the previous owner. + */ + if (p_conn_info->mode == MCREG_INITIAL_REG) { +#ifdef _NOT_YET + //?? Need to ensure someone does not double register. + for (i = 0; i < MAX_CONNS; i++) { + if (p_disk->conn_tbl[i].p_ctx_info == p_ctx_info) { + do_master_unregister(p_disk, &p_disk->conn_tbl[i]); + } + } +#endif /* _NOT_YET */ + + } + + p_conn_info->rx = rx_ready; /* it is now registered, go to ready state */ + return 0; + +} + +/* + * NAME: do_master_unregister + * + * FUNCTION: Unregister a user AFU context with master. + * + * INPUTS: + * p_disk - Pointer to afu struct + * p_conn_info - Pointer to connection the request came in + * + * OUTPUTS: + * none + * + * RETURNS: + * 0 - Success + * errno - Failure + * + * NOTES: + * When successful: + * a. RHT_START, RHT_CNT & CTX_CAP registers for the + * context are cleared + * b. There is no need to clear RHT entries since + * RHT_CNT=0. + * c. goes to rx_mcreg state to allow re-registration + */ +int +do_master_unregister(cusfs_serv_disk_t *p_disk, + conn_info_t *p_conn_info) +{ + + + + TRACE_3(LOG_INFO, p_disk, "%s: %s, client_pid=%d client_fd=%d\n", + __func__, + p_conn_info->client_pid, + p_conn_info->client_fd); + + //?? Need lock to unlock all locks for this client. + p_conn_info->rx = rx_mcreg; /* client can now send another MCREG */ + + return 0; +} + + + + +/* + * NAME: do_master_lock + * + * FUNCTION: Get a shared filesystem lock + * + * INPUTS: + * p_disk - Pointer to afu struct + * p_conn_info - Pointer to connection the request came in + * flags - Permission flags + * + * OUTPUTS: + * p_res_hndl - Pointer to allocated resource handle + * + * RETURNS: + * 0 - Success + * errno - Failure + * + * NOTES: + * When successful, the RHT entry contains + * a. non-zero NMASK, format & permission bits. + * b. LXT_START & LXT_CNT are still zeroed. For all purposes, + * the resource handle is opened in SW, but invalid in HW + * due to 0 size. + * + * A zero NMASK means the RHT entry is free/closed. + */ +int +do_master_lock(cusfs_serv_disk_t *p_disk, + conn_info_t *p_conn_info, + cflsh_usfs_master_lock_t lock, + uint64_t flags) +{ + int rc = 0; + + if (lock >= CFLSH_USFS_MASTER_LAST_ITEM) { + + errno = EINVAL; + return -1; + } + + if (p_disk->locks[lock].client_pid == 0) { + + p_disk->locks[lock].client_pid = p_conn_info->client_pid; + } else { + + CUSFS_Q_NODE_TAIL(p_disk->locks[lock].head,p_disk->locks[lock].tail,p_conn_info,prev,next); + rc = EBUSY; + } + + + TRACE_4(LOG_INFO, p_disk, "%s: %s, client_pid=%d client_fd=%d lock=%d\n", + __func__, + p_conn_info->client_pid, + p_conn_info->client_fd, + lock); + + return rc; +} + + +/* + * NAME: do_master_unlock + * + * FUNCTION: Release a shared filesystem lock + * + * + * INPUTS: + * p_disk - Pointer to afu struct + * p_conn_info - Pointer to connection the request came in + * res_hndl - resource handle to close + * + * OUTPUTS: + * none + * + * RETURNS: + * 0 - Success + * errno - Failure + * + * NOTES: + * When successful, the RHT entry is cleared. + */ +int +do_master_unlock(cusfs_serv_disk_t *p_disk, + conn_info_t *p_conn_info, + cflsh_usfs_master_lock_t lock, + uint64_t flags) +{ + conn_info_t *p_conn_info_next; + cflsh_usfs_master_resp_t mc_resp; + int rc = 0; + + + + + if (p_disk->locks[lock].client_pid == p_conn_info->client_pid) { + + p_disk->locks[lock].client_pid = 0; + p_conn_info_next = p_disk->locks[lock].head; + + if (p_conn_info_next) { + CUSFS_DQ_NODE(p_disk->locks[lock].head,p_disk->locks[lock].tail,p_conn_info,prev,next); + + // Send response back to the client. + + p_conn_info_next->mc_resp.header.status = 0; + + rc = xfer_data(p_conn_info_next->fd, XFER_OP_WRITE, + (void *)&(p_conn_info_next->mc_resp), sizeof(mc_resp)); + if (rc) { + TRACE_1(LOG_ERR, p_disk, + "%s: write of cmd response failed, rc %d\n", rc); + + } + } + } + + TRACE_4(LOG_INFO, p_disk, "%s: %s, client_pid=%d client_fd=%d lock=%d\n", + __func__, + p_conn_info->client_pid, + p_conn_info->client_fd, + lock); + + + return 0; +} + + +/***************************************************************************** + * Procedure: xfer_data + * + * Description: Perform a transfer operation for the given + * socket file descriptor. + * + * Parameters: + * fd: Socket File Descriptor + * op: Read or Write Operation + * buf: Buffer to either read from or write to + * exp_size: Size of data transfer + * + * Return: 0, if successful + * non-zero otherwise + *****************************************************************************/ +int +xfer_data(int fd, int op, void *buf, ssize_t exp_size) +{ + int rc = 0; + ssize_t offset = 0; + ssize_t bytes_xfer = 0; + ssize_t target_size = exp_size; + struct iovec iov; + struct msghdr msg; + + while ( 1 ) + { + // Set up IO vector for IO operation. + memset(&msg, 0, sizeof(struct msghdr)); + iov.iov_base = buf + offset; + iov.iov_len = target_size; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + // Check to see if we are sending or receiving data + if ( op == XFER_OP_READ ) + { + bytes_xfer = recvmsg(fd, &msg, MSG_WAITALL); + } + else + { + bytes_xfer = sendmsg(fd, &msg, MSG_NOSIGNAL); + } + + if ( -1 == bytes_xfer ) + { + if ( EAGAIN == errno || EWOULDBLOCK == errno || EINTR == errno) + { + // just retry the whole request + continue; + } + else + { + // connection closed by the other end + rc = 1; + break; + } + } + else if ( 0 == bytes_xfer ) + { + // connection closed by the other end + rc = 1; + break; + } + else if ( bytes_xfer == target_size ) + { + // We have transfered all the bytes we wanted, we + // can stop now. + rc = 0; + break; + } + else + { + // less than target size - partial condition + // set up to transfer for the remainder of the request + offset += bytes_xfer; + target_size = (target_size - bytes_xfer); + } + } + + return rc; +} + +// rx fcn on a fresh connection waiting a MCREG. +// On receipt of a MCREG, it will go to the rx_ready state where +// all cmds except a MCREG is accepted. +// +int rx_mcreg(cusfs_serv_disk_t *p_disk, conn_info_t *p_conn_info, + cflsh_usfs_master_req_t *p_req, mc_resp_t *p_resp) +{ + int status; + + switch (p_req->header.command) + { + case CFLSH_USFS_CMD_MASTER_REG: + // first command on a fresh connection + // copy input fields to connection info + p_conn_info->client_pid = p_req->reg.client_pid; + p_conn_info->client_fd = p_req->reg.client_fd; + + + + status = do_master_register(p_disk, p_conn_info, p_req->reg.challenge); + + break; + + default: + // fail everything else + TRACE_1(LOG_ERR, p_disk, "%s: bad command code %d in wait_mcreg state\n", + p_req->header.command); + status = EINVAL; + break; + } + + return status; +} + +int rx_ready(cusfs_serv_disk_t *p_disk, conn_info_t *p_conn_info, + cflsh_usfs_master_req_t *p_req, mc_resp_t *p_resp) +{ + int status; + + switch (p_req->header.command) + { + case CFLSH_USFS_CMD_MASTER_UNREG: + (void) do_master_unregister(p_disk, p_conn_info); + status = 0; + break; + + case CFLSH_USFS_CMD_MASTER_LOCK: + status = do_master_lock(p_disk, p_conn_info, + p_req->lock.lock, + p_req->lock.flags); + break; + + case CFLSH_USFS_CMD_MASTER_UNLOCK: + status = do_master_unlock(p_disk, p_conn_info, + p_req->lock.lock, + p_req->lock.flags); + break; + + default : + TRACE_1(LOG_ERR, p_disk, "%s: bad command code %d in ready state\n", + p_req->header.command); + status = EINVAL; + break; + } + + return status; +} + +void *afu_ipc_rx(void *arg) { + int conn_fd; + int rc; + int i; + int nready; + socklen_t len; + struct sockaddr_un cli_addr; +#ifndef _AIX + struct epoll_event epoll_event; +#endif /* !_AIX */ + cflsh_usfs_master_req_t mc_req; + cflsh_usfs_master_resp_t mc_resp; + conn_info_t *p_conn_info; + conn_info_t *p_conn_info_new; + + cusfs_serv_disk_t *p_disk = (cusfs_serv_disk_t *) arg; + + // The listen is delayed until everything is ready. This prevent + // clients from connecting to the server until it reaches this + // point. + rc = listen(p_disk->listen_fd, SOMAXCONN); + if ( rc ) { + TRACE_2(LOG_ERR, p_disk, "%s: listen failed, rc %d, errno %d\n", rc, errno); + exit(-1); + } + + while (1) { + + +#ifndef _AIX + + // + // Wait for an event on any of the watched file descriptors + // block for ever if no events. + // + // Note we poll all file descriptors in the epoll structure + // but receive only up to MAX_CONN_TO_POLL ready fds at a time. + // + nready = epoll_wait(p_disk->epfd, &p_disk->events[0], + MAX_CONN_TO_POLL, -1); +#else + nready = poll(&p_disk->events,p_disk->num_poll_events,1); + +#endif /* _AIX */ + + + if ((nready == -1) && (errno == EINTR)) { + continue; + } + + + +#ifndef _AIX + + for (i = 0; i < nready; i++) { + p_conn_info = (conn_info_t*) p_disk->events[i].data.ptr; +#else + + if (nready == 0) { + continue; + } + + for (i = 0; i < p_disk->num_poll_events; i++) { + + + if (p_disk->events[i].revents == 0) { + + continue; + } + p_conn_info = &(p_disk->conn_tbl[i]); + +#endif /* _AIX */ + +#ifndef _AIX + if (p_disk->events[i].events & (EPOLLHUP | EPOLLERR)) { +#else + if (p_disk->events[i].revents & (POLLHUP | POLLERR)) { +#endif /* _AIX */ + + + // Something bad happened with the connection. Assume the + // worse and remove the file descriptor from the array of + // watched FD's + TRACE_3(LOG_ERR, p_disk, + "%s: connection is bad, %d (0x%08X), client fd %d\n", + p_conn_info->fd, p_disk->events[i].events, + p_conn_info->client_fd); + + +#ifndef _AIX + rc = epoll_ctl(p_disk->epfd, EPOLL_CTL_DEL, p_conn_info->fd, + &p_disk->events[i]); + if (rc) { + TRACE_2(LOG_ERR, p_disk, + "%s: epoll_ctl failed for DEL: %d (%d)\n", rc, errno); + } + +#endif /* !_AIX */ + + close(p_conn_info->fd); + if (p_conn_info->p_ctx_info != NULL) { /* if registered */ + (void) do_master_unregister(p_disk, p_conn_info); + } + free_connection(p_disk, p_conn_info); + + continue; + } + + // Is this the listening FD...would mean a new connection + if (p_conn_info->fd == p_disk->listen_fd) { + len = sizeof(cli_addr); + conn_fd = accept(p_disk->listen_fd, (struct sockaddr *)&cli_addr, &len); + if (conn_fd == -1) { + TRACE_2(LOG_ERR, p_disk, + "%s: accept failed, fd %d, errno %d\n", conn_fd, errno); + continue; + } + + p_conn_info_new = alloc_connection(p_disk, conn_fd); + if (p_conn_info_new == NULL) { + TRACE_0(LOG_ERR, p_disk, + "%s: too many connections, closing new connection\n"); + close(conn_fd); + continue; + } + +#ifndef _AIX + // Add the connection to the watched array + epoll_event.events = EPOLLIN; + epoll_event.data.ptr = p_conn_info_new; + rc = epoll_ctl(p_disk->epfd, EPOLL_CTL_ADD, conn_fd, &epoll_event); + if (rc == -1) { + TRACE_1(LOG_ERR, p_disk, + "%s: epoll_ctl ADD failed, errno %d\n", errno); + close(conn_fd); + continue; + } +#endif /* !_AIX */ + + } + else { + // We can now assume that we have data to be read from a client + // Read the entire command + rc = xfer_data(p_conn_info->fd, XFER_OP_READ, + (void *)&mc_req, sizeof(mc_req)); + if (rc) { + TRACE_1(LOG_ERR, p_disk, "%s: read of cmd failed, rc %d\n", rc); + + +#ifdef _AIX + /* + * This indicates the connection was closed on the + * other end. Let's close our connection and clean + * up. + */ + + close(p_conn_info->fd); + if (p_conn_info->p_ctx_info != NULL) { /* if registered */ + (void) do_master_unregister(p_disk, p_conn_info); + } + free_connection(p_disk, p_conn_info); +#endif /* _AIX */ + continue; + } + + TRACE_4(LOG_INFO, p_disk, + "%s: command code=%d, tag=%d, size=%d, conn_info=%p\n", + mc_req.header.command, + mc_req.header.tag, + mc_req.header.size, + p_conn_info); + + // process command and fill in response while protecting from + // other threads accessing the disk + + + p_conn_info->mc_resp.header.command = mc_req.header.command; + p_conn_info->mc_resp.header.tag = mc_req.header.tag; + p_conn_info->mc_resp.header.size = sizeof(mc_resp); + + if (p_conn_info->rx) { + pthread_mutex_lock(&p_disk->mutex); + + p_conn_info->mc_resp.header.status = + (*p_conn_info->rx)(p_disk, p_conn_info, &mc_req, &mc_resp); + + pthread_mutex_unlock(&p_disk->mutex); + } else { + + + TRACE_0(LOG_ERR, p_disk, + "%s: rx function pointer is null"); + } + + + if (p_conn_info->mc_resp.header.status!= EBUSY) { + + // Send response back to the client. + rc = xfer_data(p_conn_info->fd, XFER_OP_WRITE, + (void *)&(p_conn_info->mc_resp), sizeof(mc_resp)); + if (rc) { + TRACE_1(LOG_ERR, p_disk, + "%s: write of cmd response failed, rc %d\n", rc); + continue; + } + } + } + } + } + + return NULL; +} + + + + +int mkdir_p(char *file_path) // path must be absolute and must end on a file +{ + struct stat s; + char *p; + char *last = NULL; + + if (file_path[0] != '/') { + return -1; + } + + if ((last = strrchr(file_path, '/'))) { + *last = '\0'; // get rid of the last component + } + + for (p = (file_path + 1); *p; p++) { + if (*p == '/') { + *p = '\0'; + if (stat(file_path, &s) == -1) { + // create dirs as traversable by group members and no one else + mkdir(file_path, 0710); + } + *p = '/'; + } + } + + if (stat(file_path, &s) == -1) { + // create dirs as traversable by group members and no one else + mkdir(file_path, 0710); + } + + if (last) { // restore original file_path + *last = '/'; + } + + return 0; +} + + + + +void usage(char *prog) { +#ifndef _AIX + fprintf(stderr, "usage-a: %s [-v level] /dev/sgN ....\n", prog); + fprintf(stderr, " : %s /dev/sg5\n", prog); +#else + fprintf(stderr, "usage-a: %s [-v level] /dev/hdiskN ....\n", prog); + fprintf(stderr, " : %s /dev/hdisk5\n", prog); +#endif /* !_AIX */ + fprintf(stderr, "usage-b: %s [-i ini_file] [-v level]\n", prog); + fprintf(stderr, " : %s -i myafu.ini\n", prog); + fprintf(stderr, " specify any number of disk paths (that have filesystems) in usage-a\n"); + fprintf(stderr, " level is trace level in decimal from 0(least) to 8(all)\n"); +} + +// the caller to validate the returned struct +// +// need: domain sock dir? +// +struct capikv_ini *parse_cmd_line(int argc, char **argv) +{ + int get_char; + __u64 lun_id = 0; /* default lun_id to use */ + int nafu; + int nbytes; + int i; + char *ini_path = NULL; + int fd; + struct stat sbuf; + void *map; + struct capikv_ini_elm *p_elm; + struct capikv_ini *p_ini; + + while ((get_char = getopt(argc, argv, "i:l:v:h")) != EOF) + { + switch (get_char) + { + case 'v' : /* trace level in decimal */ + sscanf(optarg, "%d", &trc_lvl); + break; + + case 'i' : /* path to init file */ + ini_path = optarg; + break; + + case 'h' : /* usage help */ + usage(argv[0]); + exit(0); + + default: + usage(argv[0]); + exit(-1); + } + } + + if (ini_path == NULL) { + nafu = argc - optind; /* number of afus specified in cmd line */ + + nbytes = sizeof(*p_ini) + + ((nafu > 1) ? (nafu - 1)*sizeof(*p_elm) : 0); + + p_ini = (struct capikv_ini *) malloc(nbytes); + if (p_ini == NULL) { + trace_1(LOG_ERR, "cannot allocate %d bytes\n", nbytes); + return NULL; + } + + memset(p_ini, 0, nbytes); + p_ini->sini_marker = 0x53494e49; + p_ini->nelm = nafu; + p_ini->size = sizeof(*p_elm); + + for (i = 0; i < nafu; i++) { + p_elm = &p_ini->elm[i]; + + // for this mode, the user enters the master dev path. + // also assume wwpns are already programmed off-line and + // master should leave them alone + // + p_elm->elmd_marker = 0x454c4d44; + strcpy(&p_elm->afu_dev_pathm[0], argv[i + optind]); + p_elm->lun_id = lun_id; + } + } + else { + fd = open(ini_path, O_RDONLY); + if (fd < 0) { + trace_2(LOG_ERR, "open of %s failed, errno %d\n", ini_path, errno); + return NULL; + } + + if (fstat(fd, &sbuf) != 0) { + trace_2(LOG_ERR, "fstat failed on %s, errno %d\n", ini_path, errno); + close(fd); + return NULL; + } + + map = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (map == MAP_FAILED) { + trace_1(LOG_ERR, "mmap failed, errno %d\n", errno); + close(fd); + return NULL; + } + + p_ini = (struct capikv_ini *) map; + } + + return p_ini; +} + + +int +main(int argc, char *argv[]) +{ + int i; + int rc; + cusfs_serv_disk_t *p_disk; + sigset_t set; + struct capikv_ini_elm *p_elm; + struct capikv_ini *p_ini; + struct afu_alloc *p_disk_a; + + + + + openlog("cusfs_serv", LOG_PERROR, LOG_USER); + + p_ini = parse_cmd_line(argc, argv); + + /* some sanity checks */ + if (p_ini == NULL || + p_ini->nelm == 0 || + p_ini->sini_marker != 0x53494e49) { + trace_0(LOG_ERR, "bad input parameters, exiting...\n"); + usage(argv[0]); // additional errors already logged by parse_cmd_line() + exit(-1); + } + gb.p_ini = p_ini; + + sigemptyset(&set); + sigaddset(&set, SIGRTMIN); + /* signal mask is inherited by all threads created from now on */ + pthread_sigmask(SIG_BLOCK, &set, NULL); + + rc = posix_memalign((void**)&p_disk_a, 0x1000, + sizeof(struct afu_alloc) * p_ini->nelm); + if (rc != 0) { + trace_1(LOG_ERR, "cannot allocate AFU structs, rc %d\n", rc); + exit(-1); + } + gb.p_disk_a = p_disk_a; + + p_elm = &p_ini->elm[0]; + for (i = 0; i < p_ini->nelm; i++) { + p_disk = &p_disk_a[i].disk; + rc = master_init(p_disk, p_elm); + if (rc != 0) { + trace_2(LOG_ERR, "error instantiating afu %s, rc %d\n", + p_elm->afu_dev_pathm, rc); + exit(-1); + } + + // advance to next element using correct size + p_elm = (struct capikv_ini_elm*) (((char*)p_elm) + p_ini->size); + } + + /* When all went well, start IPC threads to take requests */ + for (i = 0; i < p_ini->nelm; i++) { + pthread_create(&p_disk_a[i].disk.ipc_thread, NULL, afu_ipc_rx, + &p_disk_a[i].disk); + } + + + + + for (i = 0; i < p_ini->nelm; i++) { + pthread_join(p_disk_a[i].disk.ipc_thread, NULL); + cusfs_serv_disk_term(&p_disk_a[i].disk); + } + + + + + closelog(); + + return 0; +} diff --git a/src/usfs/cusfs_serv.h b/src/usfs/cusfs_serv.h new file mode 100644 index 00000000..10540c72 --- /dev/null +++ b/src/usfs/cusfs_serv.h @@ -0,0 +1,384 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/master/mserv.h $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +#ifndef _MSERVE_H +#define _MSERVE_H + + +#include +#include + + +#ifndef _AIX +#include +#else +#include +#endif /* !_AIX */ +#include +#include +#include "cflsh_usfs_client.h" + + +/******************************************************************** + * Queueing Macros + * + * The CUSFS_Q/DQ_NODE macros enqueue and dequeue nodes to the head + * or tail of a doubly-linked list. They assume that 'next' and 'prev' + * are the names of the queueing pointers. + * + *******************************************************************/ + +/* + * Give a node and the head and tail pointer for a list, enqueue + * the node at the head of the list. Assumes the list is + * doubly-linked and NULL-terminated at both ends, and that node + * is non-NULL. Casts to void allow commands of different data + * types than the list to be queued into the list. This is useful + * if one data type is a subset of another. + */ +#define CUSFS_Q_NODE_HEAD(head, tail, node,_node_prev,_node_next) \ +do { \ + \ + /* If node is valid */ \ + if ((node) != NULL) \ + { \ + (node)->_node_prev = NULL; \ + (node)->_node_next = (head); \ + \ + if ((head) == NULL) { \ + \ + /* List is empty; 'node' is also the tail */ \ + (tail) = (node); \ + \ + } else { \ + \ + /* List isn't empty;old head must point to 'node' */ \ + (head)->_node_prev = (node); \ + } \ + \ + (head) = (node); \ + } \ + \ +} while (0) + + + +/* + * Give a node and the head and tail pointer for a list, enqueue + * the node at the tail of the list. Assumes the list is + * doubly-linked and NULL-terminated at both ends, and that node + * is non-NULL. Casts to void allow commands of different data + * types than the list to be queued into the list. This is useful + * if one data type is a subset of another. + */ +#define CUSFS_Q_NODE_TAIL(head, tail, node,_node_prev,_node_next) \ +do { \ + \ + /* If node is valid */ \ + if ((node) != NULL) \ + { \ + \ + (node)->_node_prev = (tail); \ + (node)->_node_next = NULL; \ + \ + if ((tail) == NULL) { \ + \ + /* List is empty; 'node' is also the head */ \ + (head) = (node); \ + \ + } else { \ + \ + /* List isn't empty;old tail must point to 'node' */ \ + (tail)->_node_next = (node); \ + } \ + \ + (tail) = (node); \ + } \ + \ + \ + \ +} while (0) + + +/* + * Given a node and the head and tail pointer for a list, dequeue + * the node from the list. Assumes the list is doubly-linked and + * NULL-terminated at both ends, and that node is non-NULL. + * + * Casts to void allow commands of different data types than the + * list to be dequeued into the list. This is useful if one data + * type is a subset of another. + */ +#define CUSFS_DQ_NODE(head, tail, node,_node_prev,_node_next) \ +do { \ + /* If node is valid */ \ + if ((node) != NULL) \ + { \ + /* If node was head, advance the head to node's next*/ \ + if ((head) == (node)) \ + { \ + (head) = ((node)->_node_next); \ + } \ + \ + /* If node was tail, retract the tail to node's prev */ \ + if ((tail) == (node)) \ + { \ + (tail) = ((node)->_node_prev); \ + } \ + \ + /* A follower's predecessor is now node's predecessor*/ \ + if ((node)->_node_next) \ + { \ + (node)->_node_next->_node_prev = ((node)->_node_prev); \ + } \ + \ + /* A predecessor's follower is now node's follower */ \ + if ((node)->_node_prev) \ + { \ + (node)->_node_prev->_node_next = ((node)->_node_next); \ + } \ + \ + (node)->_node_next = NULL; \ + (node)->_node_prev = NULL; \ + } \ + \ + \ +} while(0) + + +/* Sizing parms: same context can be registered multiple times. + Therefore we allow MAX_CONNS > MAX_CONTEXT. +*/ + +#define MAX_RHT_PER_CONTEXT 16 /* num resource hndls per context */ +#define MAX_CONNS 512 /* num client connections per disk */ +#define MAX_CONN_TO_POLL 64 /* num fds to poll once */ +#define NUM_RRQ_ENTRY 16 /* for master issued cmds */ +#define NUM_CMDS 16 /* must be <= NUM_RRQ_ENTRY */ + + + + +/* flags in IOA status area for host use */ +#define B_DONE 0x01 +#define B_ERROR 0x02 /* set with B_DONE */ +#define B_TIMEOUT 0x04 /* set with B_DONE & B_ERROR */ + + + +#define MC_CRC_THRESH 100 /* threshold in 5 mins */ + +#define CL_SIZE 128 /* Processor cache line size */ +#define CL_SIZE_MASK 0x7F /* Cache line size mask */ + +#ifdef _REMOVE +/* + * A resource handle table (RHT) can be pointed to by multiple contexts. + * This happens when one context is duped to another. + * W/o dup, each context has its own resource handles that is visible + * only from that context. + * + * The rht_info refers to all resource handles of a context and not to + * a particular RHT entry or a single resource handle. + */ +typedef struct rht_info +{ + + int ref_cnt; /* num ctx_infos pointing to me */ +} rht_info_t; +#endif + + +/* Single AFU context can be pointed to by multiple client connections. + * The client can create multiple endpoints (mc_hndl_t) to the same + * (context + AFU). + */ +typedef struct ctx_info +{ + + + /* The rht_info pointer is initialized when the context is first + registered, can be changed on dup. + */ + + + /* all dup'ed contexts are in a doubly linked circular list */ + struct ctx_info *p_forw; + struct ctx_info *p_next; + + int ref_cnt; /* num conn_infos pointing to me */ +} ctx_info_t; + +/* forward decls */ +struct capikv_ini_elm; +typedef struct cusfs_serv_disk_s cusfs_serv_disk_t; +typedef struct conn_info conn_info_t; + + + +typedef cflsh_usfs_master_req_t mc_req_t; +typedef struct mc_resp mc_resp_t; + +/* On accept, the server allocates a connection info. + A connection can be associated with at most one AFU context. + */ +typedef struct conn_info +{ + int fd; /* accepted fd on server, -1 means entry is free */ + pid_t client_pid; /* client PID - trace use only */ + int client_fd; /* client's socket fd - trace use only */ + + int mode; /* registration mode - see cblk.h */ + + /* IPC receive rx fcn depends on state */ + + int (*rx)(cusfs_serv_disk_t *p_disk, struct conn_info *p_conn_info, + cflsh_usfs_master_req_t *p_req, cflsh_usfs_master_resp_t *p_resp); + struct conn_info *prev; + struct conn_info *next; + cflsh_usfs_master_resp_t mc_resp; + ctx_info_t *p_ctx_info;/* ptr to bound context + or NULL if not registered */ +} conn_info_t; + +typedef struct client_locks_s { + pid_t client_pid; + + conn_info_t *head; + conn_info_t *tail; + +} client_locks_t; + + +enum undo_level { + UNDO_NONE = 0, + UNDO_MLOCK, + UNDO_AFU_OPEN, + UNDO_OPEN_SOCK, + UNDO_BIND_SOCK, + UNDO_LISTEN, + UNDO_EPOLL_CREATE, + UNDO_EPOLL_ADD, + UNDO_AFU_ALL /* must be last */ +}; + +typedef struct cusfs_serv_disk_s { + /* Stuff requiring alignment go first. */ + + + /* Housekeeping data */ + char master_dev_path[MC_PATHLEN]; /* e. g. /dev/sg5 */ + struct conn_info conn_tbl[MAX_CONNS]; /* conn_tbl[0] is rsvd for listening */ + + char *name; /* ptr to last component in master_dev_path, e.g. afu1.0m */ + pthread_mutex_t mutex; /* for anything that needs serialization + e. g. to access afu */ + pthread_mutex_t err_mutex; /* for signalling error thread */ + pthread_cond_t err_cv; + int err_flag; +#define E_SYNC_INTR 0x1 /* synchronous error interrupt */ +#define E_ASYNC_INTR 0x2 /* asynchronous error interrupt */ + + /* AFU Shared Data */ + + /* LXTs are allocated dynamically in groups */ + + /* AFU HW */ + int afu_fd; + + char event_buf[0x1000]; /* Linux cxl event buffer (interrupts) */ + + client_locks_t locks[CFLSH_USFS_MASTER_LAST_ITEM]; + /* client IPC */ + int listen_fd; + int epfd; + struct sockaddr_un svr_addr; +#ifndef _AIX + struct epoll_event events[MAX_CONN_TO_POLL]; /* ready events */ +#else + struct pollfd events[MAX_CONN_TO_POLL]; /* ready events */ + int num_poll_events; +#endif /* !_AIX */ + + /* per AFU threads */ + pthread_t ipc_thread; + pthread_t rrq_thread; + pthread_t err_thread; + +} __attribute__ ((aligned (0x1000))) cusfs_serv_disk_t; + + +struct afu_alloc { + cusfs_serv_disk_t disk; + uint8_t page_pad[0x1000 - (sizeof(cusfs_serv_disk_t) & 0xFFF)]; +}; + + + +conn_info_t *alloc_connection(cusfs_serv_disk_t *p_disk, int fd); +void free_connection(cusfs_serv_disk_t *p_disk, conn_info_t *p_conn_info); +int afu_init(cusfs_serv_disk_t *p_disk, struct capikv_ini_elm *p_elm); +void undo_afu_init(cusfs_serv_disk_t *p_disk, enum undo_level level); +int cusfs_serv_disk_term(cusfs_serv_disk_t *p_disk); +void afu_err_intr_init(cusfs_serv_disk_t *p_disk); + + + +int do_master_register(cusfs_serv_disk_t *p_disk, + conn_info_t *p_conn_info, + uint64_t challenge); + +int do_master_unregister(cusfs_serv_disk_t *p_disk, + conn_info_t *p_conn_info); + +int do_master_lock(cusfs_serv_disk_t *p_disk, + conn_info_t *p_conn_info, + cflsh_usfs_master_lock_t lock, + uint64_t flags); + +int do_master_unlock(cusfs_serv_disk_t *p_disk, + conn_info_t *p_conn_info, + cflsh_usfs_master_lock_t lock, + uint64_t flags); + +int xfer_data(int fd, int op, void *buf, ssize_t exp_size); + +int rx_mcreg(cusfs_serv_disk_t *p_disk, struct conn_info *p_conn_info, + cflsh_usfs_master_req_t *p_req, mc_resp_t *p_resp); +int rx_ready(cusfs_serv_disk_t *p_disk, struct conn_info *p_conn_info, + cflsh_usfs_master_req_t *p_req, mc_resp_t *p_resp); + + + + +void *afu_ipc_rx(void *arg); + +int mkdir_p(char *file_path); + + +void periodic_hb(); + +void *sig_rx(void *arg); + +#endif /* _MSERVE_H */ diff --git a/src/usfs/libcflsh_usfs.exp b/src/usfs/libcflsh_usfs.exp new file mode 100644 index 00000000..2a4b6cb1 --- /dev/null +++ b/src/usfs/libcflsh_usfs.exp @@ -0,0 +1,163 @@ +* @(#)96 1.1 src/bos/usr/ccs/lib/libcflsh_block/libcflsh_block.exp, sysxcflashblock, bos720, 1534A_720 8/12/15 15:38:49 +* IBM_PROLOG_BEGIN_TAG +* This is an automatically generated prolog. +* +* bos720 src/bos/usr/ccs/lib/libcflsh_block/libcflsh_block.exp 1.1 +* +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ +* COMPONENT_NAME: (sysxcflashblock) +* +* FUNCTIONS: +* libcflsh_block.a shared library export symbols +* +* ORIGINS: 27 +* +* -- ( when +* combined with the aggregated modules for this product) +* OBJECT CODE ONLY SOURCE MATERIALS +* (C) COPYRIGHT International Business Machines Corp. 2015 +* All Rights Reserved +* +* US Government Users Restricted Rights - Use, duplication or +* disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +* +* shared library export symbols +* data +* text +cusfs_create_fs +cusfs_query_fs +cusfs_statfs +cusfs_statfs64 +cusfs_remove_fs +cusfs_mkdir +cusfs_rmdir +cusfs_opendir64 +cusfs_closedir64 +cusfs_readdir64_r +cusfs_readdir64 +cusfs_readdir +cusfs_telldir64 +cusfs_seekdir64 +cusfs_create_file +cusfs_create_link +cusfs_remove_file +cusfs_list_files +cusfs_rename +cusfs_init +cusfs_term +cusfs_open +cusfs_creat +cusfs_close +cusfs_read +cusfs_write +cusfs_fsync +cusfs_lseek +cusfs_llseek +cusfs_lseek64 +cusfs_fstat +cusfs_fstat64 +cusfs_fstat64x +cusfs_stat +cusfs_stat64x +cusfs_stat64 +cusfs_lstat +cusfs_lstat64 +cusfs_lstat64x +cusfs_readlink +cusfs_access +cusfs_fchown +cusfs_fchmod +cusfs_ftruncate +cusfs_ftruncate64 +cusfs_utime +cusfs_link +cusfs_unlink +cusfs_fsck +cusfs_aio_read +cusfs_aio_read64 +cusfs_aio_write +cusfs_aio_write64 +cusfs_aio_error +cusfs_aio_error64 +cusfs_aio_return +cusfs_aio_return64 +cusfs_aio_fsync +cusfs_aio_fsync64 +cusfs_aio_cancel +cusfs_aio_cancel64 +cusfs_aio_suspend +cusfs_aio_suspend64 +statfs +statfs64 +fcntl +dup2 +dup +posix_fallocate +posix_fadvise +mkdir +rmdir +opendir64 +closedir64 +readdir64_r +readdir64 +readdir +rename +creat +open +open64 +close +read +write +lseek +lseek64 +llseek +fstat +fstatx +fstat64 +fstat64x +statx +stat64 +stat64x +lstat64 +lstat64x +readlink +access +fchmod +fchown +ftruncate +ftruncate64 +link +unlink +utime +_posix_aio_read +_posix_aio_read64 +_posix_aio_write +_posix_aio_write64 +_posix_aio_error +_posix_aio_error64 +_posix_aio_return +_posix_aio_return64 +_posix_aio_fsync +_posix_aio_fsync64 +_posix_aio_cancel +_posix_aio_cancel64 +_posix_aio_suspend +_posix_aio_suspend64 diff --git a/src/usfs/libcflsh_usfs_exportmap b/src/usfs/libcflsh_usfs_exportmap new file mode 100644 index 00000000..96f53f0f --- /dev/null +++ b/src/usfs/libcflsh_usfs_exportmap @@ -0,0 +1,151 @@ +'/* +* %Z%%M% %I% %W% %G% %U% +*/ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/block/libcflash_exportmap $ */ +/* */ +/* IBM Data Engine for NoSQL - Power Systems Edition User Library Project */ +/* */ +/* Contributors Listed Below - COPYRIGHT 2014,2015 */ +/* [+] International Business Machines Corp. */ +/* */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); */ +/* you may not use this file except in compliance with the License. */ +/* You may obtain a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or */ +/* implied. See the License for the specific language governing */ +/* permissions and limitations under the License. */ +/* */ +/* IBM_PROLOG_END_TAG */ + + +/* +* shared library export symbols +* data +* text +*/ +{ +global: cusfs_init; cusfs_term; + cusfs_create_fs; + cusfs_query_fs; + cusfs_statfs; + cusfs_statfs64; + cusfs_remove_fs; + cusfs_mkdir; + cusfs_rmdir; + cusfs_opendir; + cusfs_closedir; + cusfs_readdir_r; + cusfs_readdir; + cusfs_telldir; + cusfs_seekdir; + cusfs_create_file; + cusfs_create_link; + cusfs_remove_file; + cusfs_list_files; + cusfs_rename; + cusfs_open; + cusfs_creat; + cusfs_close; + cusfs_read; + cusfs_write; + cusfs_fsync; + cusfs_lseek; + cusfs_llseek; + cusfs_lseek64; + cusfs_lstat; + cusfs_lstat64; + cusfs_fstat; + cusfs_fstat64; + cusfs_stat; + cusfs_stat64; + cusfs_readlink; + cusfs_access; + cusfs_fchown; + cusfs_fchmod; + cusfs_ftruncate; + cusfs_ftruncate64; + cusfs_utime; + cusfs_link; + cusfs_unlink; + cusfs_fsck; + cusfs_aio_read; + cusfs_aio_read64; + cusfs_aio_write; + cusfs_aio_write64; + cusfs_aio_error; + cusfs_aio_error64; + cusfs_aio_return; + cusfs_aio_return64; + cusfs_aio_fsync; + cusfs_aio_fsync64; + cusfs_aio_cancel; + cusfs_aio_cancel64; + cusfs_aio_suspend; + cusfs_aio_suspend64; + statfs; + statfs64; + fcntl; + dup; + dup2; + posix_fallocate; + posix_fallocate64; + posix_fadvise; + posix_fadvise64; + mkdir; + rmdir; + opendir; + closedir; + readdir_r; + readdir; + rename; + creat; + open; + open64; + close; + read; + write; + lseek; + lseek64; + __xstat; + __xstat64; + __lxstat; + __lxstat64; + __fxstat; + __fxstat64; + fstat; + stat; + lstat; + readlink; + access; + fchmod; + fchown; + ftruncate; + ftruncate64; + link; + unlink; + utime; + aio_read; + aio_read64; + aio_write; + aio_write64; + aio_error; + aio_error64; + aio_return; + aio_return64; + aio_fsync; + aio_fsync64; + aio_cancel; + aio_cancel64; + aio_suspend; + aio_suspend64; +local: *; +}; diff --git a/src/usfs/makefile b/src/usfs/makefile new file mode 100644 index 00000000..5c5e1d9d --- /dev/null +++ b/src/usfs/makefile @@ -0,0 +1,78 @@ +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/usfs/makefile $ +# +# IBM Data Engine for NoSQL - Power Systems Edition User Library Project +# +# Contributors Listed Below - COPYRIGHT 2014,2015 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +ROOTPATH = ../.. + +CFLAGS += -I$(ROOTPATH)/src/include -Wno-format-security +CFLAGS64 += -I$(ROOTPATH)/src/include + +#if BLOCK_FILEMODE is enabled, then tell the userspace code as much +#pass down as a #define to the underlying code +ifdef BLOCK_FILEMODE_ENABLED +ifeq ($(BLOCK_FILEMODE_ENABLED),1) + CUSTOMFLAGS += -DBLOCK_FILEMODE_ENABLED +endif +endif + +# Incase user want to use semop based locking mechanism ; +# _MASTER_LOCK is required to define here +#CUSTOMFLAGS += -D_MASTER_LOCK + +# Incase user want to use unix socket based locking mechanism; +# _MASTER_LOCK_CLIENT and __MASTER_LOCK both is required to define here +#CUSTOMFLAGS += -D_MASTER_LOCK -D_MASTER_LOCK_CLIENT + +MODULE = cflsh_usfs + +OBJS = cflsh_usfs.o cflsh_usfs_utils.o cflsh_usfs_disk.o cflsh_usfs_inode.o cflsh_usfs_client.o cflsh_usfs_wrapper.o + +OBJS64 = cflsh_usfs.64o cflsh_usfs_utils.64o cflsh_usfs_disk.64o cflsh_usfs_inode.64o cflsh_usfs_client.64o cflsh_usfs_wrapper.64o + + +UNAME=$(shell uname) +ifeq ($(UNAME),AIX) +MODLIBS = -lpthreads -lcflsh_block +MODULE_LINKLIBS = ${MODLIBS} +EXPFLAGS = -bE:libcflsh_usfs.exp +else +MODLIBS = -lpthread -lcflsh_block -ldl +MODULE_LINKLIBS = ${MODLIBS} -Wl,--version-script=libcflsh_usfs_exportmap +endif + +ifeq ($(UNAME),AIX) +LINKLIBS +=-lpthreads -lcflsh_block -l${MODULE} +else +LINKLIBS +=-lpthread -lcflsh_block -ldl -l${MODULE} -lrt +endif + + +LIBPATHS = -L${ROOTPATH}/img + +#SUBDIRS = test.d + +PGMS = cusfs_serv +PROGRAMS = $(addprefix ${PGMDIR}/, ${PGMS}) +#PROGRAMS64 = $(addprefix ${PGMDIR}/, ${PGMS}) + +include ${ROOTPATH}/config.mk