Skip to content

Commit 7581a88

Browse files
secwallsecwall
authored
Allow luajit to coexist with built-in lua
Signed-off-by: secwall <secwall@yandex-team.ru>
1 parent 2be43e2 commit 7581a88

11 files changed

Lines changed: 280 additions & 73 deletions

‎.github/workflows/tests.yml‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,25 +8,40 @@ on:
88
branches: [ master ]
99

1010
jobs:
11-
tests:
12-
name: tests
11+
replacement-tests:
12+
name: Tests - LuaJIT as replacement
1313
runs-on: ubuntu-24.04
1414
steps:
1515
- name: Check out code
1616
uses: actions/checkout@v6
1717

1818
- name: Update submodule
19-
run: |
20-
git submodule update --init --recursive
19+
run: git submodule update --init --recursive
2120

2221
- name: Get deps
23-
run: |
24-
sudo apt-get install pkg-config tcl8.6 tclx cmake libssl-dev
22+
run: sudo apt-get install pkg-config tcl8.6 tclx cmake libssl-dev
2523

2624
- name: Build
27-
run: |
28-
./build.sh --with-tests
25+
run: ./build.sh --with-tests
2926

30-
- name: Run tests
31-
run: |
32-
./tests/run-valkey-tests.sh --skiptest "Process title set as expected"
27+
- name: Run replacement tests
28+
run: ./tests/replacement/run-replacement-tests.sh --skiptest "Process title set as expected"
29+
30+
coexistence-tests:
31+
name: Tests - LuaJIT coexistence
32+
runs-on: ubuntu-24.04
33+
steps:
34+
- name: Check out code
35+
uses: actions/checkout@v6
36+
37+
- name: Update submodule
38+
run: git submodule update --init --recursive
39+
40+
- name: Get deps
41+
run: sudo apt-get install pkg-config tcl8.6 tclx cmake libssl-dev
42+
43+
- name: Build
44+
run: ./build.sh --with-tests
45+
46+
- name: Run coexistence tests
47+
run: ./tests/coexistence/run-coexistence-tests.sh --skiptest "Process title set as expected"

‎setup-tests.sh‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,21 +22,12 @@ else
2222
echo "Valkey already downloaded at $VALKEY_DIR"
2323
fi
2424

25-
# Build Valkey without built-in Lua
25+
# Build Valkey (with built-in Lua)
2626
echo ""
2727
echo "Building Valkey server..."
2828
cd "$VALKEY_DIR"
29-
make BUILD_LUA=no -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
30-
31-
# Copy custom test files to Valkey test directory
32-
echo ""
33-
echo "Copying custom tests..."
34-
if [ -d "$SCRIPT_DIR/tests" ] && [ -n "$(ls -A $SCRIPT_DIR/tests/*.tcl 2>/dev/null)" ]; then
35-
cp "$SCRIPT_DIR/tests/"*.tcl "$VALKEY_DIR/tests/unit/"
36-
echo " Custom tests copied to $VALKEY_DIR/tests/unit/"
37-
else
38-
echo " No custom test files found in $SCRIPT_DIR/tests/"
39-
fi
29+
make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
30+
echo "Built Valkey with built-in Lua"
4031

4132
echo ""
4233
echo "== Test setup complete! =="

‎src/engine_luajit.c‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,12 @@
3636
extern int luaopen_ffi(lua_State *L);
3737
extern int luaopen_jit(lua_State *L);
3838

39-
#define LUA_ENGINE_NAME "LUA"
39+
#define DEFAULT_ENGINE_NAME "LUA"
4040
#define REGISTRY_FUNC_CACHE_NAME "__func_cache"
4141

42+
static ValkeyModuleString *engine_name_str = NULL;
43+
static const char *engine_name_cstr = NULL;
44+
4245
static int luajitFFIGetCurrentContext(lua_State *lua) {
4346
lua_getfield(lua, LUA_REGISTRYINDEX, "__ffi_ctx");
4447
return 1;
@@ -862,6 +865,23 @@ static int ffiSetConfig(const char *name, int val, void *privdata, ValkeyModuleS
862865
return VALKEYMODULE_OK;
863866
}
864867

868+
static ValkeyModuleString *engineNameGet(const char *name, void *privdata) {
869+
VALKEYMODULE_NOT_USED(name);
870+
VALKEYMODULE_NOT_USED(privdata);
871+
return engine_name_str;
872+
}
873+
874+
static int engineNameSet(const char *name, ValkeyModuleString *val, void *privdata, ValkeyModuleString **err) {
875+
VALKEYMODULE_NOT_USED(name);
876+
VALKEYMODULE_NOT_USED(privdata);
877+
VALKEYMODULE_NOT_USED(err);
878+
if (engine_name_str) ValkeyModule_FreeString(NULL, engine_name_str);
879+
ValkeyModule_RetainString(NULL, val);
880+
engine_name_str = val;
881+
engine_name_cstr = ValkeyModule_StringPtrLen(val, NULL);
882+
return VALKEYMODULE_OK;
883+
}
884+
865885
static luajitEngineCtx *engine_ctx = NULL;
866886

867887
int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx,
@@ -889,8 +909,25 @@ int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx,
889909
return VALKEYMODULE_ERR;
890910
}
891911

912+
if (ValkeyModule_RegisterStringConfig(ctx,
913+
"engine-name",
914+
DEFAULT_ENGINE_NAME,
915+
VALKEYMODULE_CONFIG_IMMUTABLE,
916+
engineNameGet,
917+
engineNameSet,
918+
NULL,
919+
NULL) == VALKEYMODULE_ERR) {
920+
ValkeyModule_Log(ctx, "warning", "Failed to register engine-name config");
921+
return VALKEYMODULE_ERR;
922+
}
923+
892924
engine_ctx = createEngineContext(ctx);
893925

926+
if (!engine_name_str) {
927+
engine_name_str = ValkeyModule_CreateString(NULL, DEFAULT_ENGINE_NAME, strlen(DEFAULT_ENGINE_NAME));
928+
engine_name_cstr = ValkeyModule_StringPtrLen(engine_name_str, NULL);
929+
}
930+
894931
if (ValkeyModule_LoadConfigs(ctx) == VALKEYMODULE_ERR) {
895932
ValkeyModule_Log(ctx, "warning", "Failed to load LuaJIT module configs");
896933
destroyEngineContext(engine_ctx);
@@ -920,12 +957,12 @@ int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx,
920957
};
921958

922959
int result = ValkeyModule_RegisterScriptingEngine(ctx,
923-
LUA_ENGINE_NAME,
960+
engine_name_cstr,
924961
engine_ctx,
925962
&methods);
926963

927964
if (result == VALKEYMODULE_ERR) {
928-
ValkeyModule_Log(ctx, "warning", "Failed to register LUA scripting engine");
965+
ValkeyModule_Log(ctx, "warning", "Failed to register '%s' scripting engine", engine_name_cstr);
929966
destroyEngineContext(engine_ctx);
930967
engine_ctx = NULL;
931968
return VALKEYMODULE_ERR;
@@ -935,20 +972,26 @@ int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx,
935972

936973
ValkeyModule_Log(ctx, "notice",
937974
"LuaJIT scripting engine registered as '%s' (per-user isolation)",
938-
LUA_ENGINE_NAME);
975+
engine_name_cstr);
939976

940977
return VALKEYMODULE_OK;
941978
}
942979

943980
int ValkeyModule_OnUnload(ValkeyModuleCtx *ctx) {
944-
if (ValkeyModule_UnregisterScriptingEngine(ctx, LUA_ENGINE_NAME) != VALKEYMODULE_OK) {
981+
if (ValkeyModule_UnregisterScriptingEngine(ctx, engine_name_cstr) != VALKEYMODULE_OK) {
945982
ValkeyModule_Log(ctx, "error", "Failed to unregister LuaJIT engine");
946983
return VALKEYMODULE_ERR;
947984
}
948985

949986
destroyEngineContext(engine_ctx);
950987
engine_ctx = NULL;
951988

989+
if (engine_name_str) {
990+
ValkeyModule_FreeString(NULL, engine_name_str);
991+
engine_name_str = NULL;
992+
engine_name_cstr = NULL;
993+
}
994+
952995
ValkeyModule_Log(ctx, "notice", "LuaJIT scripting engine unloaded");
953996

954997
return VALKEYMODULE_OK;

‎tests/coexistence/engine-routing.tcl‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
proc get_function_code {engine library_name function_name body} {
2+
return [format "#!%s name=%s\nserver.register_function('%s', function(KEYS, ARGV)\n %s \nend)" $engine $library_name $function_name $body]
3+
}
4+
5+
start_server {tags {"engine-coexistence"} overrides {luajit.enable-ffi-api yes}} {
6+
test {EVAL uses built-in Lua engine} {
7+
set result [r EVAL "return _VERSION" 0]
8+
assert_match {*Lua 5.*} $result
9+
}
10+
11+
test {FUNCTION with #!lua shebang uses built-in Lua} {
12+
r FUNCTION LOAD [get_function_code lua lib1 get_version {return _VERSION}]
13+
set result [r FCALL get_version 0]
14+
assert_match {*Lua 5.*} $result
15+
}
16+
17+
test {FUNCTION with #!luajit shebang uses LuaJIT} {
18+
r SET KEY VALUE
19+
r FUNCTION LOAD [get_function_code luajit lib2 get_test {return VKM.random_key()}]
20+
set result [r FCALL get_test 0]
21+
assert_match {*KEY*} $result
22+
}
23+
}

‎tests/coexistence/function-coexistence.tcl‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
proc get_function_code {engine library_name function_name body} {
2+
return [format "#!%s name=%s\nserver.register_function('%s', function(KEYS, ARGV)\n %s \nend)" $engine $library_name $function_name $body]
3+
}
4+
5+
start_server {tags {"function-coexistence"}} {
6+
test {Both engines can be used simultaneously} {
7+
r FUNCTION LOAD [get_function_code lua simlib1 from_lua {return 'from-Lua'}]
8+
r FUNCTION LOAD [get_function_code luajit simlib2 from_luajit {return 'from-LuaJIT'}]
9+
10+
assert_equal [r FCALL from_lua 0] {from-Lua}
11+
assert_equal [r FCALL from_luajit 0] {from-LuaJIT}
12+
}
13+
14+
test {Both engines can interact with the same keys} {
15+
r FUNCTION LOAD [get_function_code lua writelib lua_write {return server.call('set', KEYS[1], ARGV[1])}]
16+
r FUNCTION LOAD [get_function_code luajit readlib luajit_read {return server.call('get', KEYS[1])}]
17+
18+
r FCALL lua_write 1 mykey hello
19+
assert_equal [r FCALL luajit_read 1 mykey] {hello}
20+
}
21+
22+
test {LuaJIT function can read data written by built-in Lua function} {
23+
r FUNCTION LOAD [get_function_code lua wlib2 lua_set {return server.call('set', KEYS[1], ARGV[1])}]
24+
r FUNCTION LOAD [get_function_code luajit rlib2 luajit_get {return server.call('get', KEYS[1])}]
25+
26+
r FCALL lua_set 1 crosskey crossvalue
27+
assert_equal [r FCALL luajit_get 1 crosskey] {crossvalue}
28+
}
29+
30+
test {Built-in Lua function can read data written by LuaJIT function} {
31+
r FUNCTION LOAD [get_function_code luajit wlib3 luajit_set {return server.call('set', KEYS[1], ARGV[1])}]
32+
r FUNCTION LOAD [get_function_code lua rlib3 lua_get {return server.call('get', KEYS[1])}]
33+
34+
r FCALL luajit_set 1 crosskey2 crossvalue2
35+
assert_equal [r FCALL lua_get 1 crosskey2] {crossvalue2}
36+
}
37+
38+
test {Error in one engine doesn't affect the other} {
39+
r FUNCTION LOAD [get_function_code lua oklib lua_ok {return 'lua-ok'}]
40+
r FUNCTION LOAD [get_function_code luajit errlib luajit_err {return server.call('invalid_command')}]
41+
r FUNCTION LOAD [get_function_code luajit oklib2 luajit_ok {return 'luajit-ok'}]
42+
43+
catch {r FCALL luajit_err 0} e
44+
assert_match {*ERR*} $e
45+
46+
assert_equal [r FCALL lua_ok 0] {lua-ok}
47+
assert_equal [r FCALL luajit_ok 0] {luajit-ok}
48+
}
49+
50+
test {Both engines can use cjson} {
51+
r FUNCTION LOAD [get_function_code lua jsonlib1 lua_json {return cjson.encode({a = 1})}]
52+
r FUNCTION LOAD [get_function_code luajit jsonlib2 luajit_json {return cjson.encode({a = 1})}]
53+
54+
set lua_result [r FCALL lua_json 0]
55+
set luajit_result [r FCALL luajit_json 0]
56+
57+
assert_match {*"a":1*} $lua_result
58+
assert_match {*"a":1*} $luajit_result
59+
}
60+
}

‎tests/coexistence/run-coexistence-tests.sh‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
set -e
3+
4+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
5+
REPO_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")"
6+
VALKEY_DIR="$REPO_ROOT/build/valkey"
7+
LUAJIT_MODULE="$REPO_ROOT/build/libvalkeyluajit.so"
8+
9+
if [ ! -f "$VALKEY_DIR/runtest" ]; then
10+
echo "Error: Valkey test runner not found at $VALKEY_DIR/runtest"
11+
echo ""
12+
echo "Please build the project first:"
13+
echo " BUILD_LUA=yes ./build.sh --with-tests"
14+
exit 1
15+
fi
16+
17+
if [ ! -f "$LUAJIT_MODULE" ]; then
18+
echo "Error: LuaJIT module not found at $LUAJIT_MODULE"
19+
echo ""
20+
echo "Please build the project first:"
21+
echo " ./build.sh"
22+
exit 1
23+
fi
24+
25+
echo "Running coexistence tests (LuaJIT + built-in Lua)..."
26+
echo "Valkey: $VALKEY_DIR"
27+
echo "Module: $LUAJIT_MODULE"
28+
echo "Engine name: LUAJIT"
29+
echo ""
30+
31+
EXTRA_SKIP_ARGS=(
32+
"--skiptest" "/FUNCTION - function stats"
33+
"--skiptest" "/FUNCTION - test function stats"
34+
"--skiptest" "CONFIG sanity"
35+
)
36+
37+
# Copy coexistence tests
38+
cp "$SCRIPT_DIR"/*.tcl "$VALKEY_DIR/tests/unit/"
39+
40+
cd "$VALKEY_DIR"
41+
./runtest --config loadmodule "$LUAJIT_MODULE" "${EXTRA_SKIP_ARGS[@]}" --config luajit.engine-name LUAJIT "$@"

tests/function-isolation.tcl renamed to tests/replacement/function-isolation.tcl

File renamed without changes.

‎tests/replacement/run-replacement-tests.sh‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#!/bin/bash
2+
set -e
3+
4+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
5+
REPO_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")"
6+
VALKEY_DIR="$REPO_ROOT/build/valkey"
7+
LUAJIT_MODULE="$REPO_ROOT/build/libvalkeyluajit.so"
8+
9+
if [ ! -f "$VALKEY_DIR/runtest" ]; then
10+
echo "Error: Valkey test runner not found at $VALKEY_DIR/runtest"
11+
echo ""
12+
echo "Please build the project first:"
13+
echo " ./build.sh --with-tests"
14+
exit 1
15+
fi
16+
17+
if [ ! -f "$LUAJIT_MODULE" ]; then
18+
echo "Error: LuaJIT module not found at $LUAJIT_MODULE"
19+
echo ""
20+
echo "Please build the project first:"
21+
echo " ./build.sh"
22+
exit 1
23+
fi
24+
25+
echo "Running replacement tests (LuaJIT in place of built-in Lua)..."
26+
echo "Valkey: $VALKEY_DIR"
27+
echo "Module: $LUAJIT_MODULE"
28+
echo ""
29+
30+
# Skip tests that require functionality not implemented in valkey-luajit:
31+
# 1. Tests expecting undefined global variables to raise errors (per-user isolation
32+
# provides security without this feature)
33+
# 2. Tests requiring Lua debug API (not implemented)
34+
# 3. Tests requiring specific error message formats (restricted APIs, security maintained)
35+
# 4. Tests for CVE-2024-2024-31449 (PUC-Rio Lua bit library stack overflow). LuaJIT's bit library
36+
# is not vulnerable to this vulnerability. The test verifies the built-in Lua fix.
37+
# Security is maintained through per-user state isolation.
38+
EXTRA_SKIP_ARGS=(
39+
"--skiptest" "Test may-replicate commands are rejected in RO scripts"
40+
"--skiptest" "Test loadfile are not available"
41+
"--skiptest" "Test dofile are not available"
42+
"--skiptest" "Test print are not available"
43+
"--skiptest" "LUA test pcall with error"
44+
"--skiptest" "Dynamic reset of lua engine with insecure API config change"
45+
"--skiptest" "LIBRARIES - math.random from function load"
46+
"--skiptest" "LIBRARIES - redis.call from function load"
47+
"--skiptest" "LIBRARIES - redis.setresp from function load"
48+
"--skiptest" "LIBRARIES - redis.set_repl from function load"
49+
"--skiptest" "LIBRARIES - redis.acl_check_cmd from function load"
50+
"--skiptest" "LIBRARIES - malicious access test"
51+
"--skiptest" "LIBRARIES - verify global protection on the load run"
52+
"--skiptest" "LIBRARIES - register function inside a function"
53+
"--skiptest" "FUNCTION - test getmetatable on script load"
54+
"--skiptest" "/trick global protection"
55+
"--skiptest" "/trick readonly table"
56+
"--skiptest" "/Globals protection"
57+
"--skiptest" "/Test scripting debug"
58+
"--skiptest" "/cmsgpack can pack and unpack circular references"
59+
"--skiptest" "lua bit.tohex bug"
60+
"--skiptest" "EVAL - JSON string encoding a string larger than 2GB"
61+
"--skiptest" "/Active Defrag eval scripts"
62+
"--skiptest" "CONFIG sanity"
63+
)
64+
65+
# Copy replacement tests
66+
cp "$SCRIPT_DIR"/*.tcl "$VALKEY_DIR/tests/unit/"
67+
68+
cd "$VALKEY_DIR"
69+
./runtest --config loadmodule "$LUAJIT_MODULE" "${EXTRA_SKIP_ARGS[@]}" "$@"

tests/scripting-ffi.tcl renamed to tests/replacement/scripting-ffi.tcl

File renamed without changes.

tests/scripting-isolation.tcl renamed to tests/replacement/scripting-isolation.tcl

File renamed without changes.

0 commit comments

Comments
 (0)