Merge pull request #244 from DrXiao/feat/support-dynamic-linking

Support Arm dynamic linking

Author: jserv

.github/workflows/main.yml

@@ -9,6 +9,14 @@ jobs:
       matrix:
         compiler: [gcc, clang]
         architecture: [arm, riscv]
+        link_mode: [static]
+        include:
+          - compiler: gcc
+            architecture: arm
+            link_mode: dynamic
+          - compiler: clang
+            architecture: arm
+            link_mode: dynamic
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -18,28 +26,52 @@ jobs:
           sudo apt-get install -q -y graphviz jq
           sudo apt-get install -q -y qemu-user
           sudo apt-get install -q -y build-essential
+          sudo apt-get install -q -y gcc-arm-linux-gnueabihf
+      - name: Determine static or dynamic linking mode
+        id: determine-mode
+        run: |
+          if [ "${{ matrix.link_mode }}" = "dynamic" ]; then
+            echo "Use dynamic linking mode"
+            echo "DYNLINK=1" >> "$GITHUB_OUTPUT"
+          else
+            echo "Use static linking mode"
+            echo "DYNLINK=0" >> "$GITHUB_OUTPUT"
+          fi
       - name: Build artifacts
         env:
           CC: ${{ matrix.compiler }}
         run: |
-          make distclean config ARCH=${{ matrix.architecture }}
+          make ARCH=${{ matrix.architecture }} DYNLINK=${{ steps.determine-mode.outputs.DYNLINK }}
       - name: IR regression tests
         run: |
-          make check-snapshot || exit 1
+          make check-snapshot DYNLINK=${{ steps.determine-mode.outputs.DYNLINK }} || exit 1
       - name: Sanitizer-enabled stage 0 tests
         env:
           CC: ${{ matrix.compiler }}
         run: |
-          make check-sanitizer || exit 1
+          make check-sanitizer DYNLINK=${{ steps.determine-mode.outputs.DYNLINK }} || exit 1
       - name: Unit tests
         run: |
-          make check || exit 1
+          make check DYNLINK=${{ steps.determine-mode.outputs.DYNLINK }} || exit 1
 
   host-arm:
     runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        link_mode: [static, dynamic]
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
+    - name: Determine static or dynamic linking mode
+      id: determine-mode
+      run: |
+        if [ "${{ matrix.link_mode }}" = "dynamic" ]; then
+          echo "Use dynamic linking mode"
+          echo "DYNLINK=1" >> "$GITHUB_OUTPUT"
+        else
+          echo "Use static linking mode"
+          echo "DYNLINK=0" >> "$GITHUB_OUTPUT"
+        fi
     - name: Build artifacts
       # The GitHub Action for non-x86 CPU
       # https://github.com/uraimo/run-on-arch-action
@@ -52,9 +84,9 @@ jobs:
           apt-get update -qq -y
           apt-get install -yqq build-essential
         run: |
-          make config ARCH=arm
-          make check-sanitizer || exit 1
-          make check || exit 1
+          make ARCH=arm DYNLINK=${{ steps.determine-mode.outputs.DYNLINK }}
+          make check-sanitizer DYNLINK=${{ steps.determine-mode.outputs.DYNLINK }} || exit 1
+          make check DYNLINK=${{ steps.determine-mode.outputs.DYNLINK }} || exit 1
 
   coding-style:
     runs-on: ubuntu-24.04

Makefile

@@ -43,12 +43,27 @@ HOST_ARCH = $(shell arch 2>/dev/null)
 SRCDIR := $(shell find src -type d)
 LIBDIR := $(shell find lib -type d)
 
+BUILTIN_LIBC_SOURCE ?= c.c
+BUILTIN_LIBC_HEADER := c.h
+STAGE0_FLAGS ?= --dump-ir
+STAGE1_FLAGS ?=
+DYNLINK ?= 0
+ifeq ($(DYNLINK),1)
+    ifeq ($(ARCH),riscv)
+        # TODO: implement dynamic linking for RISC-V.
+        $(error "Dynamic linking mode is not implemented for RISC-V")
+    endif
+    STAGE0_FLAGS += --dynlink
+    STAGE1_FLAGS += --dynlink
+endif
+
 SRCS := $(wildcard $(patsubst %,%/main.c, $(SRCDIR)))
 OBJS := $(SRCS:%.c=$(OUT)/%.o)
 deps := $(OBJS:%.o=%.o.d)
 TESTS := $(wildcard tests/*.c)
 TESTBINS := $(TESTS:%.c=$(OUT)/%.elf)
-SNAPSHOTS := $(foreach SNAPSHOT_ARCH,$(ARCHS), $(patsubst tests/%.c, tests/snapshots/%-$(SNAPSHOT_ARCH).json, $(TESTS)))
+SNAPSHOTS = $(foreach SNAPSHOT_ARCH,$(ARCHS), $(patsubst tests/%.c, tests/snapshots/%-$(SNAPSHOT_ARCH)-static.json, $(TESTS)))
+SNAPSHOTS += $(patsubst tests/%.c, tests/snapshots/%-arm-dynamic.json, $(TESTS))
 
 all: config bootstrap
 
@@ -72,44 +87,61 @@ config:
 
 $(OUT)/tests/%.elf: tests/%.c $(OUT)/$(STAGE0)
 	$(VECHO) "  SHECC\t$@\n"
-	$(Q)$(OUT)/$(STAGE0) --dump-ir -o $@ $< > $(basename $@).log ; \
+	$(Q)$(OUT)/$(STAGE0) $(STAGE0_FLAGS) -o $@ $< > $(basename $@).log ; \
 	chmod +x $@ ; $(PRINTF) "Running $@ ...\n"
 	$(Q)$(TARGET_EXEC) $@ && $(call pass)
 
-check: check-stage0 check-stage2
+check: check-stage0 check-stage2 check-abi-stage0 check-abi-stage2
 
 check-stage0: $(OUT)/$(STAGE0) $(TESTBINS) tests/driver.sh
 	$(VECHO) "  TEST STAGE 0\n"
-	tests/driver.sh 0
+	tests/driver.sh 0 $(DYNLINK)
 
 check-stage2: $(OUT)/$(STAGE2) $(TESTBINS) tests/driver.sh
 	$(VECHO) "  TEST STAGE 2\n"
-	tests/driver.sh 2
+	tests/driver.sh 2 $(DYNLINK)
 
 check-sanitizer: $(OUT)/$(STAGE0)-sanitizer tests/driver.sh
 	$(VECHO) "  TEST STAGE 0 (with sanitizers)\n"
 	$(Q)cp $(OUT)/$(STAGE0)-sanitizer $(OUT)/shecc
-	tests/driver.sh 0
+	tests/driver.sh 0 $(DYNLINK)
 	$(Q)rm $(OUT)/shecc
 
 check-snapshots: $(OUT)/$(STAGE0) $(SNAPSHOTS) tests/check-snapshots.sh
-	$(Q)$(foreach SNAPSHOT_ARCH, $(ARCHS), $(MAKE) distclean config check-snapshot ARCH=$(SNAPSHOT_ARCH) --silent;)
-	$(VECHO) "Switching backend back to %s\n" $(ARCH)
-	$(Q)$(MAKE) distclean config ARCH=$(ARCH) --silent
+	$(Q)$(foreach SNAPSHOT_ARCH, $(ARCHS), $(MAKE) distclean config check-snapshot ARCH=$(SNAPSHOT_ARCH) DYNLINK=0 --silent;)
+	$(Q)$(MAKE) distclean config check-snapshot ARCH=arm DYNLINK=1 --silent
+	$(VECHO) "Switching backend back to %s (DYNLINK=0)\n" arm
+	$(Q)$(MAKE) distclean config ARCH=arm DYNLINK=0 --silent
 
 check-snapshot: $(OUT)/$(STAGE0) tests/check-snapshots.sh
-	$(VECHO) "Checking snapshot for %s\n" $(ARCH)
-	tests/check-snapshots.sh $(ARCH)
+	$(VECHO) "Checking snapshot for %s (DYNLINK=%s)\n" $(ARCH) $(DYNLINK)
+	tests/check-snapshots.sh $(ARCH) $(DYNLINK)
 	$(VECHO) "  OK\n"
 
+# TODO: Add an ABI conformance test suite for the RISC-V architecture
+check-abi-stage0: $(OUT)/$(STAGE0)
+	$(Q)if [ "$(ARCH)" = "arm" ]; then \
+		tests/$(ARCH)-abi.sh 0 $(DYNLINK); \
+	else \
+		echo "Skip ABI compliance validation"; \
+	fi
+
+check-abi-stage2: $(OUT)/$(STAGE2)
+	$(Q)if [ "$(ARCH)" = "arm" ]; then \
+		tests/$(ARCH)-abi.sh 2 $(DYNLINK); \
+	else \
+		echo "Skip ABI compliance validation"; \
+	fi
+
 update-snapshots: tests/update-snapshots.sh
-	$(Q)$(foreach SNAPSHOT_ARCH, $(ARCHS), $(MAKE) distclean config update-snapshot ARCH=$(SNAPSHOT_ARCH) --silent;)
-	$(VECHO) "Switching backend back to %s\n" $(ARCH)
-	$(Q)$(MAKE) distclean config ARCH=$(ARCH) --silent
+	$(Q)$(foreach SNAPSHOT_ARCH, $(ARCHS), $(MAKE) distclean config update-snapshot ARCH=$(SNAPSHOT_ARCH) DYNLINK=0 --silent;)
+	$(Q)$(MAKE) distclean config update-snapshot ARCH=arm DYNLINK=1 --silent
+	$(VECHO) "Switching backend back to %s (DYNLINK=0)\n" arm
+	$(Q)$(MAKE) distclean config ARCH=arm DYNLINK=0 --silent
 
 update-snapshot: $(OUT)/$(STAGE0) tests/update-snapshots.sh
-	$(VECHO) "Updating snapshot for %s\n" $(ARCH)
-	tests/update-snapshots.sh $(ARCH)
+	$(VECHO) "Updating snapshot for %s (DYNLINK=%s)\n" $(ARCH) $(DYNLINK)
+	tests/update-snapshots.sh $(ARCH) $(DYNLINK)
 	$(VECHO) "  OK\n"
 
 $(OUT)/%.o: %.c
@@ -122,11 +154,12 @@ $(OUT)/norm-lf: tools/norm-lf.c
 	$(VECHO) "  CC+LD\t$@\n"
 	$(Q)$(CC) $(CFLAGS) -o $@ $^
 
-$(OUT)/libc.inc: $(OUT)/inliner $(OUT)/norm-lf $(LIBDIR)/c.c
+$(OUT)/libc.inc: $(OUT)/inliner $(OUT)/norm-lf $(LIBDIR)/$(BUILTIN_LIBC_SOURCE) $(LIBDIR)/$(BUILTIN_LIBC_HEADER)
 	$(VECHO) "  GEN\t$@\n"
-	$(Q)$(OUT)/norm-lf $(LIBDIR)/c.c $(OUT)/c.normalized.c
-	$(Q)$(OUT)/inliner $(OUT)/c.normalized.c $@
-	$(Q)$(RM) $(OUT)/c.normalized.c
+	$(Q)$(OUT)/norm-lf $(LIBDIR)/$(BUILTIN_LIBC_SOURCE) $(OUT)/c.normalized.c
+	$(Q)$(OUT)/norm-lf $(LIBDIR)/$(BUILTIN_LIBC_HEADER) $(OUT)/c.normalized.h
+	$(Q)$(OUT)/inliner $(OUT)/c.normalized.c $(OUT)/c.normalized.h $@
+	$(Q)$(RM) $(OUT)/c.normalized.c $(OUT)/c.normalized.h
 
 $(OUT)/inliner: tools/inliner.c
 	$(VECHO) "  CC+LD\t$@\n"
@@ -143,12 +176,12 @@ $(OUT)/$(STAGE0)-sanitizer: $(OUT)/libc.inc $(OBJS)
 $(OUT)/$(STAGE1): $(OUT)/$(STAGE0)
 	$(Q)$(STAGE1_CHECK_CMD)
 	$(VECHO) "  SHECC\t$@\n"
-	$(Q)$(OUT)/$(STAGE0) --dump-ir -o $@ $(SRCDIR)/main.c > $(OUT)/shecc-stage1.log
+	$(Q)$(OUT)/$(STAGE0) $(STAGE0_FLAGS) -o $@ $(SRCDIR)/main.c > $(OUT)/shecc-stage1.log
 	$(Q)chmod a+x $@
 
 $(OUT)/$(STAGE2): $(OUT)/$(STAGE1)
 	$(VECHO) "  SHECC\t$@\n"
-	$(Q)$(TARGET_EXEC) $(OUT)/$(STAGE1) -o $@ $(SRCDIR)/main.c
+	$(Q)$(TARGET_EXEC) $(OUT)/$(STAGE1) $(STAGE1_FLAGS) -o $@ $(SRCDIR)/main.c
 
 bootstrap: $(OUT)/$(STAGE2)
 	$(Q)chmod 775 $(OUT)/$(STAGE2)

README.md

@@ -74,10 +74,19 @@ To execute the snapshot test, install the packages below:
 $ sudo apt-get install graphviz jq
 ```
 
+Additionally, because `shecc` supports the dynamic linking mode for the Arm architecture,
+it needs to install the ARM GNU toolchain to obtain the ELF interpreter and other dependencies:
+```shell
+$ sudo apt-get install gcc-arm-linux-gnueabihf
+```
+Another approach is to manually download and install the toolchain from [ARM Developer website](https://developer.arm.com/downloads/-/arm-gnu-toolchain-downloads).
+
+Select "x86_64 Linux hosted cross toolchains" - "AArch32 GNU/Linux target with hard float (arm-none-linux-gnueabihf)" to download the toolchain.
+
 ## Build and Verify
 
 Configure which backend you want, `shecc` supports ARMv7-A and RV32IM backend:
-```
+```shell
 $ make config ARCH=arm
 # Target machine code switch to Arm
 
@@ -86,13 +95,29 @@ $ make config ARCH=riscv
 ```
 
 Run `make` and you should see this:
+```shell
+$ make
+  CC+LD	out/inliner
+  GEN	out/libc.inc
+  CC	out/src/main.o
+  LD	out/shecc
+  SHECC	out/shecc-stage1.elf
+  SHECC	out/shecc-stage2.elf
 ```
+
+Run `make DYNLINK=1` to use the dynamic linking mode and generate the dynamically linked compiler:
+```shell
+# If using the dynamic linking mode, you should add 'DYNLINK=1' for each 'make' command.
+$ make DYNLINK=1
   CC+LD	out/inliner
   GEN	out/libc.inc
   CC	out/src/main.o
   LD	out/shecc
   SHECC	out/shecc-stage1.elf
   SHECC	out/shecc-stage2.elf
+
+$ file out/shecc-stage2.elf
+out/shecc-stage2.elf: ELF 32-bit LSB executable, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, not stripped
 ```
 
 For development builds with memory safety checks:
@@ -103,22 +128,32 @@ $ make check-sanitizer
 
 File `out/shecc` is the first stage compiler. Its usage:
 ```shell
-$ shecc [-o output] [+m] [--no-libc] [--dump-ir] <infile.c>
+$ shecc [-o output] [+m] [--no-libc] [--dump-ir] [--dynlink] <infile.c>
 ```
 
 Compiler options:
 - `-o` : Specify output file name (default: `out.elf`)
 - `+m` : Use hardware multiplication/division instructions (default: disabled)
 - `--no-libc` : Exclude embedded C library (default: embedded)
 - `--dump-ir` : Dump intermediate representation (IR)
+- `--dynlink` : Use dynamic linking (default: disabled)
 
-Example:
+Example 1: static linking mode
 ```shell
 $ out/shecc -o fib tests/fib.c
 $ chmod +x fib
 $ qemu-arm fib
 ```
 
+Example 2: dynamic linking mode
+
+Notice that `/usr/arm-linux-gnueabihf` is the ELF interpreter prefix. Since the path may be different if you manually install the ARM GNU toolchain instead of using `apt-get`, you should set the prefix to the actual path.
+```shell
+$ out/shecc --dynlink -o fib tests/fib.c
+$ chmod +x fib
+$ qemu-arm -L /usr/arm-linux-gnueabihf fib
+```
+
 ### IR Regression Tests
 
 To ensure the consistency of frontend (lexer, parser) behavior when working on it, the snapshot test is introduced.
@@ -142,6 +177,7 @@ use `update-snapshot` / `check-snapshot` instead.
 
 `shecc` comes with a comprehensive test suite (200+ test cases). To run the tests:
 ```shell
+# Add 'DYNLINK=1' if using the dynamic linking mode.
 $ make check          # Run all tests (stage 0 and stage 2)
 $ make check-stage0   # Test stage 0 compiler only
 $ make check-stage2   # Test stage 2 compiler only