diff --git a/.github/workflows/test_repository.yml b/.github/workflows/test_repository.yml
index 9c2e5b9c..33a32b0f 100644
--- a/.github/workflows/test_repository.yml
+++ b/.github/workflows/test_repository.yml
@@ -25,8 +25,6 @@ jobs:
         - x86_64
         EESSI_OS:
         - linux
-        python:
-        - 3.12
         exclude:
           - EESSI_VERSION: '2023.06'
             EESSI_ARCH: riscv64
@@ -67,19 +65,5 @@ jobs:
               echo '>> ${EPREFIX}/startprefix <<< "which bash; qlist -IRv; exit"'
               ${EPREFIX}/startprefix <<< "which bash; qlist -IRv; exit"
 
-        - name: Set up Python
-          uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
-          with:
-            python-version: ${{matrix.python}}
-            architecture: x64
-
-        - name: Install ReFrame
-          run: |
-            pip install reframe-hpc requests
-
         - name: Run ReFrame test suite
-          run: |
-            export EESSI_ARCH=${{matrix.EESSI_ARCH}}
-            export EESSI_OS=${{matrix.EESSI_OS}}
-            export EESSI_VERSION=${{matrix.EESSI_VERSION}}
-            reframe -r -v -c ./test/compat_layer.py
+          run: ./test_compatibility_layer.sh --arch ${{matrix.EESSI_ARCH}} --os ${{matrix.EESSI_OS}} --version ${{matrix.EESSI_VERSION}} --verbose
diff --git a/test_compatibility_layer.sh b/test_compatibility_layer.sh
index 374ec7a2..80fd07b6 100755
--- a/test_compatibility_layer.sh
+++ b/test_compatibility_layer.sh
@@ -96,14 +96,9 @@ fi
 if ! command -v "reframe" &> /dev/null; then
   REFRAME_TMPDIR=$(mktemp -t -d eessi.XXXXXXXXXX)
   [[ ${VERBOSE} == '-vvv' ]] && echo "ReFrame command not found, trying to install it to a temporary directory ${REFRAME_TMPDIR}..."
-  if command -v "pip3" &> /dev/null; then
-    pip3 install --ignore-installed -t ${REFRAME_TMPDIR} reframe-hpc &> /dev/null
-    export PYTHONPATH=${REFRAME_TMPDIR}
-    export PATH="${REFRAME_TMPDIR}/bin/:${PATH}"
-  else
-    echo "Neither Reframe nor pip3 is available, please install ReFrame manually and add it to your \$PATH."
-    exit 1
-  fi
+  ${COMPAT_LAYER_PREFIX}/usr/bin/python3 -m venv ${REFRAME_TMPDIR}
+  source ${REFRAME_TMPDIR}/bin/activate
+  pip3 install reframe-hpc &> /dev/null
 fi
 
 [[ ${VERBOSE} == '-vvv' ]] && echo "Trying to run 'reframe --version' as sanity check..."