test/system: Connect system tests to Meson

A bit of plumbing in the build files has been done to accommodate the
change as much as possible. If we're to use meson.build in the
'test/system' subdir we better also install the data from there.

This also adjust the CI playbooks to launch the tests via Meson by
specifying the specific test suite ('system').

This change also means the test suite needs to be run from the
'test/system' subdirectory for the bats helper libraries to be
discovered correctly.

#1062

Author: martymichal

meson.build

@@ -19,13 +19,8 @@ subid_dep = cc.find_library('subid', has_headers: ['shadow/subid.h'])
 go = find_program('go')
 go_md2man = find_program('go-md2man')
 
-bats = find_program('bats', required: false)
 codespell = find_program('codespell', required: false)
-htpasswd = find_program('htpasswd', required: false)
-openssl = find_program('openssl', required: false)
-podman = find_program('podman', required: false)
 shellcheck = find_program('shellcheck', required: false)
-skopeo = find_program('skopeo', required: false)
 
 bashcompletionsdir = get_option('bash_completions_dir')
 if bashcompletionsdir == ''
@@ -84,29 +79,6 @@ if shellcheck.found()
   test('shellcheck toolbox (deprecated)', shellcheck, args: [toolbox_sh])
 endif
 
-install_subdir(
-  'test',
-  install_dir: get_option('datadir') / meson.project_name(),
-  exclude_files: [
-    'system/libs/bats-assert/.git',
-    'system/libs/bats-assert/.gitignore',
-    'system/libs/bats-assert/.travis.yml',
-    'system/libs/bats-assert/package.json',
-    'system/libs/bats-support/.git',
-    'system/libs/bats-support/.gitignore',
-    'system/libs/bats-support/.travis.yml',
-    'system/libs/bats-support/package.json'
-    ],
-  exclude_directories: [
-    'system/libs/bats-assert/.git',
-    'system/libs/bats-assert/script',
-    'system/libs/bats-assert/test',
-    'system/libs/bats-support/.git',
-    'system/libs/bats-support/script',
-    'system/libs/bats-support/test'
-  ]
-)
-
 subdir('data')
 subdir('doc')
 subdir('profile.d')

playbooks/system-test.yaml

@@ -20,9 +20,9 @@
     - include_tasks: build.yaml
 
     - name: Run system tests
-      command: bats --timing ./test/system
+      command: meson test --suite system
       environment:
         PODMAN: '/usr/bin/podman'
         TOOLBOX: '/usr/local/bin/toolbox'
       args:
-        chdir: '{{ zuul.project.src_dir }}'
+        chdir: '{{ zuul.project.src_dir }}/builddir'

test/system/README.md

@@ -29,8 +29,19 @@ by running `git submodule init` and `git submodule update`.
 
 First, make sure you have all the dependencies installed.
 
-- Enter the toolbox root folder
-- Invoke command `bats ./test/system/` and the test suite should fire up
+### Meson (recommended)
+
+> This method is optimal when bulding from source
+
+- Set up the project per instructions at https://containertoolbx.org/install
+- Invoke command `meson test --suite system` and the test suite should fire up
+
+### Manually (legacy)
+
+> This method is optimal if you want to test an existing build of Toolbx
+
+- Enter the `test/system` directory in the project root directory
+- Invoke command `bats ./` and the test suite should fire up
 
 Mocking of images is done automatically to prevent potential networking issues
 and to speed up the cases.
@@ -41,7 +52,7 @@ By default the test suite uses the system versions of `podman`, `skopeo` and
 If you have a `podman`, `skopeo` or `toolbox` installed in a nonstandard
 location then you can use the `PODMAN`, `SKOPEO` and `TOOLBOX` environmental
 variables to set the path to the binaries. So the command to invoke the test
-suite could look something like this: `PODMAN=/usr/libexec/podman TOOLBOX=./toolbox bats ./test/system/`.
+suite could look something like this: `PODMAN=/usr/libexec/podman TOOLBOX=./toolbox bats ./`.
 
 When running the tests, make sure the `test suite: [job]` jobs are successful.
 These jobs set up the whole environment and are a strict requirement for other

test/system/libs/meson.build

@@ -0,0 +1,42 @@
+test_system_libs_dir = test_system_dir / 'libs'
+
+test_system_libs_files = files(
+  'helpers.bash'
+)
+
+install_data(
+  install_dir: test_system_libs_dir,
+  sources: test_system_libs_files
+)
+
+install_subdir(
+  'bats-support',
+  install_dir: test_system_libs_dir,
+  exclude_files: [
+    '.git',
+    '.gitignore',
+    '.travis.yml',
+    'package.json'
+  ],
+  exclude_directories: [
+    '.git',
+    'script',
+    'test'
+  ]
+)
+
+install_subdir(
+  'bats-assert',
+  install_dir: test_system_libs_dir,
+  exclude_files: [
+    '.git',
+    '.gitignore',
+    '.travis.yml',
+    'package.json'
+  ],
+  exclude_directories: [
+    '.git',
+    'script',
+    'test'
+  ]
+)

test/system/meson.build

@@ -1,3 +1,12 @@
+test_system_dir = get_option('datadir') / meson.project_name() / 'test'
+
+awk = find_program('awk', required: false)
+bats = find_program('bats', required: false)
+htpasswd = find_program('htpasswd', required: false)
+openssl = find_program('openssl', required: false)
+podman = find_program('podman', required: false)
+skopeo = find_program('skopeo', required: false)
+
 test_system = files(
   '001-version.bats',
   '002-help.bats',
@@ -15,9 +24,41 @@ test_system = files(
   '210-ulimit.bats',
   '211-dbus.bats',
   'setup_suite.bash',
-  'libs/helpers.bash',
 )
 
 if shellcheck.found()
   test('shellcheck test/system', shellcheck, args: [test_system])
 endif
+
+if fs.exists('/run/.containerenv')
+  warning('System tests can not be run in a container.')
+elif not awk.found() or not bats.found() or not htpasswd.found() or not openssl.found() or not podman.found() or not skopeo.found()
+  warning('System tests require: awk, bats, htpasswd, openssl, podman and skopeo being installed.')
+else
+  system_tests_env = environment()
+  system_tests_env.set('TOOLBOX', toolbox_go.full_path())
+
+  test(
+    'system tests',
+    bats,
+    args: [
+      '--formatter', 'tap',
+      '--timing',
+      './'],
+    depends: toolbox_go,
+    env: system_tests_env,
+    is_parallel: false,
+    protocol: 'tap',
+    suite: 'system',
+    timeout: 3600,
+    verbose: true,
+    workdir: meson.current_source_dir(),
+  )
+endif
+
+install_data(
+  install_dir: test_system_dir,
+  sources: [test_system, 'README.md'],
+)
+
+subdir('libs')

test/system/setup_suite.bash

@@ -17,7 +17,7 @@
 
 missing_dependencies=false
 
-if [ -f test/system/libs/bats-assert/load.bash ] && [ -f test/system/libs/bats-support/load.bash ]; then
+if [ -f libs/bats-assert/load.bash ] && [ -f libs/bats-support/load.bash ]; then
   load 'libs/helpers'
 else
   missing_dependencies=true