From c087db18b667938ef68d9ecf41f003f00bc560c2 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 10:21:31 +0100 Subject: [PATCH 01/13] Imported ollama-serve role from hackathon repo. Co-authored-by: sverhoeven --- roles/ollama-serve/README.md | 44 +++++++++++++++++++ roles/ollama-serve/defaults/main.yml | 9 ++++ roles/ollama-serve/handlers/main.yml | 3 ++ roles/ollama-serve/meta/main.yml | 6 +++ roles/ollama-serve/tasks/main.yml | 42 ++++++++++++++++++ .../templates/ollama-serve.service.j2 | 16 +++++++ roles/ollama-serve/tests/inventory | 3 ++ roles/ollama-serve/tests/test.yml | 6 +++ roles/ollama-serve/vars/main.yml | 4 ++ 9 files changed, 133 insertions(+) create mode 100644 roles/ollama-serve/README.md create mode 100644 roles/ollama-serve/defaults/main.yml create mode 100644 roles/ollama-serve/handlers/main.yml create mode 100644 roles/ollama-serve/meta/main.yml create mode 100644 roles/ollama-serve/tasks/main.yml create mode 100644 roles/ollama-serve/templates/ollama-serve.service.j2 create mode 100644 roles/ollama-serve/tests/inventory create mode 100644 roles/ollama-serve/tests/test.yml create mode 100644 roles/ollama-serve/vars/main.yml diff --git a/roles/ollama-serve/README.md b/roles/ollama-serve/README.md new file mode 100644 index 0000000..1ca5045 --- /dev/null +++ b/roles/ollama-serve/README.md @@ -0,0 +1,44 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +Run with + +```shell +ansible-playbook -i localhost, -b -c local ./playbook.yml +``` + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/roles/ollama-serve/defaults/main.yml b/roles/ollama-serve/defaults/main.yml new file mode 100644 index 0000000..03b4c80 --- /dev/null +++ b/roles/ollama-serve/defaults/main.yml @@ -0,0 +1,9 @@ +#SPDX-License-Identifier: MIT-0 +--- +# defaults file for ollama-serve +ollama_serve_model_dir: /opt/ollama/models +ollama_serve_env: + OLLAMA_FLASH_ATTENTION: "1" + OLLAMA_MODELS: "{{ ollama_serve_model_dir }}" +ollama_serve_model: "" +ollama_serve_version: 12.0.10 diff --git a/roles/ollama-serve/handlers/main.yml b/roles/ollama-serve/handlers/main.yml new file mode 100644 index 0000000..eaa20f5 --- /dev/null +++ b/roles/ollama-serve/handlers/main.yml @@ -0,0 +1,3 @@ +#SPDX-License-Identifier: MIT-0 +--- +# handlers file for ollama-serve diff --git a/roles/ollama-serve/meta/main.yml b/roles/ollama-serve/meta/main.yml new file mode 100644 index 0000000..2db5d03 --- /dev/null +++ b/roles/ollama-serve/meta/main.yml @@ -0,0 +1,6 @@ +#SPDX-License-Identifier: MIT-0 +galaxy_info: + author: Stefan Verhoeven & Dawa Ometto + description: serve ollama + license: MIT + min_ansible_version: 2.1 diff --git a/roles/ollama-serve/tasks/main.yml b/roles/ollama-serve/tasks/main.yml new file mode 100644 index 0000000..1bb8680 --- /dev/null +++ b/roles/ollama-serve/tasks/main.yml @@ -0,0 +1,42 @@ +#SPDX-License-Identifier: MIT-0 +--- +# tasks file for ollama-serve +- name: ollama user + ansible.builtin.user: + name: ollama + system: true + create_home: true + +- name: Create model dir + ansible.builtin.file: + path: "{{ ollama_serve_model_dir }}" + state: directory + owner: ollama + group: ollama + mode: "0755" + +- name: Fetch and unarchive ollama + ansible.builtin.unarchive: + src: "{{ ollama_serve_tarball }}" + remote_src: true + dest: /opt/ollama + creates: /opt/ollama/bin/ollama + +- name: Place systemd config file + ansible.builtin.template: + dest: /etc/systemd/system/ollama-serve.service + src: templates/ollama-serve.service.j2 + mode: "0644" + +- name: Run ollama service + ansible.builtin.systemd: + name: ollama-serve + state: started + daemon_reload: true + enabled: true + +- name: Pull requested model + ansible.builtin.command: + cmd: /opt/ollama/bin/ollama pull "{{ ollama_serve_model }}" + when: ollama_serve_model | length > 0 + delay: 10 diff --git a/roles/ollama-serve/templates/ollama-serve.service.j2 b/roles/ollama-serve/templates/ollama-serve.service.j2 new file mode 100644 index 0000000..9f03d0c --- /dev/null +++ b/roles/ollama-serve/templates/ollama-serve.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Service for Ollama +After=network.target + +[Service] +User=ollama +Group=ollama +StandardOutput=append:/var/log/ollama.log +StandardError=append:/var/log/ollama_err.log +{% for var_name, value in ollama_serve_env.items() %} +Environment="{{ var_name }}={{ value }}" +{% endfor %} +ExecStart=/opt/ollama/bin/ollama serve + +[Install] +WantedBy=multi-user.target diff --git a/roles/ollama-serve/tests/inventory b/roles/ollama-serve/tests/inventory new file mode 100644 index 0000000..03ca42f --- /dev/null +++ b/roles/ollama-serve/tests/inventory @@ -0,0 +1,3 @@ +#SPDX-License-Identifier: MIT-0 +localhost + diff --git a/roles/ollama-serve/tests/test.yml b/roles/ollama-serve/tests/test.yml new file mode 100644 index 0000000..5144190 --- /dev/null +++ b/roles/ollama-serve/tests/test.yml @@ -0,0 +1,6 @@ +#SPDX-License-Identifier: MIT-0 +--- +- hosts: localhost + remote_user: root + roles: + - ollama-serve diff --git a/roles/ollama-serve/vars/main.yml b/roles/ollama-serve/vars/main.yml new file mode 100644 index 0000000..e6822df --- /dev/null +++ b/roles/ollama-serve/vars/main.yml @@ -0,0 +1,4 @@ +#SPDX-License-Identifier: MIT-0 +--- +# vars file for ollama-serve +ollama_serve_tarball: https://github.com/ollama/ollama/releases/download/v{{ ollama_serve_version }}/ollama-linux-amd64.tgz \ No newline at end of file From 771f8c38e5b11e248965ad323f6101e8c68363c9 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 10:22:59 +0100 Subject: [PATCH 02/13] Import playbook from hackathon repo. Co-authored-by: sverhoeven --- playbook.yml | 65 ++++++++-------------------------------------------- 1 file changed, 10 insertions(+), 55 deletions(-) diff --git a/playbook.yml b/playbook.yml index dc2057b..9531a32 100644 --- a/playbook.yml +++ b/playbook.yml @@ -1,57 +1,12 @@ --- -- name: Example component - hosts: localhost # On ResearchCloud, the target host is always simply 'localhost'. - vars: - # You can define variables here. - # One use for this is to ensure that variables expected to be passed from the SRC portal have sane defaults. - # For example: - _src_component_foo: "{{ src_component_foo | default('fallback value') }}" - _src_component_bar: "Xone,Xtwo,Xthree" - _src_component_boolean: "{{ src_component_boolean | default(true, true) | bool }}" - # But of course you can also define ordinary variables: - testfile: /tmp/test - test_pip_packages: [] - gather_facts: true - +- name: Install and configure ollama serve + hosts: + - all + - localhost + gather_facts: false + vars: {} roles: - - role: uusrc.general.fact_regular_users - - tasks: - # Some example tasks below to introduce Ansible - - - name: Loop over all non-system users and display their names - ansible.builtin.debug: - msg: The user {{ item.user }} exists on the system. - with_items: "{{ fact_regular_users }}" - when: ansible_os_family == 'Debian' # we can use Ansible OS facts because we have set gather_facts to true - - - name: This is a block of tasks that belong together - when: _src_component_boolean # this condition is applied to all tasks in the block - tags: molecule-idempotence-notest # same for tags - block: - - - name: Copy some content to a file - ansible.builtin.copy: - dest: "{{ testfile }}" - mode: "0700" - owner: root - group: root - content: "{{ _src_component_bar | split | map('regex_replace', 'X', '') | join }}" # You can create pipes using filters - # src: foo.txt # instead of 'content', you can also pass the 'src' argument to copy an entire file - - - name: Cat the contents of this file - ansible.builtin.command: - cmd: cat {{ testfile }} - register: cat_testfile # store the results of this module in a new variable - - - name: Debug the results of our cat command - ansible.builtin.debug: - var: cat_testfile.stdout - - - name: Unlike the command module, shell module can use shell features like redirection - ansible.builtin.shell: - cmd: cat {{ testfile }} > /tmp/test2 - - - name: Install a number of pip packages - ansible.builtin.pip: # the pip module has many useful arguments, for instance related to venvs - name: "{{ test_pip_packages }}" + - role: ollama-serve + vars: + ollama_serve_model: "{{ model }}" + ollama_serve_version: "{{ ollama_version | default('0.12.10', true) }}" From 1987a24577de69cecd58d3bffd905d63787be2ae Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 10:27:49 +0100 Subject: [PATCH 03/13] Configure model for molecule tests --- molecule/default/molecule.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 5b04fb0..f4dca86 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -6,4 +6,4 @@ provisioner: - name: test_component path: playbook.yml parameters: - # Add parameters that you expect to be set in the ResearchCloud portal + model: smollm:135m From 7b449b305f80f947229977603dad818e51254a9f Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 13:15:42 +0100 Subject: [PATCH 04/13] Remove ext molecule config --- molecule/ext/molecule-src/.ansible-lint | 4 - .../ext/molecule-src/.github/dependabot.yml | 11 -- .../.github/workflows/ansible-lint.yml | 17 -- .../.github/workflows/molecule.yml | 47 ----- molecule/ext/molecule-src/README.md | 185 ------------------ molecule/ext/molecule-src/_run_component.yml | 49 ----- molecule/ext/molecule-src/converge.yml | 20 -- molecule/ext/molecule-src/default.env.yml | 8 - molecule/ext/molecule-src/molecule.yml | 30 --- molecule/ext/molecule-src/prepare.yml | 41 ---- molecule/ext/molecule-src/requirements.txt | 6 - molecule/ext/molecule-src/requirements.yml | 6 - 12 files changed, 424 deletions(-) delete mode 100644 molecule/ext/molecule-src/.ansible-lint delete mode 100644 molecule/ext/molecule-src/.github/dependabot.yml delete mode 100644 molecule/ext/molecule-src/.github/workflows/ansible-lint.yml delete mode 100644 molecule/ext/molecule-src/.github/workflows/molecule.yml delete mode 100644 molecule/ext/molecule-src/README.md delete mode 100644 molecule/ext/molecule-src/_run_component.yml delete mode 100644 molecule/ext/molecule-src/converge.yml delete mode 100644 molecule/ext/molecule-src/default.env.yml delete mode 100644 molecule/ext/molecule-src/molecule.yml delete mode 100644 molecule/ext/molecule-src/prepare.yml delete mode 100644 molecule/ext/molecule-src/requirements.txt delete mode 100644 molecule/ext/molecule-src/requirements.yml diff --git a/molecule/ext/molecule-src/.ansible-lint b/molecule/ext/molecule-src/.ansible-lint deleted file mode 100644 index c0c2fa6..0000000 --- a/molecule/ext/molecule-src/.ansible-lint +++ /dev/null @@ -1,4 +0,0 @@ ---- -profile: basic -exclude_paths: - - .github diff --git a/molecule/ext/molecule-src/.github/dependabot.yml b/molecule/ext/molecule-src/.github/dependabot.yml deleted file mode 100644 index 9d866e3..0000000 --- a/molecule/ext/molecule-src/.github/dependabot.yml +++ /dev/null @@ -1,11 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file - -version: 2 -updates: - - package-ecosystem: "pip" # See documentation for possible values - directory: "/" # Location of package manifests - schedule: - interval: "weekly" diff --git a/molecule/ext/molecule-src/.github/workflows/ansible-lint.yml b/molecule/ext/molecule-src/.github/workflows/ansible-lint.yml deleted file mode 100644 index 30d0f3a..0000000 --- a/molecule/ext/molecule-src/.github/workflows/ansible-lint.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: Ansible Lint -on: - push: - branches: - - main - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - name: Ansible Lint - steps: - # Important: This sets up your GITHUB_WORKSPACE environment variable - - uses: actions/checkout@v3 - - name: Run ansible-lint - uses: ansible/ansible-lint@v6.20.3 diff --git a/molecule/ext/molecule-src/.github/workflows/molecule.yml b/molecule/ext/molecule-src/.github/workflows/molecule.yml deleted file mode 100644 index 47cbdc1..0000000 --- a/molecule/ext/molecule-src/.github/workflows/molecule.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -name: Molecule Run -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - molecule: - runs-on: ubuntu-latest - env: - DOCKER_USER: ${{ github.actor }} - DOCKER_PW: ${{ secrets.GITHUB_TOKEN }} - DOCKER_REGISTRY: ghcr.io - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_STDOUT_CALLBACK: yaml - MOLECULE_CONFIG: molecule/ext/molecule-src/molecule.yml - REQUIREMENTS_FILE: requirements.txt - PLAYBOOK_DIR: ../../../ # relative to molecule/ext/molecule-src/molecule.yml - CRUN_VER: 1.11.2 - steps: - - name: Workaround crun issue on ubuntu # https://github.com/UtrechtUniversity/SRC-catalog-items/issues/3 - run: | - mkdir -p "${HOME}/.local/bin" - curl -L "https://github.com/containers/crun/releases/download/${CRUN_VER}/crun-${CRUN_VER}-linux-amd64" -o "${HOME}/.local/bin/crun" - chmod +x "${HOME}/.local/bin/crun" - crun --version - mkdir -p "${HOME}/.config/containers" - cat << EOF > "${HOME}/.config/containers/containers.conf" - [engine.runtimes] - crun = [ - "${HOME}/.local/bin/crun", - "/usr/bin/crun" - ] - EOF - - name: Checkout - uses: actions/checkout@v4 - - run: git clone https://github.com/UtrechtUniversity/SRC-molecule-test.git molecule - - run: mkdir molecule/ext && cd molecule/ext && ln -s ../../ molecule-src - - run: pip install -r ${{ env.REQUIREMENTS_FILE }} - - run: ansible-galaxy install -r requirements.yml - - name: Molecule tests - run: | - molecule -vv -c ${{ env.MOLECULE_CONFIG }} test diff --git a/molecule/ext/molecule-src/README.md b/molecule/ext/molecule-src/README.md deleted file mode 100644 index 0705a72..0000000 --- a/molecule/ext/molecule-src/README.md +++ /dev/null @@ -1,185 +0,0 @@ -# SURF Research Cloud Molecule Testing Configuration - -This repository contains default configuration for running [Molecule](https://ansible.readthedocs.io/projects/molecule/) tests for SURF Research Cloud (SRC) Components and Catalog Items. The repository is meant to be included into other repositories as a subtree. - -## Getting Started - -Follow the installation and setup instructions below, then run: - -`molecule -c molecule/ext/molecule-src/molecule.yml test -s ` - -...where `` is the name of one of the subdirectories of the `molecule` directory, e.g. `playbook-security_updates`. - -### Requirements - -1. Docker and/or Podman (to spin up test containers) -1. Python and pip -1. Ansible -1. Molecule -1. Access to the [test container images](https://github.com/UtrechtUniversity/SRC-test-workspace) - -Before you start, run: - -* `pip install -r molecule/ext/molecule-src/requirements.txt` to install Molecule itself and other python dependencies. -* `ansible-galaxy install -r molecule/ext/molecule-src/requirements.yml` to install Ansible collection dependencies. - -The default `molecule.yml` is configured to use the images from [this package](https://github.com/UtrechtUniversity/SRC-test-workspace/), but you can override this to use other images. - -### Getting the container images - -The default `molecule.yml` is configured to use the images from [this package](https://github.com/UtrechtUniversity/SRC-test-workspace/), but you can also override this to use other images. There are two ways to provide Molecule with access to the right images: - -1. Manually pull the images from the container registry. If the images are already available locally, there is no need to authenticate with the container registry. - * `docker login ghcr.io` (or `podman login`) - * enter your github username and a valid token - * `docker pull ` (or `podman pull`) - * See [the package](https://github.com/UtrechtUniversity/SRC-test-workspace/) for the image names -2. If you set the right variables when running `molecule`, it will pull the images automatically. - * `export DOCKER_REGISTRY=ghcr.io` - * `export DOCKER_USER=githubusername` - * `export DOCKER_PW=githubtoken` - -### Install SRC-specific configuration - -NB: this is not necessary for running tests on a repository already containing the configuration files from this repository, only to add tests to a repository that does not contain them yet. - -To add Molecule tests to your SRC component or catalog item repository, follow these steps: - -* create a `molecule` directory in your repository's root: `mkdir molecule` -* include the contents of this repository as a subtree, under `molecule/ext/molecule-src` - * `git remote add molecule-src https://github.com/UtrechtUniversity/SRC-molecule.git` - * `git subtree add --prefix molecule/ext/molecule-src molecule-src main --squash` -* copy the default `.env.yml` file to your repository root: `cp molecule/ext/molecule-src/default.env.yml .env.yml` - * optionally edit the contents of `.env.yml`, if your playbooks are not in the default location (the repository root) -* run `pip install -r molecule/ext/molecule-src/requirements.txt` - -That's it for setup! You're now ready to start [adding your own scenarios](#adding-scenarios). - -### Scenarios - -Molecule runs tests for each *scenario* defined under the `molecule` directory. You can think of each scenario as a self-contained test suite. A scenario is just a subdirectory of the `molecule` directory that contains a number of configuration files and playbooks. For our purposes, these are minimally: - -* `molecule.yml` - sets general configuration variables for the test suite (e.g. what platform to run on). Overrides the defaults in `molecule/ext/molecule-src/molecule.yml`. -* `prepare.yml` - a playbook used to make preparations on the container before running the Ansible code that we want to test on it. -* `converge.yml` - a playbook that specifies how to deploy the Ansible role or playbook that we want to test on the container. -* `verify.yml` - a playbook that contains additional test assertions after the `converge` step is run. - -We want to test our SURF Research Cloud (SRC) components in a specific way, namely by: - -1. copying them onto the workspace -2. then executing them with Ansible *on the workspace* (as opposed to using Ansible on the host). - -We want this procedure because that is [how components are actually deployed onto SRC workspaces](https://github.com/UtrechtUniversity/SRC-test-workspace#how-it-works). - -To help us achieve this, the `default` scenario contains a `prepare.yml` that takes care of 1., and a `converge.yml` that takes care of 2. Each individual scenario will automatically use these default playbooks, as long as `molecule` is run with the ` -c molecule/default/molecule.yml`. - -### Adding scenarios - -To create a Molecule scenario to test, just create a subdirectory of the `molecule` directory, e.g.: - -`mkdir molecule/my-component` - -Now add a `molecule.yml` file to the `my-component` subdirectory, with contents like the following: - -```yaml -provisioner: - name: ansible - env: - components: - - name: my-component - path: my-component.yaml - parameters: # Define all parameters needed by the component here - my_component_param1: 'Foo' - - name: my-git-component # You can also provide components that should be cloned onto the workspace using git - git: https://github.com/foo/bar.git - version: my_branch - path: playbook.ymk -``` - -The above config file does not provide a `platforms` key, so tests for this scenario will use the default platforms (containers) specified in `molecule/ext/molecule-src/molecule.yml` (Ubuntu Focal and Ubuntu Jammy). You can override this by adding your own platform definition, e.g.: - -```yaml -platforms: - - name: workspace-src-ubuntu_focal-desktop - image: ghcr.io/utrechtuniversity/src-test-workspace:ubuntu_focal-desktop # You can also change this to another image - pre_build_image: true - # The following lines are an example of how to specify a container registry to pull the image from, if it is not already available locally. - registry: - url: $DOCKER_REGISTRY # Use environment variables to define the URL and credentials - credentials: - username: $DOCKER_USER -``` - -Note that you can also define multiple platforms. - -### Specifying a Driver - -The molecule tests can use either Docker or Podman (default). You can override the default driver in your scenario's `molecule.yml`. Or even easier: set an environment variable like `DRIVER=docker`. - -### Adding additional preparation tasks - -Sometimes it may be helpful to perform certain tasks before the `converge` step. The default `prepare.yml` playbook will check if the user has set the `extra_prepare_tasks` key in the scenario's `molecule.yml` `env` section. Set this key to a relative path to a playbook (relative to the location of the default molecule configuration), and the tasks in that playbook will be executed at the end of the `prepare` step. - -### Adding additional assertions - -You can a `verify.yml` to your scenario to perform additional assertions. See the [Molecule docs](https://ansible.readthedocs.io/projects/molecule/configuration/#verifier). - -The main thing we are testing using Molecule is whether our playbooks are deployable on SRC workspaces. Since Ansible itself checks whether each task is actually successful (i.e. *changes* the target in the required way), we do not need to assert that everything we do in a playbook actually happens: when a playbook contains the instruction to e.g. install a certain package, and the playbook deploys on our test containers without error, we *know* that the container is in the required state. - -However, in some cases, it may be desirable to perform additional assertions after deployment is complete. Molecule allows you to do this by adding a `verify.yml` playbook to your scenario. You can use this perform assertions either using Ansible's own [assert module](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/assert_module.html), or using [testinfra](https://ansible.readthedocs.io/projects/molecule/configuration/#molecule.verifier.testinfra.Testinfra). - -What can/should be tested in a `verify.yml`? For instance: - -* behaviour that is conditional on certain parameters, and therefore not already triggered by simply executing the playbook on the test container (in `converge.yml`) -* lower level behaviour: e.g., not whether service `apache2` is running (we know that it is, since we require this in our playbook), but whether connecting to port 80 actually yields the desired webservice (e.g. a Jupyter notebook). - -## Debugging - -### Inspecting the workspace - -For debugging and development purposes, it can be useful to inspect what's going on on a container after executing a playbook or role on it. There are two ways of doing this: - -1. Run `molecule` with the `create` command instead of the `test` command: `molecule -c molecule/default/molecule.yml create -s ` -2. Run `molecule` with the `--destroy=never` flag: `molecule -c molecule/default/molecule.yml test -s --destroy=never` - -Both of these methods ensure that the container is not destroyed after molecule is done. This means you can then login to the container and see what's going on. For instance, after molecule is done, you can: - -1. `docker container list` - * see that the container `workspace-src-ubuntu_focal` is still running -2. `docker exec -it workspace-src-ubuntu_focal bash` - * login to the container as root - -### The component cannot be found on the workspace - -Problem: the `converge` step fails with the following error message in Ansible's `stderr` result: - -``` -'ERROR! the playbook: /rsc/plugins/componentname/playbookname.yml could not be found' -``` - -This can occassionally occur when Molecule thinks it has already run the `prepare` step on your container, but actually hasn't. (This happens, for instance, when the `converge` step uses as an old container that is still running because you used `--destroy=never` in a previous run.) - -Try resetting your molecule cache: - -`molecule -c molecule/ext/molecule-src/molecule.yml reset -s playbook-aptly` - -This will stop the container and flush the cache. Sometimes manually removing the cache may also be useful during troubleshooting: - -`rm -rf ~/.cache/molecule` - -### Container unreachable error - -Molecule runs sometimes fail with an error message like the following: - -``` -task path: /home/user/researchcloud-items/molecule/ext/molecule-src/prepare.yml:2 -fatal: [workspace-src-ubuntu_jammy]: UNREACHABLE! => changed=false - msg: 'Failed to create temporary directory. In some cases, you may have been able to authenticate and did not have permissions on the target directory. Consider changing the remote tmp path in ansible.cfg to a path rooted in "/tmp", for more error information use -vvv. Failed command was: ( umask 77 && mkdir -p "` echo ~/.ansible/tmp `"&& mkdir "` echo ~/.ansible/tmp/ansible-tmp-1710760454.6766412-70112-138145577949983 `" && echo ansible-tmp-1710760454.6766412-70112-138145577949983="` echo ~/.ansible/tmp/ansible-tmp-1710760454.6766412-70112-138145577949983 `" ), exited with result 1' - unreachable: true -``` - -This may occur when trying to initalize a container with the command parameter set to `/sbin/init`, i.e. when trying to run a container controlled by `systemd`. Some component tests may need this, because they are testing functionality of `systemd` services. However, in some circumstances, starting a container with `/sbin/init` fails: - -* Docker support for `systemd` is not excellent, try using Podman instead. -* The error also occurs when using container emulation (e.g. using an `amd64` image on an `arm64` host). Get a native image instead. -* Adding the `privileged: true` option to the platform usually takes care of the problem (even when using Docker!), but this is only recommended as a workaround [for security reasons](https://www.trendmicro.com/en_vn/research/19/l/why-running-a-privileged-container-in-docker-is-a-bad-idea.html). diff --git a/molecule/ext/molecule-src/_run_component.yml b/molecule/ext/molecule-src/_run_component.yml deleted file mode 100644 index 70db46c..0000000 --- a/molecule/ext/molecule-src/_run_component.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -- name: Install component Ansible Galaxy dependencies - tags: molecule-idempotence-notest - block: - - name: Check for plugin requirements file - stat: - path: /rsc/plugins/{{ remote_plugin.script_folder }}/requirements.yml - register: register_requirements_1 - - - name: Check for plugin requirements file 2 - stat: - path: /rsc/plugins/{{ remote_plugin.script_folder }}/{{ remote_plugin.path | dirname }}/requirements.yml - register: register_requirements_2 - - - name: Set path to file 1 if it exists - set_fact: - requirements_path: "{{ register_requirements_1.stat.path }}" - when: register_requirements_1.stat.exists - - - name: Set path to file 2 if file 1 does not exist and file 2 exists - set_fact: - requirements_path: "{{ register_requirements_2.stat.path }}" - when: not register_requirements_1.stat.exists and register_requirements_2.stat.exists - - - name: Install role dependencies for Ansible plugin - when: requirements_path is defined - command: | - ansible-galaxy role install -r {{ requirements_path }} -p /rsc/plugins/{{ remote_plugin.script_folder | basename }} - args: - chdir: /rsc/plugins/{{ remote_plugin.script_folder | basename }} - executable: /bin/bash - - - name: Install collection dependencies for Ansible plugin - when: requirements_path is defined - command: | - ansible-galaxy collection install -r {{ requirements_path }} -p /rsc/plugins/{{ remote_plugin.script_folder | basename }}/collections - args: - chdir: /rsc/plugins/{{ remote_plugin.script_folder | basename }} - executable: /bin/bash - -- name: Test the component by executing it using ansible on the workspace - ansible.builtin.command: > - ansible-playbook -c local -v -b {{ remote_plugin.arguments }} -e='{{ remote_plugin.parameters }}' /rsc/plugins/{{ remote_plugin.script_folder - }}/{{remote_plugin.path }} - register: ansible_on_workspace - changed_when: > - ansible_on_workspace.stdout_lines is not defined or - 'changed=0' not in - ansible_on_workspace.stdout_lines[ lookup('ansible.utils.index_of', ansible_on_workspace.stdout_lines, 'regex', '\s*PLAY RECAP\s*')+1 ] diff --git a/molecule/ext/molecule-src/converge.yml b/molecule/ext/molecule-src/converge.yml deleted file mode 100644 index a45d60b..0000000 --- a/molecule/ext/molecule-src/converge.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: Converge - hosts: all - gather_facts: false - tasks: - - name: Debug -- list all components to be executed - ansible.builtin.debug: - msg: "{{ item.name }}" - with_items: "{{ lookup('env', 'components') }}" - - - name: Run components - ansible.builtin.include_tasks: _run_component.yml - vars: - remote_plugin: - script_type: Ansible PlayBook - arguments: -i 127.0.0.1, --skip-tags {{ ansible_skip_tags | join(',') }} - parameters: "{{ item.parameters | default({}) | to_json }}" - script_folder: "{{ item.name }}" - path: "{{ item.path }}" - with_items: "{{ lookup('env', 'components') }}" diff --git a/molecule/ext/molecule-src/default.env.yml b/molecule/ext/molecule-src/default.env.yml deleted file mode 100644 index 6f07e00..0000000 --- a/molecule/ext/molecule-src/default.env.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# Default Molecule .env.yml file -# NOTE: to make molecule use this file, place it in the root of your project (or from whichever directory you are calling 'molecule'), and rename it to .env.yml -PLAYBOOK_DIR: ../../../ # Relative to the default molecule.yml (molecule/ext/molecule-src/molecule.yml) -ANSIBLE_VERBOSITY: "2" -ANSIBLE_ROLES_PATH: ../../roles # Relative to your scenario (e.g. molecule/role-foo/..) -REQUIREMENTS_FILE: requirements.txt -ANSIBLE_STDOUT_CALLBACK: yaml diff --git a/molecule/ext/molecule-src/molecule.yml b/molecule/ext/molecule-src/molecule.yml deleted file mode 100644 index 50a470d..0000000 --- a/molecule/ext/molecule-src/molecule.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -driver: - name: ${DRIVER-podman} - image_settings: &image_settings - pre_build_image: true - registry: - url: $DOCKER_REGISTRY - credentials: - username: $DOCKER_USER - password: $DOCKER_PW -platforms: # Test on ubuntu focal and jammy by default - - name: workspace-src-ubuntu_focal - image: ghcr.io/utrechtuniversity/src-test-workspace:ubuntu_focal - <<: *image_settings - - name: workspace-src-ubuntu_jammy - image: ghcr.io/utrechtuniversity/src-test-workspace:ubuntu_jammy - <<: *image_settings -provisioner: - name: ansible - config_options: - defaults: - remote_tmp: /tmp - playbooks: - converge: ../ext/molecule-src/converge.yml - prepare: ../ext/molecule-src/prepare.yml - env: - PLAYBOOK_DIR: ${PLAYBOOK_DIR:-"../../"} - ANSIBLE_VERBOSITY: ${ANSIBLE_VERBOSITY:-"2"} - ANSIBLE_ROLES_PATH: ${ANSIBLE_ROLES_PATH:-'../../roles'} - REQUIREMENTS_FILE: ${REQUIREMENTS_FILE:-"../../requirements.txt"} diff --git a/molecule/ext/molecule-src/prepare.yml b/molecule/ext/molecule-src/prepare.yml deleted file mode 100644 index 351a530..0000000 --- a/molecule/ext/molecule-src/prepare.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- name: Prepare - hosts: all - gather_facts: true - tasks: - - name: Clone git component - when: item.git is defined - ansible.builtin.git: - repo: "{{ item.git }}" - dest: /rsc/plugins/{{ item.name }}/ - version: "{{ item.version | default(omit) }}" - with_items: "{{ lookup('env', 'components') }}" - tags: skip_ansible_lint # linter complains about idempotence of git module - - - name: Copy local component - when: item.git is not defined - ansible.posix.synchronize: - src: "{{ item.dir | default(lookup('env', 'PLAYBOOK_DIR')) }}/" - dest: /rsc/plugins/{{ item.name }}/ - archive: false - links: true - recursive: true - rsync_opts: - - --exclude=".*" - ssh_connection_multiplexing: true - with_items: "{{ lookup('env', 'components') }}" - - # Apt cache is normally updated at deploy time by the SRC-OS component. - # Make sure it is fresh so our tests use recent apt repo information. - - name: Update apt cache - ansible.builtin.apt: - update_cache: true - when: ansible_os_family == 'Debian' - - - name: Set extra_preparations var - ansible.builtin.set_fact: - extra_preparations: "{{ lookup('env', 'extra_prepare_tasks' | default(omit)) }}" - - - name: Execute optional extra preparations - ansible.builtin.include_tasks: "{{ extra_preparations }}" - when: extra_preparations != "" diff --git a/molecule/ext/molecule-src/requirements.txt b/molecule/ext/molecule-src/requirements.txt deleted file mode 100644 index d9a0ad8..0000000 --- a/molecule/ext/molecule-src/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Default requirements file for running SRC molecule tests -molecule~=24.8 -molecule-plugins~=23.5 -jmespath~=1.0 -docker~=7.1 -podman~=5.0 diff --git a/molecule/ext/molecule-src/requirements.yml b/molecule/ext/molecule-src/requirements.yml deleted file mode 100644 index f03ed64..0000000 --- a/molecule/ext/molecule-src/requirements.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# This file is just to specify requirements for running the liter on the SRC-Molecule repository. -collections: - - name: ansible.posix - - name: community.general - - name: ansible.utils From c5c7b8ad9227af627517883ca3083a8017bf8e71 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 13:15:52 +0100 Subject: [PATCH 05/13] Squashed 'molecule/ext/molecule-src/' content from commit a78fe5a git-subtree-dir: molecule/ext/molecule-src git-subtree-split: a78fe5a6acf5c5df0d2bbafef05d565bf538a0e2 --- .ansible-lint | 4 + .github/dependabot.yml | 16 +++ .github/workflows/ansible-lint.yml | 17 +++ .github/workflows/molecule.yml | 47 ++++++++ README.md | 185 +++++++++++++++++++++++++++++ _run_component.yml | 49 ++++++++ converge.yml | 20 ++++ default.env.yml | 8 ++ molecule.yml | 30 +++++ prepare.yml | 41 +++++++ requirements.txt | 6 + requirements.yml | 6 + 12 files changed, 429 insertions(+) create mode 100644 .ansible-lint create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ansible-lint.yml create mode 100644 .github/workflows/molecule.yml create mode 100644 README.md create mode 100644 _run_component.yml create mode 100644 converge.yml create mode 100644 default.env.yml create mode 100644 molecule.yml create mode 100644 prepare.yml create mode 100644 requirements.txt create mode 100644 requirements.yml diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000..c0c2fa6 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,4 @@ +--- +profile: basic +exclude_paths: + - .github diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..65c1d2e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "pip" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml new file mode 100644 index 0000000..e29f837 --- /dev/null +++ b/.github/workflows/ansible-lint.yml @@ -0,0 +1,17 @@ +--- +name: Ansible Lint +on: + push: + branches: + - main + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + name: Ansible Lint + steps: + # Important: This sets up your GITHUB_WORKSPACE environment variable + - uses: actions/checkout@v6 + - name: Run ansible-lint + uses: ansible/ansible-lint@v25.11.1 diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml new file mode 100644 index 0000000..a496fd3 --- /dev/null +++ b/.github/workflows/molecule.yml @@ -0,0 +1,47 @@ +--- +name: Molecule Run +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + molecule: + runs-on: ubuntu-latest + env: + DOCKER_USER: ${{ github.actor }} + DOCKER_PW: ${{ secrets.GITHUB_TOKEN }} + DOCKER_REGISTRY: ghcr.io + ANSIBLE_FORCE_COLOR: '1' + ANSIBLE_RESULT_FORMAT: yaml + MOLECULE_CONFIG: molecule/ext/molecule-src/molecule.yml + REQUIREMENTS_FILE: requirements.txt + PLAYBOOK_DIR: ../../../ # relative to molecule/ext/molecule-src/molecule.yml + CRUN_VER: 1.11.2 + steps: + - name: Workaround crun issue on ubuntu # https://github.com/UtrechtUniversity/SRC-catalog-items/issues/3 + run: | + mkdir -p "${HOME}/.local/bin" + curl -L "https://github.com/containers/crun/releases/download/${CRUN_VER}/crun-${CRUN_VER}-linux-amd64" -o "${HOME}/.local/bin/crun" + chmod +x "${HOME}/.local/bin/crun" + crun --version + mkdir -p "${HOME}/.config/containers" + cat << EOF > "${HOME}/.config/containers/containers.conf" + [engine.runtimes] + crun = [ + "${HOME}/.local/bin/crun", + "/usr/bin/crun" + ] + EOF + - name: Checkout + uses: actions/checkout@v6 + - run: git clone https://github.com/UtrechtUniversity/SRC-molecule-test.git molecule + - run: mkdir molecule/ext && cd molecule/ext && ln -s ../../ molecule-src + - run: pip install -r ${{ env.REQUIREMENTS_FILE }} + - run: ansible-galaxy install -r requirements.yml + - name: Molecule tests + run: | + molecule -vv -c ${{ env.MOLECULE_CONFIG }} test diff --git a/README.md b/README.md new file mode 100644 index 0000000..0705a72 --- /dev/null +++ b/README.md @@ -0,0 +1,185 @@ +# SURF Research Cloud Molecule Testing Configuration + +This repository contains default configuration for running [Molecule](https://ansible.readthedocs.io/projects/molecule/) tests for SURF Research Cloud (SRC) Components and Catalog Items. The repository is meant to be included into other repositories as a subtree. + +## Getting Started + +Follow the installation and setup instructions below, then run: + +`molecule -c molecule/ext/molecule-src/molecule.yml test -s ` + +...where `` is the name of one of the subdirectories of the `molecule` directory, e.g. `playbook-security_updates`. + +### Requirements + +1. Docker and/or Podman (to spin up test containers) +1. Python and pip +1. Ansible +1. Molecule +1. Access to the [test container images](https://github.com/UtrechtUniversity/SRC-test-workspace) + +Before you start, run: + +* `pip install -r molecule/ext/molecule-src/requirements.txt` to install Molecule itself and other python dependencies. +* `ansible-galaxy install -r molecule/ext/molecule-src/requirements.yml` to install Ansible collection dependencies. + +The default `molecule.yml` is configured to use the images from [this package](https://github.com/UtrechtUniversity/SRC-test-workspace/), but you can override this to use other images. + +### Getting the container images + +The default `molecule.yml` is configured to use the images from [this package](https://github.com/UtrechtUniversity/SRC-test-workspace/), but you can also override this to use other images. There are two ways to provide Molecule with access to the right images: + +1. Manually pull the images from the container registry. If the images are already available locally, there is no need to authenticate with the container registry. + * `docker login ghcr.io` (or `podman login`) + * enter your github username and a valid token + * `docker pull ` (or `podman pull`) + * See [the package](https://github.com/UtrechtUniversity/SRC-test-workspace/) for the image names +2. If you set the right variables when running `molecule`, it will pull the images automatically. + * `export DOCKER_REGISTRY=ghcr.io` + * `export DOCKER_USER=githubusername` + * `export DOCKER_PW=githubtoken` + +### Install SRC-specific configuration + +NB: this is not necessary for running tests on a repository already containing the configuration files from this repository, only to add tests to a repository that does not contain them yet. + +To add Molecule tests to your SRC component or catalog item repository, follow these steps: + +* create a `molecule` directory in your repository's root: `mkdir molecule` +* include the contents of this repository as a subtree, under `molecule/ext/molecule-src` + * `git remote add molecule-src https://github.com/UtrechtUniversity/SRC-molecule.git` + * `git subtree add --prefix molecule/ext/molecule-src molecule-src main --squash` +* copy the default `.env.yml` file to your repository root: `cp molecule/ext/molecule-src/default.env.yml .env.yml` + * optionally edit the contents of `.env.yml`, if your playbooks are not in the default location (the repository root) +* run `pip install -r molecule/ext/molecule-src/requirements.txt` + +That's it for setup! You're now ready to start [adding your own scenarios](#adding-scenarios). + +### Scenarios + +Molecule runs tests for each *scenario* defined under the `molecule` directory. You can think of each scenario as a self-contained test suite. A scenario is just a subdirectory of the `molecule` directory that contains a number of configuration files and playbooks. For our purposes, these are minimally: + +* `molecule.yml` - sets general configuration variables for the test suite (e.g. what platform to run on). Overrides the defaults in `molecule/ext/molecule-src/molecule.yml`. +* `prepare.yml` - a playbook used to make preparations on the container before running the Ansible code that we want to test on it. +* `converge.yml` - a playbook that specifies how to deploy the Ansible role or playbook that we want to test on the container. +* `verify.yml` - a playbook that contains additional test assertions after the `converge` step is run. + +We want to test our SURF Research Cloud (SRC) components in a specific way, namely by: + +1. copying them onto the workspace +2. then executing them with Ansible *on the workspace* (as opposed to using Ansible on the host). + +We want this procedure because that is [how components are actually deployed onto SRC workspaces](https://github.com/UtrechtUniversity/SRC-test-workspace#how-it-works). + +To help us achieve this, the `default` scenario contains a `prepare.yml` that takes care of 1., and a `converge.yml` that takes care of 2. Each individual scenario will automatically use these default playbooks, as long as `molecule` is run with the ` -c molecule/default/molecule.yml`. + +### Adding scenarios + +To create a Molecule scenario to test, just create a subdirectory of the `molecule` directory, e.g.: + +`mkdir molecule/my-component` + +Now add a `molecule.yml` file to the `my-component` subdirectory, with contents like the following: + +```yaml +provisioner: + name: ansible + env: + components: + - name: my-component + path: my-component.yaml + parameters: # Define all parameters needed by the component here + my_component_param1: 'Foo' + - name: my-git-component # You can also provide components that should be cloned onto the workspace using git + git: https://github.com/foo/bar.git + version: my_branch + path: playbook.ymk +``` + +The above config file does not provide a `platforms` key, so tests for this scenario will use the default platforms (containers) specified in `molecule/ext/molecule-src/molecule.yml` (Ubuntu Focal and Ubuntu Jammy). You can override this by adding your own platform definition, e.g.: + +```yaml +platforms: + - name: workspace-src-ubuntu_focal-desktop + image: ghcr.io/utrechtuniversity/src-test-workspace:ubuntu_focal-desktop # You can also change this to another image + pre_build_image: true + # The following lines are an example of how to specify a container registry to pull the image from, if it is not already available locally. + registry: + url: $DOCKER_REGISTRY # Use environment variables to define the URL and credentials + credentials: + username: $DOCKER_USER +``` + +Note that you can also define multiple platforms. + +### Specifying a Driver + +The molecule tests can use either Docker or Podman (default). You can override the default driver in your scenario's `molecule.yml`. Or even easier: set an environment variable like `DRIVER=docker`. + +### Adding additional preparation tasks + +Sometimes it may be helpful to perform certain tasks before the `converge` step. The default `prepare.yml` playbook will check if the user has set the `extra_prepare_tasks` key in the scenario's `molecule.yml` `env` section. Set this key to a relative path to a playbook (relative to the location of the default molecule configuration), and the tasks in that playbook will be executed at the end of the `prepare` step. + +### Adding additional assertions + +You can a `verify.yml` to your scenario to perform additional assertions. See the [Molecule docs](https://ansible.readthedocs.io/projects/molecule/configuration/#verifier). + +The main thing we are testing using Molecule is whether our playbooks are deployable on SRC workspaces. Since Ansible itself checks whether each task is actually successful (i.e. *changes* the target in the required way), we do not need to assert that everything we do in a playbook actually happens: when a playbook contains the instruction to e.g. install a certain package, and the playbook deploys on our test containers without error, we *know* that the container is in the required state. + +However, in some cases, it may be desirable to perform additional assertions after deployment is complete. Molecule allows you to do this by adding a `verify.yml` playbook to your scenario. You can use this perform assertions either using Ansible's own [assert module](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/assert_module.html), or using [testinfra](https://ansible.readthedocs.io/projects/molecule/configuration/#molecule.verifier.testinfra.Testinfra). + +What can/should be tested in a `verify.yml`? For instance: + +* behaviour that is conditional on certain parameters, and therefore not already triggered by simply executing the playbook on the test container (in `converge.yml`) +* lower level behaviour: e.g., not whether service `apache2` is running (we know that it is, since we require this in our playbook), but whether connecting to port 80 actually yields the desired webservice (e.g. a Jupyter notebook). + +## Debugging + +### Inspecting the workspace + +For debugging and development purposes, it can be useful to inspect what's going on on a container after executing a playbook or role on it. There are two ways of doing this: + +1. Run `molecule` with the `create` command instead of the `test` command: `molecule -c molecule/default/molecule.yml create -s ` +2. Run `molecule` with the `--destroy=never` flag: `molecule -c molecule/default/molecule.yml test -s --destroy=never` + +Both of these methods ensure that the container is not destroyed after molecule is done. This means you can then login to the container and see what's going on. For instance, after molecule is done, you can: + +1. `docker container list` + * see that the container `workspace-src-ubuntu_focal` is still running +2. `docker exec -it workspace-src-ubuntu_focal bash` + * login to the container as root + +### The component cannot be found on the workspace + +Problem: the `converge` step fails with the following error message in Ansible's `stderr` result: + +``` +'ERROR! the playbook: /rsc/plugins/componentname/playbookname.yml could not be found' +``` + +This can occassionally occur when Molecule thinks it has already run the `prepare` step on your container, but actually hasn't. (This happens, for instance, when the `converge` step uses as an old container that is still running because you used `--destroy=never` in a previous run.) + +Try resetting your molecule cache: + +`molecule -c molecule/ext/molecule-src/molecule.yml reset -s playbook-aptly` + +This will stop the container and flush the cache. Sometimes manually removing the cache may also be useful during troubleshooting: + +`rm -rf ~/.cache/molecule` + +### Container unreachable error + +Molecule runs sometimes fail with an error message like the following: + +``` +task path: /home/user/researchcloud-items/molecule/ext/molecule-src/prepare.yml:2 +fatal: [workspace-src-ubuntu_jammy]: UNREACHABLE! => changed=false + msg: 'Failed to create temporary directory. In some cases, you may have been able to authenticate and did not have permissions on the target directory. Consider changing the remote tmp path in ansible.cfg to a path rooted in "/tmp", for more error information use -vvv. Failed command was: ( umask 77 && mkdir -p "` echo ~/.ansible/tmp `"&& mkdir "` echo ~/.ansible/tmp/ansible-tmp-1710760454.6766412-70112-138145577949983 `" && echo ansible-tmp-1710760454.6766412-70112-138145577949983="` echo ~/.ansible/tmp/ansible-tmp-1710760454.6766412-70112-138145577949983 `" ), exited with result 1' + unreachable: true +``` + +This may occur when trying to initalize a container with the command parameter set to `/sbin/init`, i.e. when trying to run a container controlled by `systemd`. Some component tests may need this, because they are testing functionality of `systemd` services. However, in some circumstances, starting a container with `/sbin/init` fails: + +* Docker support for `systemd` is not excellent, try using Podman instead. +* The error also occurs when using container emulation (e.g. using an `amd64` image on an `arm64` host). Get a native image instead. +* Adding the `privileged: true` option to the platform usually takes care of the problem (even when using Docker!), but this is only recommended as a workaround [for security reasons](https://www.trendmicro.com/en_vn/research/19/l/why-running-a-privileged-container-in-docker-is-a-bad-idea.html). diff --git a/_run_component.yml b/_run_component.yml new file mode 100644 index 0000000..70db46c --- /dev/null +++ b/_run_component.yml @@ -0,0 +1,49 @@ +--- +- name: Install component Ansible Galaxy dependencies + tags: molecule-idempotence-notest + block: + - name: Check for plugin requirements file + stat: + path: /rsc/plugins/{{ remote_plugin.script_folder }}/requirements.yml + register: register_requirements_1 + + - name: Check for plugin requirements file 2 + stat: + path: /rsc/plugins/{{ remote_plugin.script_folder }}/{{ remote_plugin.path | dirname }}/requirements.yml + register: register_requirements_2 + + - name: Set path to file 1 if it exists + set_fact: + requirements_path: "{{ register_requirements_1.stat.path }}" + when: register_requirements_1.stat.exists + + - name: Set path to file 2 if file 1 does not exist and file 2 exists + set_fact: + requirements_path: "{{ register_requirements_2.stat.path }}" + when: not register_requirements_1.stat.exists and register_requirements_2.stat.exists + + - name: Install role dependencies for Ansible plugin + when: requirements_path is defined + command: | + ansible-galaxy role install -r {{ requirements_path }} -p /rsc/plugins/{{ remote_plugin.script_folder | basename }} + args: + chdir: /rsc/plugins/{{ remote_plugin.script_folder | basename }} + executable: /bin/bash + + - name: Install collection dependencies for Ansible plugin + when: requirements_path is defined + command: | + ansible-galaxy collection install -r {{ requirements_path }} -p /rsc/plugins/{{ remote_plugin.script_folder | basename }}/collections + args: + chdir: /rsc/plugins/{{ remote_plugin.script_folder | basename }} + executable: /bin/bash + +- name: Test the component by executing it using ansible on the workspace + ansible.builtin.command: > + ansible-playbook -c local -v -b {{ remote_plugin.arguments }} -e='{{ remote_plugin.parameters }}' /rsc/plugins/{{ remote_plugin.script_folder + }}/{{remote_plugin.path }} + register: ansible_on_workspace + changed_when: > + ansible_on_workspace.stdout_lines is not defined or + 'changed=0' not in + ansible_on_workspace.stdout_lines[ lookup('ansible.utils.index_of', ansible_on_workspace.stdout_lines, 'regex', '\s*PLAY RECAP\s*')+1 ] diff --git a/converge.yml b/converge.yml new file mode 100644 index 0000000..513d9d5 --- /dev/null +++ b/converge.yml @@ -0,0 +1,20 @@ +--- +- name: Converge + hosts: all + gather_facts: false + tasks: + - name: Debug -- list all components to be executed + ansible.builtin.debug: + msg: "{{ item.name }}" + with_items: "{{ lookup('env', 'components') | from_yaml }}" + + - name: Run components + ansible.builtin.include_tasks: _run_component.yml + vars: + remote_plugin: + script_type: Ansible PlayBook + arguments: -i 127.0.0.1, --skip-tags {{ ansible_skip_tags | join(',') }} + parameters: "{{ item.parameters | default({}) | to_json }}" + script_folder: "{{ item.name }}" + path: "{{ item.path }}" + with_items: "{{ lookup('env', 'components') | from_yaml }}" diff --git a/default.env.yml b/default.env.yml new file mode 100644 index 0000000..1a566a9 --- /dev/null +++ b/default.env.yml @@ -0,0 +1,8 @@ +--- +# Default Molecule .env.yml file +# NOTE: to make molecule use this file, place it in the root of your project (or from whichever directory you are calling 'molecule'), and rename it to .env.yml +PLAYBOOK_DIR: ../../../ # Relative to the default molecule.yml (molecule/ext/molecule-src/molecule.yml) +ANSIBLE_VERBOSITY: "2" +ANSIBLE_ROLES_PATH: ../../roles # Relative to your scenario (e.g. molecule/role-foo/..) +REQUIREMENTS_FILE: requirements.txt +ANSIBLE_RESULT_FORMAT: yaml diff --git a/molecule.yml b/molecule.yml new file mode 100644 index 0000000..50a470d --- /dev/null +++ b/molecule.yml @@ -0,0 +1,30 @@ +--- +driver: + name: ${DRIVER-podman} + image_settings: &image_settings + pre_build_image: true + registry: + url: $DOCKER_REGISTRY + credentials: + username: $DOCKER_USER + password: $DOCKER_PW +platforms: # Test on ubuntu focal and jammy by default + - name: workspace-src-ubuntu_focal + image: ghcr.io/utrechtuniversity/src-test-workspace:ubuntu_focal + <<: *image_settings + - name: workspace-src-ubuntu_jammy + image: ghcr.io/utrechtuniversity/src-test-workspace:ubuntu_jammy + <<: *image_settings +provisioner: + name: ansible + config_options: + defaults: + remote_tmp: /tmp + playbooks: + converge: ../ext/molecule-src/converge.yml + prepare: ../ext/molecule-src/prepare.yml + env: + PLAYBOOK_DIR: ${PLAYBOOK_DIR:-"../../"} + ANSIBLE_VERBOSITY: ${ANSIBLE_VERBOSITY:-"2"} + ANSIBLE_ROLES_PATH: ${ANSIBLE_ROLES_PATH:-'../../roles'} + REQUIREMENTS_FILE: ${REQUIREMENTS_FILE:-"../../requirements.txt"} diff --git a/prepare.yml b/prepare.yml new file mode 100644 index 0000000..d09f1a3 --- /dev/null +++ b/prepare.yml @@ -0,0 +1,41 @@ +--- +- name: Prepare + hosts: all + gather_facts: true + tasks: + - name: Clone git component + when: item.git is defined + ansible.builtin.git: + repo: "{{ item.git }}" + dest: /rsc/plugins/{{ item.name }}/ + version: "{{ item.version | default(omit) }}" + with_items: "{{ lookup('env', 'components') }}" + tags: skip_ansible_lint # linter complains about idempotence of git module + + - name: Copy local component + when: item.git is not defined + ansible.posix.synchronize: + src: "{{ item.dir | default(lookup('env', 'PLAYBOOK_DIR')) }}/" + dest: /rsc/plugins/{{ item.name }}/ + archive: false + links: true + recursive: true + rsync_opts: + - --exclude=".*" + ssh_connection_multiplexing: true + with_items: "{{ lookup('env', 'components') | from_yaml }}" + + # Apt cache is normally updated at deploy time by the SRC-OS component. + # Make sure it is fresh so our tests use recent apt repo information. + - name: Update apt cache + ansible.builtin.apt: + update_cache: true + when: ansible_os_family == 'Debian' + + - name: Set extra_preparations var + ansible.builtin.set_fact: + extra_preparations: "{{ lookup('env', 'extra_prepare_tasks' | default(omit)) }}" + + - name: Execute optional extra preparations + ansible.builtin.include_tasks: "{{ extra_preparations }}" + when: extra_preparations != "" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..12ecf73 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +# Default requirements file for running SRC molecule tests +molecule~=25.9 +molecule-plugins~=25.8 +jmespath~=1.0 +docker~=7.1 +podman~=5.6 diff --git a/requirements.yml b/requirements.yml new file mode 100644 index 0000000..f03ed64 --- /dev/null +++ b/requirements.yml @@ -0,0 +1,6 @@ +--- +# This file is just to specify requirements for running the liter on the SRC-Molecule repository. +collections: + - name: ansible.posix + - name: community.general + - name: ansible.utils From b03d6ae0f99ba81c861cacea32c59286c87ffba5 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 13:17:08 +0100 Subject: [PATCH 06/13] Rename default scenario (default does not work with overriden playbooks) --- molecule/{default => openwebui}/molecule.yml | 0 molecule/{default => openwebui}/verify.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename molecule/{default => openwebui}/molecule.yml (100%) rename molecule/{default => openwebui}/verify.yml (100%) diff --git a/molecule/default/molecule.yml b/molecule/openwebui/molecule.yml similarity index 100% rename from molecule/default/molecule.yml rename to molecule/openwebui/molecule.yml diff --git a/molecule/default/verify.yml b/molecule/openwebui/verify.yml similarity index 100% rename from molecule/default/verify.yml rename to molecule/openwebui/verify.yml From 8f081b04d0380915faaeec279eea097174c746b5 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 13:20:26 +0100 Subject: [PATCH 07/13] Don't test on Ubuntu 20 --- molecule/openwebui/molecule.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/molecule/openwebui/molecule.yml b/molecule/openwebui/molecule.yml index f4dca86..aa967f9 100644 --- a/molecule/openwebui/molecule.yml +++ b/molecule/openwebui/molecule.yml @@ -1,4 +1,17 @@ --- +driver: + name: ${DRIVER-podman} + image_settings: &image_settings + pre_build_image: true + registry: + url: $DOCKER_REGISTRY + credentials: + username: $DOCKER_USER + password: $DOCKER_PW +platforms: # Test on ubuntu jammy by default + - name: workspace-src-ubuntu_jammy + image: ghcr.io/utrechtuniversity/src-test-workspace:ubuntu_jammy + <<: *image_settings provisioner: name: ansible env: From 1c4b71633faa375d4914aaf41d72c5d1149a2f11 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 13:20:34 +0100 Subject: [PATCH 08/13] Replace deprecated env var --- .env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.yml b/.env.yml index 3606ed6..f9f79e4 100644 --- a/.env.yml +++ b/.env.yml @@ -4,5 +4,5 @@ PLAYBOOK_DIR: ../../../../ # Relative to the default molecule.yml (molecule/ext/molecule-src/molecule.yml) ANSIBLE_VERBOSITY: '2' ANSIBLE_ROLES_PATH: ../../../roles # Relative to your scenario (e.g. molecule/role-foo/..) -ANSIBLE_STDOUT_CALLBACK: yaml +ANSIBLE_RESULT_FORMAT: yaml REQUIREMENTS_FILE: requirements.txt # Relative to the default molecule.yml From ddd15ea897a82cf67abc444d44a80edad74645c8 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 13:32:28 +0100 Subject: [PATCH 09/13] Squashed 'molecule/ext/molecule-src/' changes from a78fe5a..03f21fd 03f21fd Update molecule version to 25.11 git-subtree-dir: molecule/ext/molecule-src git-subtree-split: 03f21fd8b80db0ab431cf53ddb41b3c18b6509eb --- molecule/ext/molecule-src/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molecule/ext/molecule-src/requirements.txt b/molecule/ext/molecule-src/requirements.txt index 12ecf73..0747abd 100644 --- a/molecule/ext/molecule-src/requirements.txt +++ b/molecule/ext/molecule-src/requirements.txt @@ -1,5 +1,5 @@ # Default requirements file for running SRC molecule tests -molecule~=25.9 +molecule~=25.11 molecule-plugins~=25.8 jmespath~=1.0 docker~=7.1 From 24a877246631d5ef6bc5e6f18f6537282d62f95c Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 13:32:56 +0100 Subject: [PATCH 10/13] Update molecule scenario paths in README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 846bac3..ffcfe89 100644 --- a/README.md +++ b/README.md @@ -58,13 +58,13 @@ To run `molecule`: * `pip install -r molecule/ext/molecule-src/requirements.txt` 1. Install the Ansible dependencies: * `ansible-galaxy install -r requirements.yml` -1. Run: `molecule -c molecule/ext/molecule-src/molecule.yml test` +1. Run: `molecule -c molecule/ext/molecule-src/molecule.yml test --all` -This will run the scenario in the `molecule/default` directory. +This will run all scenarios in the `molecule/` directory. To add and run a new scenario, simply: -1. Copy `molecule/default` into `molecule/`. +1. Copy `molecule/playbook-test` into `molecule/`. 1. Make your desired changes. 1. Run: `molecule -c molecule/ext/molecule-src/molecule.yml test -s ` From 993f6f40c71aa388f8dc7d7970755b93afa47625 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 13:43:21 +0100 Subject: [PATCH 11/13] Fix deprecated env var in CI --- .github/workflows/molecule.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index ab877cf..b8f2b6c 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest env: ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_STDOUT_CALLBACK: yaml + ANSIBLE_RESULT_FORMAT: yaml CRUN_VER: 1.11.2 REQUIREMENTS_FILE: molecule/ext/molecule-src/requirements.txt GALAXY_REQUIREMENTS_FILE: molecule/ext/molecule-src/requirements.yml From c6288d3c87200bd3dd1040e9a3922436b3843d60 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 13:49:11 +0100 Subject: [PATCH 12/13] Test container: explicitly use /sbin/init as starting command --- molecule/openwebui/molecule.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/molecule/openwebui/molecule.yml b/molecule/openwebui/molecule.yml index aa967f9..f638719 100644 --- a/molecule/openwebui/molecule.yml +++ b/molecule/openwebui/molecule.yml @@ -11,6 +11,7 @@ driver: platforms: # Test on ubuntu jammy by default - name: workspace-src-ubuntu_jammy image: ghcr.io/utrechtuniversity/src-test-workspace:ubuntu_jammy + command: /sbin/init <<: *image_settings provisioner: name: ansible From 4518eaae9f793460ab028915dd3adba1f1f842a7 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 27 Nov 2025 15:58:40 +0100 Subject: [PATCH 13/13] role ollama-serve: mark task as not idempotent --- roles/ollama-serve/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/ollama-serve/tasks/main.yml b/roles/ollama-serve/tasks/main.yml index 1bb8680..eac2c0e 100644 --- a/roles/ollama-serve/tasks/main.yml +++ b/roles/ollama-serve/tasks/main.yml @@ -36,6 +36,7 @@ enabled: true - name: Pull requested model + tags: molecule-idempotence-notest ansible.builtin.command: cmd: /opt/ollama/bin/ollama pull "{{ ollama_serve_model }}" when: ollama_serve_model | length > 0