ref: use org secret and bump base images and toolkits (#651)

* ref: use org secret

* update to non-EOL base images

* fix: use miniforge since mambaforge is deprecated

* update base test images

* Changed mirror link to http://vault.centos.org to retrieve yum packages. Removed curl and localedef in dependencies. Fix #653

* replace more occurances

* fix more fedora references

* fix: update jq source links

* update localdef packages

* update jq versions

* remove jq 1.5 references

* add default to test template path

* fix jq builds

* fix: adjust config for different versions of jq

* fix: adjust config for jq 1.6

* updating apptainer version and some tests

* use apptainer when set for tests

* build without sudo

* try if a different version works

* remove default and add yum config

---------

Co-authored-by: iishiishii <[email protected]>

Author: satra

.github/workflows/create_workflows.py

@@ -31,12 +31,12 @@
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 
 apt_based = [
+    "ubuntu:24.04",
     "ubuntu:22.04",
-    "ubuntu:18.04",
+    "debian:bookworm-slim",
     "debian:bullseye-slim",
-    "debian:buster-slim",
 ]
-yum_based = ["fedora:36", "centos:7"]
+yum_based = ["fedora:40", "centos:8"]
 
 """
 Add a "skip_versions" key to the software dictionary if you want to skip

.github/workflows/pull-request.yml

@@ -19,7 +19,7 @@ jobs:
         steps:
         -   name: Install Apptainer
             env:
-                VERSION: 1.1.5
+                VERSION: 1.3.6
             run: |
                 sudo apt-get update
                 sudo apt-get install -y wget

.github/workflows/release.yml

@@ -41,4 +41,4 @@ jobs:
             run: |
                 ~/auto shipit -vv
             env:
-                GH_TOKEN: ${{ secrets.AUTO_USER_TOKEN }}
+                GH_TOKEN: ${{ secrets.AUTO_TOKEN }}

docs/user_guide/examples.rst

@@ -117,7 +117,7 @@ Docker
 
     neurodocker generate docker \
         --pkg-manager yum \
-        --base-image fedora:36 \
+        --base-image fedora:40 \
         --afni method=binaries version=latest \
     > afni-binaries.Dockerfile
 
@@ -130,7 +130,7 @@ This does not install AFNI's R packages. To install relevant R things, use the f
 
     neurodocker generate docker \
         --pkg-manager yum \
-        --base-image fedora:36 \
+        --base-image fedora:40 \
         --afni method=binaries version=latest install_r_pkgs=true \
     > afni-binaries-r.Dockerfile
 
@@ -149,7 +149,7 @@ This does not install AFNI's R packages. To install relevant R things, use the f
 
 ..     neurodocker generate docker \
 ..         --pkg-manager yum \
-..         --base-image fedora:36 \
+..         --base-image fedora:40 \
 ..         --afni method=source version=master \
 ..     > afni-source.Dockerfile
 

neurodocker/cli/minify/_trace.sh

@@ -38,9 +38,9 @@ function install_missing_dependencies() {
 
 
 function install_conda_reprozip() {
-  TMP_CONDA_INSTALLER=/tmp/miniconda.sh
+  TMP_CONDA_INSTALLER=/tmp/miniforge.sh
   ls /tmp
-  curl -sSL -o "$TMP_CONDA_INSTALLER" "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh"
+  curl -sSL -o "$TMP_CONDA_INSTALLER" "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
   bash $TMP_CONDA_INSTALLER -b -f -p $REPROZIP_CONDA
   rm -f $TMP_CONDA_INSTALLER
   ${REPROZIP_CONDA}/bin/mamba install -c conda-forge -y reprozip

neurodocker/cli/minify/tests/test_minify.py

@@ -16,7 +16,7 @@
 @skip_arm_on_mac
 def test_minify():
     client = docker.from_env()
-    container = client.containers.run("python:3.9-slim", detach=True, tty=True)
+    container = client.containers.run("python:3.10-slim", detach=True, tty=True)
     commands = ["python --version", """python -c 'print(123)'"""]
     try:
         runner = CliRunner()
@@ -35,7 +35,8 @@ def test_minify():
             assert ret == 0, f"unexpected non-zero return code when running '{cmd}'"
 
         # This should fail.
-        ret, result = container.exec_run("pip --help")
+        cmd = "pip --help"
+        ret, result = container.exec_run(cmd)
         assert ret != 0, f"unexpected zero return code when running '{cmd}'"
 
     finally:
@@ -46,7 +47,7 @@ def test_minify():
 @skip_arm_on_mac
 def test_minify_abort():
     client = docker.from_env()
-    container = client.containers.run("python:3.9-slim", detach=True, tty=True)
+    container = client.containers.run("python:3.10-slim", detach=True, tty=True)
     commands = ["python --version", """python -c 'print(123)'"""]
     try:
         runner = CliRunner()
@@ -64,7 +65,8 @@ def test_minify_abort():
             assert ret == 0, f"unexpected non-zero return code when running '{cmd}'"
 
         # This should still succeed.
-        ret, result = container.exec_run("pip --help")
+        cmd = "pip --help"
+        ret, result = container.exec_run(cmd)
         assert ret == 0, f"unexpected non-zero return code when running '{cmd}'"
 
     finally:
@@ -80,7 +82,7 @@ def test_minify_with_mounted_volume(tmp_path: Path):
     (tmp_path / "foobar.txt").write_text("Foobar")
 
     container = client.containers.run(
-        "python:3.8-slim",
+        "python:3.10-slim",
         detach=True,
         tty=True,
         volumes={str(tmp_path): {"bind": "/work", "mode": "rw"}},

neurodocker/cli/tests/sample-template-jq.yaml

@@ -17,12 +17,17 @@ binaries:
         ND_BAZ: '1234'
         ND_FOO: BAR
     instructions: |
+        export DUMMY="dummy"
+        {%- if self.pkg_manager == "yum" %}
+        sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
+        sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
+        {%- endif %}
         {{ self.install_dependencies() }}
         curl -fsSL --output /usr/local/bin/jq {{ self.urls[self.version]}}
         chmod +x /usr/local/bin/jq
     urls:
-        '1.5': https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
-        '1.6': https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+        '1.6': https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64
+        '1.7': https://github.com/jqlang/jq/releases/download/jq-1.7/jq-linux64
 source:
     arguments:
         required:
@@ -48,13 +53,21 @@ source:
         ND_BAZ: '1234'
         ND_FOO: BAR
     instructions: |
+        export DUMMY="dummy"
+        {%- if self.pkg_manager == "yum" %}
+        sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
+        sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
+        {%- endif %}
         {{ self.install_dependencies() }}
         mkdir jq
         cd jq
-        curl -fsSL https://github.com/stedolan/jq/releases/download/jq-{{self.version}}/jq-{{self.version}}.tar.gz \
+        curl -fsSL https://github.com/jqlang/jq/releases/download/jq-{{self.version}}/jq-{{self.version}}.tar.gz \
         | tar xz --strip-components 1
-        # Only required in version 1.6
-        autoreconf -fi
-        ./configure --disable-maintainer-mode
+        {%- if self.version == '1.6' %}
+            autoreconf -i
+            ./configure --with-oniguruma=builtin --disable-maintainer-mode
+        {%- else %}
+            ./configure --with-oniguruma=builtin
+        {%- endif %}
         make
         make install

neurodocker/cli/tests/test_build_images_with_cli.py

@@ -26,7 +26,8 @@
     ],
 )
 @pytest.mark.parametrize(
-    ["pkg_manager", "base_image"], [("apt", "debian:buster-slim"), ("yum", "centos:7")]
+    ["pkg_manager", "base_image"],
+    [("apt", "debian:bullseye-slim"), ("yum", "centos:8")],
 )
 def test_build_image_from_registered(
     tmp_path: Path, cmd: str, pkg_manager: str, base_image: str
@@ -46,11 +47,11 @@ def test_build_image_from_registered(
             "--pkg-manager",
             pkg_manager,
             "--jq",
-            "version=1.5",
+            "version=1.7",
         ],
     )
     assert result.exit_code == 0, result.output
-    assert "jq-1.5/jq-linux64" in result.output
+    assert "jq-1.7/jq-linux64" in result.output
 
     spec = "Dockerfile" if cmd == "docker" else "Singularity"
     (tmp_path / spec).write_text(result.output)
@@ -59,6 +60,8 @@ def test_build_image_from_registered(
     with build_fn(tmp_path) as img:
         stdout, _ = run_fn(img, args=["jq", "--help"])
         assert "jq is a tool for processing JSON" in stdout
+        stdout, _ = run_fn(img, args=["jq", "--version"])
+        assert "jq-1.7" in stdout
 
 
 @pytest.mark.long
@@ -82,7 +85,7 @@ def test_json_roundtrip(cmd: str, inputs: str, tmp_path: Path):
             cmd,
             "--json",
             "--base-image",
-            "debian:buster-slim",
+            "debian:bullseye-slim",
             "--pkg-manager",
             "apt",
             "--install",

neurodocker/cli/tests/test_cli.py

@@ -165,18 +165,18 @@ def test_render_registered(cmd: str, pkg_manager: str):
         [
             cmd,
             "--base-image",
-            "debian:buster",
+            "debian:bullseye",
             "--pkg-manager",
             pkg_manager,
             "--jq",
-            "version=1.5",
-            "--jq",
             "version=1.6",
+            "--jq",
+            "version=1.7",
         ],
     )
     assert result.exit_code == 0, result.output
-    assert "jq-1.5/jq-linux64" in result.output
     assert "jq-1.6/jq-linux64" in result.output
+    assert "jq-1.7/jq-linux64" in result.output
 
 
 # Test that we add the default header and default/custom entrypoints
@@ -188,7 +188,7 @@ def test_default_header_and_entrypoint(cmd: str, pkg_manager: str, entrypoint: s
     cmd_ = [
         cmd,
         "--base-image",
-        "debian:buster",
+        "debian:bullseye",
         "--pkg-manager",
         pkg_manager,
     ]

neurodocker/reproenv/tests/sample-template-jq.yaml

@@ -12,7 +12,7 @@ binaries:
         -   ca-certificates
         -   curl
         debs:
-    # Not required for jq -- used to test if packackages are installed properly with dpkg.
+    # Not required for jq -- used to test if packages are installed properly with dpkg.
         -   http://ftp.us.debian.org/debian/pool/main/r/rust-fd-find/fd-find_7.2.0-2_amd64.deb
         yum:
         -   curl
@@ -25,8 +25,8 @@ binaries:
         curl -fsSL --output /usr/local/bin/jq {{ self.urls[self.version]}}
         chmod +x /usr/local/bin/jq
     urls:
-        '1.5': https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
-        '1.6': https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+        '1.6': https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64
+        '1.7': https://github.com/jqlang/jq/releases/download/jq-1.7/jq-linux64
 source:
     arguments:
         required:
@@ -55,10 +55,13 @@ source:
         {{ self.install_dependencies() }}
         mkdir jq
         cd jq
-        curl -fsSL https://github.com/stedolan/jq/releases/download/jq-{{self.version}}/jq-{{self.version}}.tar.gz \
+        curl -fsSL https://github.com/jqlang/jq/releases/download/jq-{{self.version}}/jq-{{self.version}}.tar.gz \
         | tar xz --strip-components 1
-        # Only required in version 1.6
-        autoreconf -fi
-        ./configure --disable-maintainer-mode
+        {%- if self.version == '1.6' %}
+            autoreconf -i
+            ./configure --with-oniguruma=builtin --disable-maintainer-mode
+        {%- else %}
+            ./configure --with-oniguruma=builtin
+        {%- endif %}
         make
         make install

neurodocker/reproenv/tests/test_build_images_from_registered_templates.py

@@ -26,7 +26,8 @@
     ],
 )
 @pytest.mark.parametrize(
-    ["pkg_manager", "base_image"], [("apt", "debian:buster-slim"), ("yum", "fedora:33")]
+    ["pkg_manager", "base_image"],
+    [("apt", "debian:bullseye-slim"), ("yum", "fedora:40")],
 )
 @pytest.mark.parametrize(
     ["jq_version", "jq_version_output", "fd_version_startswith"],
@@ -81,12 +82,13 @@ def test_build_using_renderer_from_dict(
     ],
 )
 @pytest.mark.parametrize(
-    ["pkg_manager", "base_image"], [("apt", "debian:buster-slim"), ("yum", "fedora:33")]
+    ["pkg_manager", "base_image"],
+    [("apt", "debian:bullseye-slim"), ("yum", "fedora:40")],
 )
 @pytest.mark.parametrize(["method"], [("binaries",), ("source",)])
 @pytest.mark.parametrize(
     ["jq_version", "jq_version_output", "fd_version_startswith"],
-    [("1.6", "jq-1.6", "fd"), ("1.5", "jq-1.5", "fd")],
+    [("1.6", "jq-1.6", "fd"), ("1.7", "jq-1.7", "fd")],
 )
 def test_build_using_renderer_instance_methods(
     cmd: str,
@@ -118,10 +120,7 @@ def test_build_using_renderer_instance_methods(
         stdout, _ = run_fn(img, args=["jq", "--help"])
         assert stdout.startswith("jq - commandline JSON processor")
         stdout, _ = run_fn(img, args=["jq", "--version"])
-        if method == "source" and jq_version == "1.5":
-            assert stdout == "jq-"
-        else:
-            assert stdout == jq_version_output
+        assert stdout == jq_version_output
         # Test that deb was installed
         if method == "binaries":
             stdout, _ = run_fn(img, args=[fd_exe, "--version"])

neurodocker/reproenv/tests/test_build_images_simple.py

@@ -21,9 +21,9 @@ def test_build_simple(cmd: str, tmp_path):
     # Create a Dockerfile.
     r = rcls("apt")
     if isinstance(r, DockerRenderer):
-        r.from_("debian:buster-slim", as_="builder")
+        r.from_("debian:bullseye-slim", as_="builder")
     else:
-        r.from_("debian:buster-slim")
+        r.from_("debian:bullseye-slim")
     r.arg("FOO")
     r.copy(["foo.txt", "tst/baz.txt"], "/opt/")
     r.env(PATH="$PATH:/opt/foo/bin")

neurodocker/reproenv/tests/test_renderers_docker.py

@@ -307,13 +307,13 @@ def test_docker_render_from_instance_methods():
     )
 
     d = DockerRenderer("apt")
-    d.from_("debian:buster-slim")
+    d.from_("debian:bullseye-slim")
     d.entrypoint(["echo", "foo bar"])
     rendered = str(d)
     rendered = prune_rendered(rendered).strip()
     assert (
         rendered
         == """\
-FROM debian:buster-slim
+FROM debian:bullseye-slim
 ENTRYPOINT ["echo", "foo bar"]"""
     )

neurodocker/reproenv/tests/test_renderers_singularity.py

@@ -323,15 +323,15 @@ def test_singularity_render_from_instance_methods():
     )
 
     s = SingularityRenderer("apt")
-    s.from_("debian:buster-slim")
+    s.from_("debian:bullseye-slim")
     s.entrypoint(["echo", "foobar baz"])
     rendered = str(s)
     rendered = prune_rendered(rendered).strip()
     assert (
         rendered
         == """\
 Bootstrap: docker
-From: debian:buster-slim
+From: debian:bullseye-slim
 
 %post