Merge branch 'main' into dependabot/pip/rich-gte-13.6.0-and-lt-14.1

Signed-off-by: Steven Elliott <[email protected]>

Author: sdelliot

.github/actions/prepare-discovery/action.yml

@@ -9,7 +9,7 @@ runs:
   steps:
     - name: Install discovery
       run: |
-        wget https://github.com/mitchnegus/minimega-discovery/releases/download/firewheel-debian_faed761/discovery.deb
+        wget https://github.com/sandia-minimega/discovery/releases/download/v0.1.0/discovery.deb
         sudo dpkg -i discovery.deb
         sudo chown -R $USER:minimega /opt/discovery
       shell: bash

.gitlab-ci.yml

@@ -5,102 +5,10 @@
 # on a variety of operating systems and versions of
 # Python.
 #
-# It makes heavy use of YAML Anchors and extends to save space.
-# see: https://docs.gitlab.com/ce/ci/yaml/README.html#anchors
-#
 ###############################
 
-
-###############################
-# Setting up before_script anchors to speed up the execution of
-# pipeline jobs which do not require each component.
-# See: https://docs.gitlab.com/ee/ci/yaml/#yaml-anchors-for-before_script-and-after_script
-#
-# This includes:
-# * create_venv_docker: Creating a Python 3.10 virtual environment
-# * install_firewheel: Install FIREWHEEL and any dependencies.
-# * configure_firewheel: Configure FIREWHEEL for use by the job.
-# * install_mcs: Clone the base and linux Model Component repositories.
-###############################
-.create_venv_docker: &create_venv_docker
-  - python --version ; pip --version
-  - pip install $PIP_ARGS virtualenv
-  - virtualenv venv
-  - source venv/bin/activate
-
-.install_firewheel_docker: &install_firewheel_docker
-  - python -m pip install $PIP_ARGS -U build setuptools pip
-  - python -m build
-  - python -m pip install -e .[dev]
-
-.install_firewheel: &install_firewheel
-  - sudo killall minimega  # Make sure minimega is not running
-  - sudo systemctl restart minimega
-  - *install_firewheel_docker
-
-.set_config: &set_config
-  - firewheel config set -s system.default_output_dir "$(pwd)/logs"
-  - firewheel config set -s cluster.compute $(hostname)
-  - firewheel config set -s cluster.control $(hostname)
-  - firewheel config set -s discovery.hostname 127.0.0.1
-  - firewheel config set -s grpc.hostname 127.0.0.1
-  - firewheel config set -s minimega.experiment_interface lo
-  - firewheel config set -s minimega.install_dir /opt/minimega
-  - firewheel config set -s python.venv $(pwd)/venv
-  - firewheel config set -s python.bin python3
-
-.configure_firewheel_docker: &configure_firewheel_docker
-  - *set_config
-  - firewheel config set -s system.default_group root
-  - python ./src/firewheel/lib/grpc/firewheel_grpc_server.py &
-
-.configure_firewheel: &configure_firewheel
-  - *set_config
-  - firewheel init
-  - firewheel sync
-  - firewheel restart hard
-
-.install_mcs_docker: &install_mcs_docker
-  - mkdir model_components
-  - pip install $PIP_ARGS firewheel-repo-base firewheel-repo-tutorials
-  - pushd model_components
-  - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/firewheel_repo_linux.git
-  - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/firewheel_repo_utilities.git
-  - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/firewheel_repo_layer2.git
-  - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/firewheel_repo_ntp.git
-  - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/firewheel_repo_dns.git
-  - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/firewheel_repo_vyos.git
-  - firewheel repository install linux
-  - firewheel repository install utilities
-  - firewheel repository install layer2
-  - firewheel repository install ntp
-  - firewheel repository install dns
-  - firewheel repository install vyos
-  - popd
-
-
-.install_mcs: &install_mcs
-  - pushd /opt/firewheel/
-  - *install_mcs_docker
-  - pushd model_components
-  - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${NTP_BIN_REPO_PATH} /tmp/ntp
-  - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${DNS_BIN_REPO_PATH} /tmp/dns
-  - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${VYOS_BIN_REPO_PATH} /tmp/vyos
-  - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${LINUX_BIN_REPO_PATH} /tmp/linux
-  - mv /tmp/ntp/ntp/ntp-trusty-server.tar /opt/firewheel/model_components/firewheel_repo_ntp/src/firewheel_repo_ntp/ntp/vm_resources/
-  - mv /tmp/dns/dns_objects/vm_resources/bind9_xenial_debs.tgz /opt/firewheel/model_components/firewheel_repo_dns/src/firewheel_repo_dns/dns_objects/vm_resources/bind9_xenial_debs.tgz
-  - mv /tmp/vyos/vyos-1.1.8/vyos-1.1.8.qc2.xz /opt/firewheel/model_components/firewheel_repo_vyos/src/firewheel_repo_vyos/vyos-1.1.8/
-  - mv /tmp/linux/linux/vm_resources/*.tgz /opt/firewheel/model_components/firewheel_repo_linux/src/firewheel_repo_linux/linux/vm_resources
-  - mv /tmp/linux/ubuntu/ubuntu/vm_resources/debs/*.tgz /opt/firewheel/model_components/firewheel_repo_linux/src/firewheel_repo_linux/ubuntu/ubuntu/vm_resources/debs/
-  - mv /tmp/linux/ubuntu/bionic/images/ubuntu* /opt/firewheel/model_components/firewheel_repo_linux/src/firewheel_repo_linux/ubuntu/bionic/images/
-  - mv /tmp/linux/ubuntu/jammy/images/ubuntu* /opt/firewheel/model_components/firewheel_repo_linux/src/firewheel_repo_linux/ubuntu/jammy/images/
-  - mv /tmp/linux/ubuntu/jammy/vm_resources/debs/*.tgz /opt/firewheel/model_components/firewheel_repo_linux/src/firewheel_repo_linux/ubuntu/jammy/vm_resources/debs/
-  - mv /tmp/linux/ubuntu/trusty/images/ubuntu* /opt/firewheel/model_components/firewheel_repo_linux/src/firewheel_repo_linux/ubuntu/trusty/images/
-  - mv /tmp/linux/ubuntu/trusty/vm_resources/debs/*.tgz /opt/firewheel/model_components/firewheel_repo_linux/src/firewheel_repo_linux/ubuntu/trusty/vm_resources/debs/
-  - mv /tmp/linux/ubuntu/xenial/images/ubuntu* /opt/firewheel/model_components/firewheel_repo_linux/src/firewheel_repo_linux/ubuntu/xenial/images/
-  - popd
-  - popd
-
+include:
+  - local: .gitlab/ci/setup.yml
 
 ###############################
 # Creating a few defaults and setting up the Pipeline stages.
@@ -123,18 +31,18 @@ stages:
 ###############################
 
 performance-ubuntu2204:
-  before_script:
-    - pushd /opt/firewheel/
-    - python3.10 -m venv venv
-    - source /opt/firewheel/venv/bin/activate
-    - popd
-    - *install_firewheel
-    - *configure_firewheel
-    - *install_mcs
   stage: test
-  needs: []
   tags:
     - ubuntu2204
+  needs: []
+  variables:
+    BASE_PYTHON: python3.10
+  before_script:
+    - !reference [.create_venv, before_script]
+    - !reference [.install_firewheel, before_script]
+    - !reference [.configure_firewheel, before_script]
+    - !reference [.install_mcs, before_script]
+    - !reference [.provision_mcs, before_script]
   script:
     - pushd /opt/firewheel
     # Enable FW to cache images
@@ -163,19 +71,18 @@ performance-ubuntu2204:
     paths:
       - profile
 
-
 ###############################
 # Create Documentation
 ###############################
 docs:
   image: $DOCKER_REGISTRY/python:3.11
-  before_script:
-    - *create_venv_docker
-    - *install_firewheel_docker
-    - *configure_firewheel_docker
-    - *install_mcs_docker
   stage: lint
   needs: []
+  before_script:
+    - !reference [.create_venv, before_script]
+    - !reference [.install_firewheel_package, before_script]
+    - !reference [.configure_firewheel_docker, before_script]
+    - !reference [.install_mcs, before_script]
   script:
     - tox -e dependencies,docs
     - mkdir documentation
@@ -191,7 +98,6 @@ docs:
   tags:
     - $CI_DOCKER_RUNNER
 
-
 pages:
   image: $DOCKER_REGISTRY/python:3.11
   before_script:
@@ -214,12 +120,8 @@ build:
   image: $DOCKER_REGISTRY/python:3.11
   stage: deploy
   before_script:
-    - python --version ; pip --version
     - apt-get update && apt-get install -y git git-lfs
-    - git lfs install
-    - pip install virtualenv
-    - virtualenv venv
-    - source venv/bin/activate
+    - !reference [.create_venv, before_script]
     - python -m pip install $PIP_ARGS --upgrade wheel setuptools pip build twine
   script:
     - git lfs pull

.gitlab/ci/setup.yml

@@ -0,0 +1,103 @@
+###########################################################
+# Setup for FIREWHEEL Continuous Integration
+#
+# Use reference tags to deduplicate CI functionality
+# and speed up the execution of pipeline jobs which do
+# not require each component.
+#
+# See: https://docs.gitlab.com/ci/yaml/yaml_optimization
+#
+###########################################################
+
+variables:
+  BASE_PYTHON: python
+  MM_INSTALL_DIR: /opt/minimega
+  FIREWHEEL_ROOT_DIR: /opt/firewheel
+  FIREWHEEL_VENV: $FIREWHEEL_ROOT_DIR/venv
+
+# Create a Python virtual environment
+.create_venv:
+  before_script:
+    - $BASE_PYTHON --version
+    - $BASE_PYTHON -m pip --version
+    - $BASE_PYTHON -m venv $FIREWHEEL_VENV
+    - source $FIREWHEEL_VENV/bin/activate
+
+# Install the FIREWHEEL Python package and any Python dependencies
+.install_firewheel_package:
+  before_script:
+    - python -m pip install $PIP_ARGS -U build setuptools pip
+    - python -m build
+    - python -m pip install -e .[dev,mcs]
+
+# Install FIREWHEEL and all dependencies (including minimega)
+.install_firewheel:
+  before_script:
+    - sudo killall minimega  # Make sure minimega is not running
+    - sudo systemctl restart minimega
+    - !reference [.install_firewheel_package, before_script]
+
+# Set the default FIREWHEEL configuration
+.configure_firewheel:
+  before_script:
+    - firewheel config set -s system.default_output_dir "$(pwd)/logs"
+    - firewheel config set -s cluster.compute $(hostname)
+    - firewheel config set -s cluster.control $(hostname)
+    - firewheel config set -s discovery.hostname 127.0.0.1
+    - firewheel config set -s grpc.hostname 127.0.0.1
+    - firewheel config set -s minimega.experiment_interface lo
+    - firewheel config set -s minimega.install_dir $MM_INSTALL_DIR
+    - firewheel config set -s python.venv $FIREWHEEL_VENV
+    - firewheel config set -s python.bin python
+
+# Configure FIREWHEEL for use in Docker containers
+.configure_firewheel_docker:
+  before_script:
+    - !reference [.configure_firewheel, before_script]
+    - firewheel config set -s system.default_group root  # Docker
+    - python ./src/firewheel/lib/grpc/firewheel_grpc_server.py &
+
+# Initialize FIREWHEEL (including configuration)
+.initialize_firewheel:
+  before_script:
+    - !reference [.configure_firewheel, before_script]
+    - firewheel init
+    - firewheel sync
+    - firewheel restart hard
+
+# Clone and/or install basic Model Component repositories
+.install_mcs:
+  before_script:
+    - |
+      pip install $PIP_ARGS firewheel-repo-base firewheel-repo-tutorials \
+                            firewheel-repo-linux firewheel-repo-vyos \
+                            firewheel-repo-layer2 firewheel-repo-ntp firewheel-repo-dns
+    - mkdir $FIREWHEEL_ROOT_DIR/model_components
+    - pushd $FIREWHEEL_ROOT_DIR/model_components
+    # Skip LFS files, we'll get them from the bin repo. Once open-source is
+    # complete, the env var can be removed and this can be pip installed
+    - GIT_LFS_SKIP_SMUDGE=1 git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/firewheel/firewheel_repo_utilities.git
+    - firewheel repository install firewheel_repo_utilities
+    - popd
+
+# Move MC binaries into the appropriate locations
+.provision_mcs:
+  before_script:
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${NTP_BIN_REPO_PATH} /tmp/ntp
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${DNS_BIN_REPO_PATH} /tmp/dns
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${VYOS_BIN_REPO_PATH} /tmp/vyos
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${LINUX_BIN_REPO_PATH} /tmp/linux
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${UTILITIES_BIN_REPO_PATH} /tmp/utilities
+    - SITE_PACKAGES=$(python3 -c 'import sysconfig; print(sysconfig.get_path("purelib"))')
+    - mv /tmp/ntp/ntp/ntp-trusty-server.tar "$SITE_PACKAGES/firewheel_repo_ntp/ntp/vm_resources/"
+    - mv /tmp/dns/dns_objects/vm_resources/bind9_xenial_debs.tgz "$SITE_PACKAGES/firewheel_repo_dns/dns_objects/vm_resources/bind9_xenial_debs.tgz"
+    - mv /tmp/vyos/vyos-1.1.8/vyos-1.1.8.qc2.xz "$SITE_PACKAGES/firewheel_repo_vyos/vyos-1.1.8/"
+    - mv /tmp/linux/linux/vm_resources/*.tgz "$SITE_PACKAGES/firewheel_repo_linux/linux/vm_resources"
+    - mv /tmp/linux/ubuntu/ubuntu/vm_resources/debs/*.tgz "$SITE_PACKAGES/firewheel_repo_linux/ubuntu/ubuntu/vm_resources/debs/"
+    - mv /tmp/linux/ubuntu/bionic/images/ubuntu* "$SITE_PACKAGES/firewheel_repo_linux/ubuntu/bionic/images/"
+    - mv /tmp/linux/ubuntu/jammy/images/ubuntu* "$SITE_PACKAGES/firewheel_repo_linux/ubuntu/jammy/images/"
+    - mv /tmp/linux/ubuntu/jammy/vm_resources/debs/*.tgz "$SITE_PACKAGES/firewheel_repo_linux/ubuntu/jammy/vm_resources/debs/"
+    - mv /tmp/linux/ubuntu/trusty/images/ubuntu* "$SITE_PACKAGES/firewheel_repo_linux/ubuntu/trusty/images/"
+    - mv /tmp/linux/ubuntu/trusty/vm_resources/debs/*.tgz "$SITE_PACKAGES/firewheel_repo_linux/ubuntu/trusty/vm_resources/debs/"
+    - mv /tmp/linux/ubuntu/xenial/images/ubuntu* "$SITE_PACKAGES/firewheel_repo_linux/ubuntu/xenial/images/"
+    - pushd /tmp/utilities; cp -r --parents * $FIREWHEEL_ROOT_DIR/model_components/firewheel_repo_utilities/; popd;

docker/README.md

@@ -89,16 +89,14 @@ To enable FIREWHEEL to work properly (without any modifications) the docker base
 These include:
 - `sudo` - and specifically `sudo systemctl`
 - `chgrp`
-- `scp`
-- `ssh`
 
 Therefore, we have created wrappers around these programs to adjust the behavior.
-However, the original programs have been kept/renamed to `<progname>-old` (e.g., `ssh-old`).
+However, the original programs have been kept/renamed to `<progname>-old` (e.g., `chgrp-old`).
 Therefore, if these programs are needed, they are still available, but will not be used by FIREWHEEL.
 
 ### Minimega Differences
 The minimega container provides a [`start-minimega.sh`](https://github.com/sandia-minimega/minimega/blob/master/docker/start-minimega.sh) script which is overwritten by a customized version in the FIREWHEEL container.
 The primary difference between these scripts is the addition of logging various errors and exiting early if minimega is already running.
 
 Additionally, we include a minimega environment variables configuration file (described above) which can be used/changed as needed.
-Within the container it is located at `/etc/default/minimega`.
+Within the container it is located at `/etc/default/minimega`.

docker/firewheel.dockerfile

@@ -18,7 +18,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
                         ethtool libpcap-dev openvswitch-switch qemu-kvm qemu-utils \
                         dnsmasq ntfs-3g iproute2 qemu-system-x86 software-properties-common \
                         dosfstools openssh-server locales locales-all python3.10 python3.10-venv \
-                        vim psmisc && \
+                        vim psmisc iputils-ping && \
     apt-get clean && rm -rf /var/lib/apt/lists/* || { echo "Package installation failed"; exit 1; }
 
 ### Locale support ###
@@ -82,15 +82,11 @@ RUN bash -c "source /fwpy/bin/activate  && \
 
 RUN firewheel repository install -s -i || { echo "Repository installation failed"; exit 1; }
 
-RUN cp /usr/bin/ssh /usr/bin/ssh-old && \
-    cp /usr/bin/scp /usr/bin/scp-old && \
-    cp /usr/bin/sudo /usr/bin/sudo-old && \
+RUN cp /usr/bin/sudo /usr/bin/sudo-old && \
     cp /usr/bin/chgrp /usr/bin/chgrp-old
 
 COPY docker/fsroot/ /
 RUN chmod +x /usr/local/bin/entry && \
-    chmod +x /usr/bin/ssh && \
-    chmod +x /usr/bin/scp && \
     chmod +x /usr/bin/sudo && \
     chmod +x /usr/bin/chgrp && \
     chmod +x /start-minimega.sh