From 216f36a107c0148fd91a272bcf1119941bdfaf90 Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Thu, 13 Jun 2024 11:38:24 +0000 Subject: [PATCH 01/16] Create "robot" dev container. --- .devcontainer/robot/Dockerfile | 29 ++++++++++++++++++++ .devcontainer/robot/devcontainer.json | 39 +++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 .devcontainer/robot/Dockerfile create mode 100644 .devcontainer/robot/devcontainer.json diff --git a/.devcontainer/robot/Dockerfile b/.devcontainer/robot/Dockerfile new file mode 100644 index 0000000..5fbb399 --- /dev/null +++ b/.devcontainer/robot/Dockerfile @@ -0,0 +1,29 @@ +# +# Dockerfile for *-robot development container +# +FROM ghcr.io/robotic-decision-making-lab/blue:jazzy-robot + +# Install ROS dependencies +# This is done in a previous stage, but we include it again here in case anyone wants to +# add new dependencies during development +ENV USERNAME=blue +ENV USER_WORKSPACE=/home/$USERNAME/ws_blue +WORKDIR $USER_WORKSPACE + +COPY --chown=$USER_UID:$USER_GID . src/blue +RUN sudo apt-get -q update \ + && sudo apt-get -q -y upgrade \ + && rosdep update \ + && rosdep install -y --from-paths src --ignore-src --rosdistro ${ROS_DISTRO} --skip-keys="gz-transport12 gz-sim7 gz-math7 gz-msgs9 gz-plugin2" \ + && sudo apt-get autoremove -y \ + && sudo apt-get clean -y \ + && sudo rm -rf /var/lib/apt/lists/* + +# Install debugging/linting Python packages +RUN /home/$USERNAME/.venv/blue/bin/python3 -m pip install \ + pre-commit \ + mypy + +# Disable the setuputils installation warning +# This prevents us from needing to pin the setuputils version (which doesn't always work) +ENV PYTHONWARNINGS="ignore" diff --git a/.devcontainer/robot/devcontainer.json b/.devcontainer/robot/devcontainer.json new file mode 100644 index 0000000..caef9a1 --- /dev/null +++ b/.devcontainer/robot/devcontainer.json @@ -0,0 +1,39 @@ +{ + "name": "Robot Dev Container", + "dockerFile": "Dockerfile", + "context": "../..", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/blue/ws_blue/src/blue,type=bind", + "workspaceFolder": "/home/blue/ws_blue/src/blue", + "remoteUser": "blue", + "runArgs": [ + "--network=host", + "--cap-add=SYS_PTRACE", + "--security-opt=seccomp:unconfined", + "--security-opt=apparmor:unconfined", + "--volume=/dev:/dev", + "--privileged", + "--volume=/run/user/1000:/run/user/1000" + ], + "containerEnv": { + "DISPLAY": "${localEnv:DISPLAY}", + "WAYLAND_DISPLAY": "${localEnv:WAYLAND_DISPLAY}", + "XDG_RUNTIME_DIR": "${localEnv:XDG_RUNTIME_DIR}", + "PULSE_SERVER": "${localEnv:PULSE_SERVER}" + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-azuretools.vscode-docker", + "ms-python.python", + "njpwerner.autodocstring", + "redhat.vscode-xml", + "redhat.vscode-yaml", + "smilerobotics.urdf", + "esbenp.prettier-vscode", + "charliermarsh.ruff", + "josetr.cmake-language-support-vscode", + "unifiedjs.vscode-mdx" + ] + } + } +} From 0374f6fc333593539a01c188586c6d0d5fe3d033 Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Thu, 13 Jun 2024 11:39:26 +0000 Subject: [PATCH 02/16] Build mavros from source. --- .docker/Dockerfile | 31 ++++++++++++++++--- .../patches/mavlink_standard_library.patch | 15 +++++++++ .dockerignore | 1 + 3 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 .docker/patches/mavlink_standard_library.patch diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 5a8289e..ae6c98f 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -18,6 +18,7 @@ RUN apt-get -q update \ clang-tools \ python3-pip \ python3-dev \ + python3-venv \ lsb-release \ wget \ gnupg \ @@ -27,10 +28,11 @@ RUN apt-get -q update \ && rm -rf /var/lib/apt/lists/* # Install all ROS dependencies +# Don't try to resolve "mavros" or "mavros_extras", we will install those manually RUN apt-get -q update \ && apt-get -q -y upgrade \ && rosdep update \ - && rosdep install -y --from-paths src --ignore-src --rosdistro ${ROS_DISTRO} --as-root=apt:false \ + && rosdep install -y --from-paths src --ignore-src --rosdistro ${ROS_DISTRO} --as-root=apt:false --skip-keys="mavros mavros_extras" \ && rm -rf src \ && apt-get autoremove -y \ && apt-get clean -y \ @@ -40,7 +42,7 @@ FROM ci as robot # Configure a new non-root user ARG USERNAME=blue -ARG USER_UID=1000 +ARG USER_UID=1001 # ros image now includes a user "ubuntu" at UID 1000 ARG USER_GID=$USER_UID RUN groupadd --gid $USER_GID $USERNAME \ @@ -56,8 +58,14 @@ ENV DEBIAN_FRONTEND=noninteractive USER $USERNAME ENV USER=$USERNAME -# Install MAVROS dependencies +# Given Ubuntu is now marked as a "Externally managed environment", +# Create venv for local python packages install by "$USERNAME" with pip +# Is there a more elegant way to handle this? WORKDIR /home/$USERNAME +RUN python3 -m venv --system-site-packages .venv/blue/ +RUN echo "source /home/$USERNAME/.venv/blue/bin/activate" >> /home/$USERNAME/.bashrc + +# Install MAVROS dependencies RUN wget https://raw.githubusercontent.com/mavlink/mavros/ros2/mavros/scripts/install_geographiclib_datasets.sh \ && chmod +x install_geographiclib_datasets.sh \ && sudo ./install_geographiclib_datasets.sh @@ -67,7 +75,12 @@ WORKDIR $USER_WORKSPACE COPY --chown=$USER_UID:$USER_GID . src/blue # Install the Python requirements that aren't available as rosdeps -RUN python3 -m pip install -r $(pwd)/src/blue/requirements-build.txt +RUN /home/$USERNAME/.venv/blue/bin/pip install -r $(pwd)/src/blue/requirements-build.txt + +# Running in a venv, I need to explicitly install some additional dependencies +# Using venv --system-packages seems to make this worse... +#RUN /home/$USERNAME/.venv/blue/bin/pip install future empy==3.3.4 catkin_pkg numpy lark lxml pytest + # Install gstreamer RUN sudo apt-get -q update \ @@ -85,10 +98,18 @@ RUN sudo apt-get -q update \ && sudo apt-get clean -y \ && sudo rm -rf /var/lib/apt/lists/* +# Manually install MAVROS source +WORKDIR $USER_WORKSPACE/src/ +RUN git clone --depth 1 -b ros2 https://github.com/mavlink/mavros.git +RUN git clone --depth 1 --recursive -b release/rolling/mavlink https://github.com/mavlink/mavlink-gbp-release.git mavlink +RUN ls -al $USER_WORKSPACE/src/blue/** +RUN patch -d mavlink -p 1 < $USER_WORKSPACE/src/blue/.docker/patches/mavlink_standard_library.patch +WORKDIR $USER_WORKSPACE + RUN sudo apt-get -q update \ && sudo apt-get -q -y upgrade \ && rosdep update \ - && rosdep install -y --from-paths src --ignore-src --rosdistro ${ROS_DISTRO} \ + && rosdep install -y --from-paths src --ignore-src --as-root=pip:false --rosdistro ${ROS_DISTRO} \ && sudo apt-get autoremove -y \ && sudo apt-get clean -y \ && sudo rm -rf /var/lib/apt/lists/* diff --git a/.docker/patches/mavlink_standard_library.patch b/.docker/patches/mavlink_standard_library.patch new file mode 100644 index 0000000..23937b5 --- /dev/null +++ b/.docker/patches/mavlink_standard_library.patch @@ -0,0 +1,15 @@ +diff --git a/pymavlink/generator/mavgen.py b/pymavlink/generator/mavgen.py +index ea62c961..35353184 100755 +--- a/pymavlink/generator/mavgen.py ++++ b/pymavlink/generator/mavgen.py +@@ -23,8 +23,8 @@ General process: + ''' + + from __future__ import print_function +-from future import standard_library +-standard_library.install_aliases() ++#from future import standard_library ++#standard_library.install_aliases() + from builtins import object + import os + import re diff --git a/.dockerignore b/.dockerignore index 211fad8..62be0f9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,4 +8,5 @@ !blue_demos !blue.repos !.docker/entrypoints +!.docker/patches !requirements-build.txt From f19a49b6537045e7e9609e42983a00ffbe70a97f Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Thu, 13 Jun 2024 11:39:50 +0000 Subject: [PATCH 03/16] Switch CI distro to "jazzy" --- .github/workflows/ci.yaml | 4 ++-- .github/workflows/docker.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8c56c7f..92b56c3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,8 +19,8 @@ jobs: fail-fast: false matrix: env: - - IMAGE: rolling-ci - ROS_DISTRO: rolling + - IMAGE: jazzy-ci + ROS_DISTRO: jazzy steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index c93452a..a00ed89 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - ROS_DISTRO: [rolling] + ROS_DISTRO: [jazzy] runs-on: ubuntu-latest permissions: packages: write @@ -63,7 +63,7 @@ jobs: strategy: fail-fast: false matrix: - ROS_DISTRO: [rolling] + ROS_DISTRO: [jazzy] runs-on: ubuntu-latest permissions: packages: write @@ -111,7 +111,7 @@ jobs: strategy: fail-fast: false matrix: - ROS_DISTRO: [rolling] + ROS_DISTRO: [jazzy] runs-on: ubuntu-latest permissions: packages: write @@ -152,7 +152,7 @@ jobs: strategy: fail-fast: false matrix: - ROS_DISTRO: [rolling] + ROS_DISTRO: [jazzy] runs-on: ubuntu-latest permissions: packages: write From b90ee8716e562e3516e5030c1bcc1cb69f2fc2a3 Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Thu, 13 Jun 2024 05:51:03 -0700 Subject: [PATCH 04/16] Run pre-commit. --- .docker/Dockerfile | 5 ----- .docker/patches/mavlink_standard_library.patch | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index ae6c98f..6d872a8 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -77,11 +77,6 @@ COPY --chown=$USER_UID:$USER_GID . src/blue # Install the Python requirements that aren't available as rosdeps RUN /home/$USERNAME/.venv/blue/bin/pip install -r $(pwd)/src/blue/requirements-build.txt -# Running in a venv, I need to explicitly install some additional dependencies -# Using venv --system-packages seems to make this worse... -#RUN /home/$USERNAME/.venv/blue/bin/pip install future empy==3.3.4 catkin_pkg numpy lark lxml pytest - - # Install gstreamer RUN sudo apt-get -q update \ && sudo apt-get -q -y upgrade \ diff --git a/.docker/patches/mavlink_standard_library.patch b/.docker/patches/mavlink_standard_library.patch index 23937b5..4f2360d 100644 --- a/.docker/patches/mavlink_standard_library.patch +++ b/.docker/patches/mavlink_standard_library.patch @@ -4,7 +4,7 @@ index ea62c961..35353184 100755 +++ b/pymavlink/generator/mavgen.py @@ -23,8 +23,8 @@ General process: ''' - + from __future__ import print_function -from future import standard_library -standard_library.install_aliases() From 08fc82152664d20aae05f1b207dc9dbeb5d4923c Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Mon, 17 Jun 2024 21:48:21 -0700 Subject: [PATCH 05/16] Switch to Gazebo "harmonic" --- .docker/Dockerfile | 17 ++++++++++++----- blue.repos | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 6d872a8..62a63fe 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -61,8 +61,13 @@ ENV USER=$USERNAME # Given Ubuntu is now marked as a "Externally managed environment", # Create venv for local python packages install by "$USERNAME" with pip # Is there a more elegant way to handle this? +# +# These two ENVs effectively "activate" the venv for all subsequent calls to python WORKDIR /home/$USERNAME -RUN python3 -m venv --system-site-packages .venv/blue/ +ENV VIRTUAL_ENV=/home/$USERNAME/.venv/blue +RUN python3 -m venv --system-site-packages $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + RUN echo "source /home/$USERNAME/.venv/blue/bin/activate" >> /home/$USERNAME/.bashrc # Install MAVROS dependencies @@ -75,7 +80,7 @@ WORKDIR $USER_WORKSPACE COPY --chown=$USER_UID:$USER_GID . src/blue # Install the Python requirements that aren't available as rosdeps -RUN /home/$USERNAME/.venv/blue/bin/pip install -r $(pwd)/src/blue/requirements-build.txt +RUN pip install -r $(pwd)/src/blue/requirements-build.txt # Install gstreamer RUN sudo apt-get -q update \ @@ -115,14 +120,14 @@ RUN echo "source ${USER_WORKSPACE}/install/setup.bash" >> /home/$USERNAME/.bashr FROM robot as desktop ENV DEBIAN_FRONTEND=noninteractive -ENV GZ_VERSION=garden +ENV GZ_VERSION=harmonic -# Install Gazebo Garden: https://gazebosim.org/docs/garden/install_ubuntu +# Install Gazebo Hamonic: https://gazebosim.org/docs/harmonic/install_ubuntu RUN sudo wget https://packages.osrfoundation.org/gazebo.gpg -O /usr/share/keyrings/pkgs-osrf-archive-keyring.gpg \ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/pkgs-osrf-archive-keyring.gpg] http://packages.osrfoundation.org/gazebo/ubuntu-stable $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/gazebo-stable.list > /dev/null \ && sudo apt-get -q update \ && sudo apt-get -y --quiet --no-install-recommends install \ - gz-garden \ + gz-${GZ_VERSION} \ && sudo apt-get autoremove -y \ && sudo apt-get clean -y \ && sudo rm -rf /var/lib/apt/lists/* @@ -131,6 +136,8 @@ RUN sudo wget https://packages.osrfoundation.org/gazebo.gpg -O /usr/share/keyrin RUN sudo apt-get -q update \ && sudo apt-get -q -y upgrade \ && sudo apt-get -q install --no-install-recommends -y \ + cppzmq-dev \ + python3-pexpect \ python3-wxgtk4.0 \ rapidjson-dev \ xterm \ diff --git a/blue.repos b/blue.repos index ca3abd9..626b08d 100644 --- a/blue.repos +++ b/blue.repos @@ -3,7 +3,7 @@ repositories: ros_gz: type: git url: https://github.com/gazebosim/ros_gz - version: ros2 + version: jazzy hydrodynamics: type: git From 8793a861c4be2adf3c29302a99c066ba11dbc57a Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Tue, 18 Jun 2024 14:38:18 -0700 Subject: [PATCH 06/16] Remove robot dir to clean MR --- .devcontainer/robot/Dockerfile | 29 -------------------- .devcontainer/robot/devcontainer.json | 39 --------------------------- 2 files changed, 68 deletions(-) delete mode 100644 .devcontainer/robot/Dockerfile delete mode 100644 .devcontainer/robot/devcontainer.json diff --git a/.devcontainer/robot/Dockerfile b/.devcontainer/robot/Dockerfile deleted file mode 100644 index 5fbb399..0000000 --- a/.devcontainer/robot/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# -# Dockerfile for *-robot development container -# -FROM ghcr.io/robotic-decision-making-lab/blue:jazzy-robot - -# Install ROS dependencies -# This is done in a previous stage, but we include it again here in case anyone wants to -# add new dependencies during development -ENV USERNAME=blue -ENV USER_WORKSPACE=/home/$USERNAME/ws_blue -WORKDIR $USER_WORKSPACE - -COPY --chown=$USER_UID:$USER_GID . src/blue -RUN sudo apt-get -q update \ - && sudo apt-get -q -y upgrade \ - && rosdep update \ - && rosdep install -y --from-paths src --ignore-src --rosdistro ${ROS_DISTRO} --skip-keys="gz-transport12 gz-sim7 gz-math7 gz-msgs9 gz-plugin2" \ - && sudo apt-get autoremove -y \ - && sudo apt-get clean -y \ - && sudo rm -rf /var/lib/apt/lists/* - -# Install debugging/linting Python packages -RUN /home/$USERNAME/.venv/blue/bin/python3 -m pip install \ - pre-commit \ - mypy - -# Disable the setuputils installation warning -# This prevents us from needing to pin the setuputils version (which doesn't always work) -ENV PYTHONWARNINGS="ignore" diff --git a/.devcontainer/robot/devcontainer.json b/.devcontainer/robot/devcontainer.json deleted file mode 100644 index caef9a1..0000000 --- a/.devcontainer/robot/devcontainer.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "Robot Dev Container", - "dockerFile": "Dockerfile", - "context": "../..", - "workspaceMount": "source=${localWorkspaceFolder},target=/home/blue/ws_blue/src/blue,type=bind", - "workspaceFolder": "/home/blue/ws_blue/src/blue", - "remoteUser": "blue", - "runArgs": [ - "--network=host", - "--cap-add=SYS_PTRACE", - "--security-opt=seccomp:unconfined", - "--security-opt=apparmor:unconfined", - "--volume=/dev:/dev", - "--privileged", - "--volume=/run/user/1000:/run/user/1000" - ], - "containerEnv": { - "DISPLAY": "${localEnv:DISPLAY}", - "WAYLAND_DISPLAY": "${localEnv:WAYLAND_DISPLAY}", - "XDG_RUNTIME_DIR": "${localEnv:XDG_RUNTIME_DIR}", - "PULSE_SERVER": "${localEnv:PULSE_SERVER}" - }, - "customizations": { - "vscode": { - "extensions": [ - "ms-azuretools.vscode-docker", - "ms-python.python", - "njpwerner.autodocstring", - "redhat.vscode-xml", - "redhat.vscode-yaml", - "smilerobotics.urdf", - "esbenp.prettier-vscode", - "charliermarsh.ruff", - "josetr.cmake-language-support-vscode", - "unifiedjs.vscode-mdx" - ] - } - } -} From 453f124b31235fa3b45bc133febf44752954b321 Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Tue, 18 Jun 2024 21:15:05 -0700 Subject: [PATCH 07/16] Use user "ubuntu" provided by upstream image. --- .docker/Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 62a63fe..1b6c88a 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -41,13 +41,13 @@ RUN apt-get -q update \ FROM ci as robot # Configure a new non-root user -ARG USERNAME=blue -ARG USER_UID=1001 # ros image now includes a user "ubuntu" at UID 1000 +# +# ros image now includes a user "ubuntu" at UID 1000 +ARG USERNAME=ubuntu +ARG USER_UID=1000 ARG USER_GID=$USER_UID -RUN groupadd --gid $USER_GID $USERNAME \ - && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ - && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ +RUN echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ && chmod 0440 /etc/sudoers.d/$USERNAME \ && usermod -a -G dialout $USERNAME \ && echo "source /usr/share/bash-completion/completions/git" >> /home/$USERNAME/.bashrc @@ -122,7 +122,7 @@ FROM robot as desktop ENV DEBIAN_FRONTEND=noninteractive ENV GZ_VERSION=harmonic -# Install Gazebo Hamonic: https://gazebosim.org/docs/harmonic/install_ubuntu +# Install Gazebo Harmonic: https://gazebosim.org/docs/harmonic/install_ubuntu RUN sudo wget https://packages.osrfoundation.org/gazebo.gpg -O /usr/share/keyrings/pkgs-osrf-archive-keyring.gpg \ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/pkgs-osrf-archive-keyring.gpg] http://packages.osrfoundation.org/gazebo/ubuntu-stable $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/gazebo-stable.list > /dev/null \ && sudo apt-get -q update \ From 5d50646fddc4eb909127beed082c4ad55afe7254 Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Tue, 18 Jun 2024 21:34:51 -0700 Subject: [PATCH 08/16] Minor updates to comments. --- .docker/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 1b6c88a..82c4dbb 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -58,11 +58,12 @@ ENV DEBIAN_FRONTEND=noninteractive USER $USERNAME ENV USER=$USERNAME -# Given Ubuntu is now marked as a "Externally managed environment", -# Create venv for local python packages install by "$USERNAME" with pip +# Python in Ubuntu is now marked as a "Externally managed environment", +# Per best practice, create a venv for local python packages # Is there a more elegant way to handle this? # -# These two ENVs effectively "activate" the venv for all subsequent calls to python +# These two ENVs effectively "activate" the venv for subsequent calls to +# python/pip in the Dockerfile WORKDIR /home/$USERNAME ENV VIRTUAL_ENV=/home/$USERNAME/.venv/blue RUN python3 -m venv --system-site-packages $VIRTUAL_ENV @@ -103,6 +104,7 @@ WORKDIR $USER_WORKSPACE/src/ RUN git clone --depth 1 -b ros2 https://github.com/mavlink/mavros.git RUN git clone --depth 1 --recursive -b release/rolling/mavlink https://github.com/mavlink/mavlink-gbp-release.git mavlink RUN ls -al $USER_WORKSPACE/src/blue/** +# Lines references in the patch caused import errors in Python3.12; comment them out RUN patch -d mavlink -p 1 < $USER_WORKSPACE/src/blue/.docker/patches/mavlink_standard_library.patch WORKDIR $USER_WORKSPACE From 0e9c698dc9dccb6613a27649d6bc99d185a08d8f Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Tue, 18 Jun 2024 21:36:08 -0700 Subject: [PATCH 09/16] Remove extraneous debug "ls" --- .docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 82c4dbb..a8c72ea 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -103,7 +103,6 @@ RUN sudo apt-get -q update \ WORKDIR $USER_WORKSPACE/src/ RUN git clone --depth 1 -b ros2 https://github.com/mavlink/mavros.git RUN git clone --depth 1 --recursive -b release/rolling/mavlink https://github.com/mavlink/mavlink-gbp-release.git mavlink -RUN ls -al $USER_WORKSPACE/src/blue/** # Lines references in the patch caused import errors in Python3.12; comment them out RUN patch -d mavlink -p 1 < $USER_WORKSPACE/src/blue/.docker/patches/mavlink_standard_library.patch WORKDIR $USER_WORKSPACE From e9f4a4e207171474cc0ec78e8fc3844366ef5fa0 Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Thu, 20 Jun 2024 11:39:10 -0700 Subject: [PATCH 10/16] Move install_geographiclib_datasets below mavros deps install. --- .docker/Dockerfile | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index a8c72ea..e25ec04 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -71,17 +71,12 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN echo "source /home/$USERNAME/.venv/blue/bin/activate" >> /home/$USERNAME/.bashrc -# Install MAVROS dependencies -RUN wget https://raw.githubusercontent.com/mavlink/mavros/ros2/mavros/scripts/install_geographiclib_datasets.sh \ - && chmod +x install_geographiclib_datasets.sh \ - && sudo ./install_geographiclib_datasets.sh - ENV USER_WORKSPACE=/home/$USERNAME/ws_blue WORKDIR $USER_WORKSPACE COPY --chown=$USER_UID:$USER_GID . src/blue # Install the Python requirements that aren't available as rosdeps -RUN pip install -r $(pwd)/src/blue/requirements-build.txt +RUN python3 -m pip install -r $(pwd)/src/blue/requirements-build.txt # Install gstreamer RUN sudo apt-get -q update \ @@ -99,11 +94,13 @@ RUN sudo apt-get -q update \ && sudo apt-get clean -y \ && sudo rm -rf /var/lib/apt/lists/* -# Manually install MAVROS source +# Manually install MAVROS from source in the ws_blue/ workspace WORKDIR $USER_WORKSPACE/src/ RUN git clone --depth 1 -b ros2 https://github.com/mavlink/mavros.git RUN git clone --depth 1 --recursive -b release/rolling/mavlink https://github.com/mavlink/mavlink-gbp-release.git mavlink -# Lines references in the patch caused import errors in Python3.12; comment them out + +# mavgen uses future.standard_library for backwards compatibility with Python2; +# However, this caused issues in Python3.12. Comment those lines out RUN patch -d mavlink -p 1 < $USER_WORKSPACE/src/blue/.docker/patches/mavlink_standard_library.patch WORKDIR $USER_WORKSPACE @@ -115,6 +112,13 @@ RUN sudo apt-get -q update \ && sudo apt-get clean -y \ && sudo rm -rf /var/lib/apt/lists/* +# Install geographiclib datasets +# Scripts uses geographiclib-tools which is a rosdep dependency of mavros installed above +WORKDIR /tmp +RUN wget https://raw.githubusercontent.com/mavlink/mavros/ros2/mavros/scripts/install_geographiclib_datasets.sh \ + && chmod +x /tmp/install_geographiclib_datasets.sh \ + && sudo /tmp/install_geographiclib_datasets.sh + RUN echo "source ${USER_WORKSPACE}/install/setup.bash" >> /home/$USERNAME/.bashrc \ && echo "source /opt/ros/${ROS_DISTRO}/setup.bash" >> /home/$USERNAME/.bashrc From 3afc1e741e3a3458c3d34d5ded3cd9b796c25e76 Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Thu, 20 Jun 2024 11:50:07 -0700 Subject: [PATCH 11/16] CI workflow uses image from this repo --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 92b56c3..9b37b72 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -37,7 +37,7 @@ jobs: - name: Run ROS Industrial CI uses: ros-industrial/industrial_ci@master env: - DOCKER_IMAGE: ghcr.io/robotic-decision-making-lab/blue:${{ matrix.env.IMAGE }} + DOCKER_IMAGE: ghcr.io/${{ github.repository_id }}:${{ matrix.env.IMAGE }} CXXFLAGS: >- -Wall -Wextra -Wpedantic -Wwrite-strings -Wunreachable-code -Wpointer-arith -Wredundant-decls CC: ${{ env.CLANG_TIDY && 'clang' }} From 5f155893f29e91cdbec29e55f5486888a81e4513 Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Thu, 20 Jun 2024 11:55:21 -0700 Subject: [PATCH 12/16] Use github.repository, not repository_id --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9b37b72..891d589 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -37,7 +37,7 @@ jobs: - name: Run ROS Industrial CI uses: ros-industrial/industrial_ci@master env: - DOCKER_IMAGE: ghcr.io/${{ github.repository_id }}:${{ matrix.env.IMAGE }} + DOCKER_IMAGE: ghcr.io/${{ github.repository }}:${{ matrix.env.IMAGE }} CXXFLAGS: >- -Wall -Wextra -Wpedantic -Wwrite-strings -Wunreachable-code -Wpointer-arith -Wredundant-decls CC: ${{ env.CLANG_TIDY && 'clang' }} From 933d2e68fe3b2c23b5b356b3211b6c67530060fc Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Fri, 21 Jun 2024 11:49:35 -0700 Subject: [PATCH 13/16] Update a few more users "blue" -> "ubuntu" --- .devcontainer/nouveau/Dockerfile | 2 +- .devcontainer/nouveau/devcontainer.json | 4 ++-- .devcontainer/nvidia/Dockerfile | 2 +- .devcontainer/nvidia/devcontainer.json | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.devcontainer/nouveau/Dockerfile b/.devcontainer/nouveau/Dockerfile index d362c8b..8122286 100644 --- a/.devcontainer/nouveau/Dockerfile +++ b/.devcontainer/nouveau/Dockerfile @@ -3,7 +3,7 @@ FROM ghcr.io/robotic-decision-making-lab/blue:rolling-desktop # Install ROS dependencies # This is done in a previous stage, but we include it again here in case anyone wants to # add new dependencies during development -ENV USERNAME=blue +ENV USERNAME=ubuntu ENV USER_WORKSPACE=/home/$USERNAME/ws_blue WORKDIR $USER_WORKSPACE diff --git a/.devcontainer/nouveau/devcontainer.json b/.devcontainer/nouveau/devcontainer.json index 5dbcef3..bae72b5 100644 --- a/.devcontainer/nouveau/devcontainer.json +++ b/.devcontainer/nouveau/devcontainer.json @@ -3,8 +3,8 @@ "dockerFile": "Dockerfile", "context": "../..", "workspaceMount": "source=${localWorkspaceFolder},target=/home/blue/ws_blue/src/blue,type=bind", - "workspaceFolder": "/home/blue/ws_blue/src/blue", - "remoteUser": "blue", + "workspaceFolder": "/home/ubuntu/ws_blue/src/blue", + "remoteUser": "ubuntu", "runArgs": [ "--network=host", "--cap-add=SYS_PTRACE", diff --git a/.devcontainer/nvidia/Dockerfile b/.devcontainer/nvidia/Dockerfile index 3135e7f..1c55005 100644 --- a/.devcontainer/nvidia/Dockerfile +++ b/.devcontainer/nvidia/Dockerfile @@ -3,7 +3,7 @@ FROM ghcr.io/robotic-decision-making-lab/blue:rolling-desktop-nvidia # Install ROS dependencies # This is done in a previous stage, but we include it again here in case anyone wants to # add new dependencies during development -ENV USERNAME=blue +ENV USERNAME=ubuntu ENV USER_WORKSPACE=/home/$USERNAME/ws_blue WORKDIR $USER_WORKSPACE diff --git a/.devcontainer/nvidia/devcontainer.json b/.devcontainer/nvidia/devcontainer.json index 9359454..e58a5ed 100644 --- a/.devcontainer/nvidia/devcontainer.json +++ b/.devcontainer/nvidia/devcontainer.json @@ -3,8 +3,8 @@ "dockerFile": "Dockerfile", "context": "../..", "workspaceMount": "source=${localWorkspaceFolder},target=/home/blue/ws_blue/src/blue,type=bind", - "workspaceFolder": "/home/blue/ws_blue/src/blue", - "remoteUser": "blue", + "workspaceFolder": "/home/ubuntu/ws_blue/src/blue", + "remoteUser": "ubuntu", "runArgs": [ "--network=host", "--cap-add=SYS_PTRACE", From 119ef0b2a76f5f2ef0c52be7c19e55b3b7ebb642 Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Mon, 24 Jun 2024 13:14:56 -0700 Subject: [PATCH 14/16] Make colcon play nice with venvs: !! Modify "#!" in /usr/bin/colcon to use /usr/bin/env - Switch order of ROS2 and venv activation --- .docker/Dockerfile | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index e25ec04..4070eba 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -54,6 +54,16 @@ RUN echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ ENV DEBIAN_FRONTEND=noninteractive +# This is ... a very bad way to solve this ... +# but it appears to be the most effective way to make colcon play +# nice with venvs +# +# Changes the first line of /usr/bin/colcon to use +# the python3 binary in the venv (if activated) +# +# #!/usr/bin/python3 --> #!/usr/bin/env python3 +RUN sed -i '1s/python3/env python3/' /usr/bin/colcon + # Switch to the non-root user for the rest of the installation USER $USERNAME ENV USER=$USERNAME @@ -66,9 +76,11 @@ ENV USER=$USERNAME # python/pip in the Dockerfile WORKDIR /home/$USERNAME ENV VIRTUAL_ENV=/home/$USERNAME/.venv/blue -RUN python3 -m venv --system-site-packages $VIRTUAL_ENV +RUN python3 -m venv --system-site-packages --symlinks $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# ROS must be activated before the venv (?) +RUN echo "source /opt/ros/${ROS_DISTRO}/setup.bash" >> /home/$USERNAME/.bashrc RUN echo "source /home/$USERNAME/.venv/blue/bin/activate" >> /home/$USERNAME/.bashrc ENV USER_WORKSPACE=/home/$USERNAME/ws_blue @@ -119,8 +131,7 @@ RUN wget https://raw.githubusercontent.com/mavlink/mavros/ros2/mavros/scripts/in && chmod +x /tmp/install_geographiclib_datasets.sh \ && sudo /tmp/install_geographiclib_datasets.sh -RUN echo "source ${USER_WORKSPACE}/install/setup.bash" >> /home/$USERNAME/.bashrc \ - && echo "source /opt/ros/${ROS_DISTRO}/setup.bash" >> /home/$USERNAME/.bashrc +RUN echo "if [ -f ${USER_WORKSPACE}/install/setup.bash ]; then source ${USER_WORKSPACE}/install/setup.bash; fi" >> /home/$USERNAME/.bashrc FROM robot as desktop From 1f81b7cab1ad8af5d89108797c9415300f9aa3fb Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Tue, 25 Jun 2024 11:58:28 -0700 Subject: [PATCH 15/16] Explicitly install libgstreamer1.0-dev and libgstreamer-plugins-base1.0-dev --- .docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 4070eba..40bf35e 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -102,6 +102,8 @@ RUN sudo apt-get -q update \ gstreamer1.0-plugins-ugly \ gstreamer1.0-plugins-bad \ gstreamer1.0-libav \ + libgstreamer1.0-dev \ + libgstreamer-plugins-base1.0-dev \ && sudo apt-get autoremove -y \ && sudo apt-get clean -y \ && sudo rm -rf /var/lib/apt/lists/* From 99c5f2673006b2a67b829edf7df1c0ece25f341b Mon Sep 17 00:00:00 2001 From: Aaron Marburg Date: Tue, 25 Jun 2024 13:50:36 -0700 Subject: [PATCH 16/16] Don't modify /usr/bin/colcon, use an alias instead. --- .docker/Dockerfile | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 40bf35e..034eb66 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -54,16 +54,6 @@ RUN echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ ENV DEBIAN_FRONTEND=noninteractive -# This is ... a very bad way to solve this ... -# but it appears to be the most effective way to make colcon play -# nice with venvs -# -# Changes the first line of /usr/bin/colcon to use -# the python3 binary in the venv (if activated) -# -# #!/usr/bin/python3 --> #!/usr/bin/env python3 -RUN sed -i '1s/python3/env python3/' /usr/bin/colcon - # Switch to the non-root user for the rest of the installation USER $USERNAME ENV USER=$USERNAME @@ -82,6 +72,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" # ROS must be activated before the venv (?) RUN echo "source /opt/ros/${ROS_DISTRO}/setup.bash" >> /home/$USERNAME/.bashrc RUN echo "source /home/$USERNAME/.venv/blue/bin/activate" >> /home/$USERNAME/.bashrc +RUN echo "\n# Ensure colcon is run in the venv\nalias colcon='python3 -m colcon'" >> /home/$USERNAME/.bashrc ENV USER_WORKSPACE=/home/$USERNAME/ws_blue WORKDIR $USER_WORKSPACE