Add CI/testing pipelines + format code

(Tutorial) Micro Autonomy Testing Pipelines.md

@@ -0,0 +1,100 @@
+## Using and Writing Tests for Micro Autonomy Pipelines
+
+Some GitHub workflows have been set up in this repo to run tests on any PRs against main. It checks for **code style, if it compiles properly, and any unit tests** you may want to add. These are nearly identical to the ones in the [monorepo](https://github.com/WATonomous/wato_monorepo), so you can also find more examples there.
+
+After making a PR, check the `Actions` panel in the repo. You should be able to find the test results run on your branch.
+
+### Using the Auto-Linter
+To use the auto-linter on a PR against main, add the green `auto-lint` label on the sidebar. The WATonomous bot will add a commit to reformat your code.
+
+### Adding Unit Tests
+
+#### 1. To add a test for a ROS node, make a `test` folder in the node with a testing class.
+
+Example structure with `gym_vis` node:
+
+```
+└── gym_vis
+    ├── assets
+    │   └── f1tenth_car.stl
+    ├── CMakeLists.txt
+    ├── include
+    │   └── gym_vis.hpp
+    ├── LICENSE
+    ├── package.xml
+    ├── src
+    │   └── gym_vis.cpp
+    └── test
+        └── test_gym_vis.cpp
+```
+
+#### 2. Write your tests in `test_{node_name}.cpp`. Make sure to import the `#include "gtest/gtest.h"` library for testing.
+
+It should look something like this.
+
+```cpp
+#include "gtest/gtest.h"
+#include <rclcpp/rclcpp.hpp>
+#include "gym_vis.hpp"
+
+TEST(GymVisTest, MakeNode) {
+    rclcpp::init(0, nullptr);
+    auto node = std::make_shared<GymVis>();
+    EXPECT_EQ(node->get_name(), std::string("gym_vis"));
+    rclcpp::shutdown();
+}
+
+TEST(GymVisTest, NodeExists) {
+    rclcpp::init(0, nullptr);
+    auto node = std::make_shared<GymVis>();
+    EXPECT_TRUE(node != nullptr);
+    rclcpp::shutdown();
+}
+// you can start by adding this test to make sure it runs the test as expected
+TEST(GymVisTest, TestTheTestSuite) { EXPECT_TRUE(true); }
+```
+
+#### 3. Add any dependencies you need in `CMakeLists.txt`
+```cmake
+if(BUILD_TESTING)
+  find_package(ament_cmake_gtest REQUIRED)
+
+  ament_add_gtest(${PROJECT_NAME}_test
+    test/test_gym_vis.cpp
+    # src/gym_vis.cpp // example
+  )
+  target_include_directories(${PROJECT_NAME}_test PUBLIC include)
+  # add your dependencies. ex:
+  # ament_target_dependencies(${PROJECT_NAME}_test rclcpp std_msgs nav_msgs visualization_msgs sensor_msgs ackermann_msgs tf2_ros tf2_geometry_msgs)
+endif()
+```
+
+### 4. Similarly, add dependencies in `package.xml`
+
+Namely, `<test_depend>ament_cmake_gtest</test_depend>`
+
+### Last thing,
+The build/testing pipeline is especially slow on GitHub so it's a good idea to run it locally first.
+
+It's something like:
+```bash
+./watod build
+./watod up
+```
+Build find, and enter your docker container (looks like: ...robot-dev...)
+```bash
+docker ps
+docker exec -it {container-id} /bin/bash
+source /opt/ros/humble/setup.bash
+```
+Build your node
+```bash
+cd {into your node}
+colcon build --symlink-install --cmake-args -DCMAKE_BUILD_TYPE=Release --packages-select {your_ros_node}
+```
+
+Run your tests
+```bash
+colcon test --packages-select {node_name}
+colcon test-result --verbose
+```

.github/templates/check_src_changes/action.yml

@@ -0,0 +1,77 @@
+name: Check source file changes
+
+outputs:
+  modified_modules:
+    description: "Space deliminated list of modified modules"
+    value: ${{ steps.output-changes.outputs.modified_modules }}
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Find changed files outside of src directory
+      id: changed-files-outside-src
+      uses: tj-actions/changed-files@v42
+      with:
+        files: '!src/**'
+
+    - name: Find changed files inside src/wato_msgs folder
+      id: changed-files-wato-msgs
+      uses: tj-actions/changed-files@v42
+      with:
+        files: src/wato_msgs/**
+
+    - name: Find changed files inside src/action folder
+      id: changed-files-action
+      uses: tj-actions/changed-files@v42
+      with:
+        files: src/action/**
+
+    - name: Get changed files inside src/interfacing folder
+      id: changed-files-interfacing
+      uses: tj-actions/changed-files@v42
+      with:
+        files: src/interfacing/**
+
+    - name: Get changed files inside src/perception folder
+      id: changed-files-perception
+      uses: tj-actions/changed-files@v42
+      with:
+        files: src/perception/**
+
+    - name: Get changed files inside src/samples folder
+      id: changed-files-samples
+      uses: tj-actions/changed-files@v42
+      with:
+        files: src/samples/**
+
+    - name: Get changed files inside src/simulation folder
+      id: changed-files-simulation
+      uses: tj-actions/changed-files@v42
+      with:
+        files: src/simulation/**
+
+    - name: Get changed files inside src/world_modeling folder
+      id: changed-files-world-modeling
+      uses: tj-actions/changed-files@v42
+      with:
+        files: src/world_modeling/**
+
+    - name: Create list of changed modules
+      id: output-changes
+      env:
+        INFRASTRUCTURE_CHANGED: >
+          ${{ steps.changed-files-wato-msgs.outputs.any_changed == 'true' 
+          || steps.changed-files-outside-src.outputs.any_changed == 'true' }}
+        ACTION_CHANGED: ${{ steps.changed-files-action.outputs.any_changed }}
+        INTERFACING_CHANGED: ${{ steps.changed-files-interfacing.outputs.any_changed }}
+        PERCEPTION_CHANGED: ${{ steps.changed-files-perception.outputs.any_changed }}
+        SAMPLES_CHANGED: ${{ steps.changed-files-samples.outputs.any_changed }}
+        SIMULATION_CHANGED: ${{ steps.changed-files-simulation.outputs.any_changed }}
+        WORLD_MODELING_CHANGED: ${{ steps.changed-files-world-modeling.outputs.any_changed }}
+
+      run: ${{ github.action_path }}/check_src_changes.sh
+      shell: bash

.github/templates/check_src_changes/check_src_changes.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -e
+
+################# Create a space delimited list of modified modules #################
+# Outputs a list of modified modules by comparing changes between main and current commit
+# References previous GitHub workflow steps
+
+# Action
+if [ $ACTION_CHANGED == 'true' ]; then
+    echo "Detected action changes"
+    MODIFIED_MODULES+="action "
+fi
+
+# Interfacing
+if [ $INTERFACING_CHANGED == 'true' ]; then
+    echo "Detected interfacing changes"
+    MODIFIED_MODULES+="interfacing "
+fi
+
+# Perception
+if [ $PERCEPTION_CHANGED == 'true' ]; then
+    echo "Detected perception changes"
+    MODIFIED_MODULES+="perception "
+fi
+
+# Samples
+if [ $SAMPLES_CHANGED == 'true' ]; then
+    echo "Detected samples changes"
+    MODIFIED_MODULES+="samples "
+fi
+
+# Simulation
+if [ $SIMULATION_CHANGED == 'true' ]; then
+    echo "Detected simulation changes"
+    MODIFIED_MODULES+="simulation "
+fi
+
+# World-modeling
+if [ $WORLD_MODELING_CHANGED == 'true' ]; then
+    echo "Detected world_modeling changes"
+    MODIFIED_MODULES+="world_modeling"
+fi
+
+# Infrastructure
+if [ $INFRASTRUCTURE_CHANGED == 'true' ]; then
+    echo "::notice:: Detected infrastructure changes"
+    MODIFIED_MODULES="infrastructure"
+else
+    echo "::notice:: MODIFIED_MODULES are $MODIFIED_MODULES" 
+fi
+
+# Output lis
+echo "modified_modules=$MODIFIED_MODULES" >> $GITHUB_OUTPUT

.github/templates/docker_context/action.yml

@@ -0,0 +1,27 @@
+name: Generate Docker Environment
+
+inputs:
+  modified_modules:
+    description: "Space deliminated list of modified modules"
+    required: true
+    default: ''
+
+outputs:
+  docker_matrix:
+    description: "list of docker compose services"
+    value: ${{ steps.environment-generator.outputs.docker_matrix }}
+  registry:
+    description: "name of the docker registry we are using"
+    value: ${{ steps.environment-generator.outputs.registry }}
+  repository:
+    description: "name of the docker repository we are using"
+    value: ${{ steps.environment-generator.outputs.repository }}
+
+runs:
+  using: "composite"
+  steps:
+    - id: environment-generator
+      env: 
+        MODIFIED_MODULES: ${{ inputs.modified_modules }}
+      run: ${{ github.action_path }}/docker_context.sh
+      shell: bash

.github/templates/docker_context/docker_context.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -e
+
+################# Sweep for Docker Services and Modules #################
+# Scans for services and modules in the wato_monorepo,
+# dynamically builds a json matrix for downstream CI build and testing
+
+# Find docker compose files in 'modules' directory
+modules=$(find modules -maxdepth 1 -name "docker-compose*")
+
+# Initialize an empty array for JSON objects
+json_objects=()
+
+# Check for infrastructure changes
+TEST_ALL=false
+if [[ $MODIFIED_MODULES = "infrastructure" ]]; then
+    TEST_ALL=true
+fi
+
+# Loop through each module
+while read -r module; do
+
+    # Retrieve docker compose service names
+    services=$(docker compose -f "$module" config --services)
+    module_out=$(echo "$module" | sed -n 's/modules\/docker-compose\.\(.*\)\.yaml/\1/p')
+
+    # Skip simulation module
+    if [[ 'simulation' = $module_out ]]; then
+        continue
+    fi
+
+    # Only work with modules that are modified
+    if [[ $MODIFIED_MODULES != *$module_out* && $TEST_ALL = "false" ]]; then
+        continue
+    fi
+
+    # Loop through each service
+    while read -r service_out; do
+        # Temporarily skip perception services that have too large image size
+        if  [[ "$service_out" == "lane_detection" ]] || \
+            [[ "$service_out" == "camera_object_detection" ]] || \
+            [[ "$service_out" == "lidar_object_detection" ]] || \
+            [[ "$service_out" == "semantic_segmentation" ]]; then
+            continue
+        fi
+        # Construct JSON object for each service with module and service name
+        json_object=$(jq -nc --arg module_out "$module_out" --arg service_out "$service_out" \
+        '{module: $module_out, service: $service_out}')
+        # Append JSON object to the array
+        json_objects+=($json_object)
+    done <<< "$services"
+done <<< "$modules"
+
+# Convert the array of JSON objects to a single JSON array
+json_services=$(jq -nc '[( $ARGS.positional[] | fromjson )]' --args -- ${json_objects[*]})
+echo "docker_matrix=$(echo $json_services | jq -c '{include: .}')" >> $GITHUB_OUTPUT
+
+################# Setup Docker Registry and Repository Name #################
+# Docker Registry to pull/push images
+REGISTRY_URL="ghcr.io/watonomous/wato_monorepo"
+# REGISTRY_URL="ghcr.io/watonomous/wato_f1tenth"
+
+REGISTRY=$(echo "$REGISTRY_URL" | sed 's|^\(.*\)/.*$|\1|')
+REPOSITORY=$(echo "$REGISTRY_URL" | sed 's|^.*/\(.*\)$|\1|')
+
+echo "registry=$REGISTRY" >> $GITHUB_OUTPUT
+echo "repository=$REPOSITORY" >> $GITHUB_OUTPUT

.github/templates/github_context/action.yml

@@ -0,0 +1,19 @@
+name: Generate GitHub Environment
+
+outputs:
+  source_branch:
+    description: "branch we are currently on"
+    value: ${{ steps.environment-generator.outputs.source_branch }}
+  target_branch:
+    description: "branch we are trying to merge into"
+    value: ${{ steps.environment-generator.outputs.target_branch }}
+
+runs:
+  using: "composite"
+  steps:
+    - id: environment-generator
+      run: ${{ github.action_path }}/branch_sanitation.sh
+      shell: bash
+      env:
+        SOURCE_BRANCH: ${{ github.head_ref || github.ref_name }}
+        TARGET_BRANCH: ${{ github.base_ref }}

.github/templates/github_context/branch_sanitation.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e
+
+# Takes in a branch name and sanitizes the name for docker tagging
+# (eg. no / in the branch name)
+sanitize_branch_name() {
+    echo $(echo $1 | sed 's/[^a-zA-Z0-9._]/-/g' | cut -c 1-128)
+}
+
+################# Setup Source and Target Branch #################
+SOURCE_BRANCH_NAME=$(sanitize_branch_name $SOURCE_BRANCH)
+TARGET_BRANCH_NAME=$(sanitize_branch_name $TARGET_BRANCH)
+
+if [ -z "$TARGET_BRANCH_NAME" ]; then
+    TARGET_BRANCH_NAME=$SOURCE_BRANCH_NAME
+fi
+
+echo "source_branch=$SOURCE_BRANCH_NAME" >> $GITHUB_OUTPUT
+echo "target_branch=$TARGET_BRANCH_NAME" >> $GITHUB_OUTPUT
+
+################# Debug notices #################
+echo "::notice:: Using $SOURCE_BRANCH_NAME as the source branch"
+echo "::notice:: Using $TARGET_BRANCH_NAME as the target branch"

.github/templates/test/action.yml

@@ -0,0 +1,18 @@
+name: Docker test
+
+inputs:
+  image:
+    description: "monorepo image to test"
+    required: true
+  tag:
+    description: "monorepo image tag to test"
+    required: true
+
+runs: 
+  using: "composite"
+  steps:
+    - run: ${{ github.action_path }}/test_image.sh
+      shell: bash
+      env:
+        IMAGE: ${{ inputs.image }}
+        TAG: ${{ inputs.tag }}