ai-dynamo · saturley-hall · Sep 5, 2025 · Sep 5, 2025
@@ -280,21 +280,55 @@ RUN --mount=type=bind,source=./container/launch_message.txt,target=/workspace/la
 ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
 CMD []
 
-#######################################
-########## Local Development #######
-#######################################
+#######################################################################
+########## DEVELOPMENT TARGETS FEATURE MATRIX #########################
+#######################################################################
+# Feature              │ local-dev Target    │ dev Target
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Purpose              │ Dev Container       │ Command-line with
+#                      │ plugin use only     │ run.sh script
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Default User         │ ubuntu user         │ root user
+# ─────────────────────┼─────────────────────┼─────────────────────
+# User Setup           │ Full ubuntu user    │ No user setup
+#                      │ with UID/GID        │
+#                      │ mapping             │
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Permissions          │ ubuntu user with    │ Root-level
+#                      │ sudo privileges     │ permissions
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Home Directory       │ /home/ubuntu        │ /root
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Working Directory    │ /home/ubuntu/dynamo │ /workspace
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Rust Toolchain       │ User's home         │ System locations
+#                      │ (~/.rustup,         │ (/usr/local/rustup,
+#                      │  ~/.cargo)          │  /usr/local/cargo)
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Python Environment   │ User-owned venv     │ System location
+#                      │                     │ (/opt/dynamo/venv)
+# ─────────────────────┼─────────────────────┼─────────────────────
+# File Permissions     │ User-level with     │ Root-level
+#                      │ proper ownership    │ permissions
+# ─────────────────────┼─────────────────────┼─────────────────────
+# Compatibility        │ MS Plug-in: Dev     │ Backward compatibility
+#                      │ Container workflow  │ with existing
+#                      │                     │ workflows
 #
-# PURPOSE: Local development
-#
-# This stage adds development tools, utilities, and dependencies specifically
-# needed for:
-# - Local development and debugging
-# - vscode/cursor development
+# USAGE GUIDELINES:
+# • Use local-dev: VS Code/Cursor Dev Container plugin only
+# • Use dev: run.sh script for command-line development
+
+
+#######################################################################
+########## Development (Dev Container only) ###########################
+#######################################################################
 #
-# Use this stage when you need a full development environment with additional
-# tooling beyond the base runtime image.
+# This stage is for Dev Container plug-in use only.
+# It provides a local development environment with extra tools and dependencies
+# not present in the base runtime image.
 
-FROM runtime AS dev
+FROM runtime AS local-dev
 
 # Install utilities
 RUN apt-get update -y && \
@@ -376,3 +410,79 @@ RUN mkdir -p /home/$USERNAME/.cache/
 
 ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
 CMD []
+
+
+###########################################################
+########## Development (run.sh, runs as root user) ########
+###########################################################
+#
+# PURPOSE: Local development environment for use with run.sh (not Dev Container plug-in)
+#
+# This stage runs as root and provides:
+# - Development tools and utilities for local debugging
+# - Support for vscode/cursor development outside the Dev Container plug-in
+#
+# Use this stage if you need a full-featured development environment with extra tools,
+# but do not use it with the Dev Container plug-in.
+
+FROM runtime AS dev
+
+# Don't want ubuntu to be editable, just change uid and gid.
+ARG WORKSPACE_DIR=/workspace
+
+# Install utilities as root
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends  \
+    # Install utilities
+    nvtop \
+    wget \
+    tmux \
+    vim \
+    git \
+    openssh-client \
+    iproute2 \
+    rsync \
+    zip \
+    unzip \
+    htop \
+    # Build Dependencies
+    autoconf \
+    automake \
+    cmake \
+    libtool \
+    meson \
+    net-tools \
+    pybind11-dev \
+    # Rust build dependencies
+    clang \
+    libclang-dev \
+    protobuf-compiler && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=runtime /usr/local/bin /usr/local/bin
+
+# Set workspace directory variable
+ENV WORKSPACE_DIR=${WORKSPACE_DIR} \
+    DYNAMO_HOME=${WORKSPACE_DIR} \
+    RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    CARGO_TARGET_DIR=/workspace/target \
+    VIRTUAL_ENV=/opt/dynamo/venv \
+    PATH=/usr/local/cargo/bin:$PATH
+
+COPY --from=dynamo_base /usr/local/rustup /usr/local/rustup
+COPY --from=dynamo_base /usr/local/cargo /usr/local/cargo
+
+# This is a slow operation (~40s on my cpu)
+# Much better than chown -R $USERNAME:$USERNAME /opt/dynamo/venv (~10min on my cpu)
+COPY --from=runtime ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+# so we can use maturin develop
+RUN uv pip install maturin[patchelf]
+
+# Make sure to sync this with the one specified on README.md.
+# This is a generic PYTHONPATH which works for all the frameworks, so some paths may not be relevant for this particular framework.
+ENV PYTHONPATH=${WORKSPACE_DIR}/components/metrics/src:${WORKSPACE_DIR}/components/frontend/src:${WORKSPACE_DIR}/components/planner/src:${WORKSPACE_DIR}/components/backends/mocker/src:${WORKSPACE_DIR}/components/backends/trtllm/src:${WORKSPACE_DIR}/components/backends/vllm/src:${WORKSPACE_DIR}/components/backends/sglang/src:${WORKSPACE_DIR}/components/backends/llama_cpp/src
+
+ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
+CMD []
@@ -443,7 +443,6 @@ error() {
 
 get_options "$@"
 
-
 # Automatically set ARCH and ARCH_ALT if PLATFORM is linux/arm64
 ARCH="amd64"
 if [[ "$PLATFORM" == *"linux/arm64"* ]]; then
@@ -465,7 +464,7 @@ fi
 # Add NIXL_REF as a build argument
 BUILD_ARGS+=" --build-arg NIXL_REF=${NIXL_REF} "
 
-if [[ $TARGET == "dev" ]]; then
+if [[ $TARGET == "local-dev" ]]; then
     BUILD_ARGS+=" --build-arg USER_UID=$(id -u) --build-arg USER_GID=$(id -g) "
 fi
 
@@ -599,8 +598,6 @@ if [ "$USE_SCCACHE" = true ]; then
     BUILD_ARGS+=" --build-arg USE_SCCACHE=true"
     BUILD_ARGS+=" --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}"
     BUILD_ARGS+=" --build-arg SCCACHE_REGION=${SCCACHE_REGION}"
-
-
 fi
 
 LATEST_TAG="--tag dynamo:latest-${FRAMEWORK,,}"

@@ -43,7 +43,6 @@ INTERACTIVE=
 USE_NIXL_GDS=
 RUNTIME=nvidia
 WORKDIR=/workspace
-USER=
 
 get_options() {
     while :; do
@@ -125,14 +124,6 @@ get_options() {
                 missing_requirement "$1"
             fi
             ;;
-        --user)
-            if [ "$2" ]; then
-                USER="$2"
-                shift
-            else
-                missing_requirement "$1"
-            fi
-            ;;
         --privileged)
             if [ "$2" ]; then
                 PRIVILEGED=$2
@@ -283,16 +274,9 @@ get_options() {
         RM_STRING=" --rm "
     fi
 
-    if [[ ${USER} == "" ]]; then
-        USER_STRING=""
-    else
-        USER_STRING="--user ${USER}"
-    fi
-
     if [ -n "$USE_NIXL_GDS" ]; then
         VOLUME_MOUNTS+=" -v /run/udev:/run/udev:ro "
         NIXL_GDS_CAPS="--cap-add=IPC_LOCK"
-
         # NOTE(jthomson04): In the KVBM disk pools, we currently allocate our files in /tmp.
         # For some arcane reason, GDS requires that /tmp be mounted.
         # This is already handled for us if we set --mount-workspace
@@ -306,14 +290,15 @@ get_options() {
     if [[ "$GPUS" == "none" || "$GPUS" == "NONE" ]]; then
             RUNTIME=""
     fi
+
     REMAINING_ARGS=("$@")
 }
 
 show_help() {
     echo "usage: run.sh"
     echo "  [--image image]"
     echo "  [--framework framework one of ${!FRAMEWORKS[*]}]"
-    echo "  [--name name for launched container, default NONE] "
+    echo "  [--name name for launched container, default NONE]"
     echo "  [--privileged whether to launch in privileged mode, default FALSE unless mounting workspace]"
     echo "  [--dry-run print docker commands without running]"
     echo "  [--hf-cache directory to volume mount as the hf cache, default is NONE unless mounting workspace]"
@@ -325,7 +310,8 @@ show_help() {
     echo "  [-- stop processing and pass remaining args as command to docker run]"
     echo "  [--workdir set the working directory inside the container]"
     echo "  [--runtime add runtime variables]"
-    echo "  [--user override the user for running the container]"
+    echo "  [--entrypoint override container entrypoint]"
+    echo "  [-h, --help show this help]"
     exit 0
 }
 
@@ -341,7 +327,6 @@ error() {
 get_options "$@"
 
 # RUN the image
-
 if [ -z "$RUN_PREFIX" ]; then
     set -x
 fi
@@ -363,7 +348,6 @@ ${RUN_PREFIX} docker run \
     ${NIXL_GDS_CAPS} \
     --ipc host \
     ${PRIVILEGED_STRING} \
-    ${USER_STRING} \
     ${NAME_STRING} \
     ${ENTRYPOINT_STRING} \
     ${IMAGE} \