Merge pull request #320 from NVIDIA/attrsvc-packaging-cleanup

Clean up attrsvc packaging ownership

Author: namitdhameja

CLAUDE.md

@@ -98,7 +98,7 @@ Shared infrastructure: `health_check.py` (GPU/NIC/storage health checks via NVML
 ## Repository Root
 
 ### `services/`
-Standalone services at the **repository root** (`services/`, not inside `src/nvidia_resiliency_ext/`): `nvrx_attrsvc/` (FastAPI server for LLM log analysis), `nvrx_smonsvc/` (SLURM job monitor).
+Deployment assets for standalone services: `attrsvc/` (FastAPI server for LLM log analysis), `smonsvc/` (SLURM job monitor), and shared scripts. Python service packages live under `src/nvidia_resiliency_ext/services/`.
 
 ## Key Environment Variables
 

build.py

@@ -165,20 +165,6 @@ def _compile_protos(proto_dir, proto_filenames):
 
 
 def build(setup_kwargs):
-    # ``pyproject.toml`` installs ``services/nvrx_attrsvc`` under the
-    # ``nvidia_resiliency_ext.attribution`` namespace via Poetry's package
-    # ``to`` target. The generated setuptools build path does not understand
-    # that target and would otherwise also emit a top-level ``nvrx_attrsvc``.
-    packages = setup_kwargs.get("packages")
-    if packages:
-        setup_kwargs["packages"] = [
-            pkg for pkg in packages if pkg != "nvrx_attrsvc" and not pkg.startswith("nvrx_attrsvc.")
-        ]
-    for package_data_key in ("package_data", "exclude_package_data"):
-        package_data = setup_kwargs.get(package_data_key)
-        if isinstance(package_data, dict):
-            package_data.pop("nvrx_attrsvc", None)
-
     # Generate gRPC Python files from .proto files
     proto_dir = os.path.join("src", "nvidia_resiliency_ext", "shared_utils", "proto")
     proto_files = [

examples/attribution/single_server_example.py

@@ -19,7 +19,10 @@
 sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
 
 from nvidia_resiliency_ext.attribution.mcp_integration.mcp_client import NVRxMCPClient
-from nvidia_resiliency_ext.attribution.svc.config import DEFAULT_LLM_BASE_URL, DEFAULT_LLM_MODEL
+from nvidia_resiliency_ext.attribution.orchestration.config import (
+    DEFAULT_LLM_BASE_URL,
+    DEFAULT_LLM_MODEL,
+)
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(

pyproject.toml

@@ -17,7 +17,6 @@ classifiers = [
 ]
 packages = [
     { include = "nvidia_resiliency_ext", from = "src" },
-    { include = "nvrx_attrsvc", from = "services", to = "nvidia_resiliency_ext/attribution" },
 ]
 
 exclude = [
@@ -60,14 +59,16 @@ pydantic-settings = { version = ">=2.0.0", optional = true }
 slowapi = { version = ">=0.1.9", optional = true }
 slack-bolt = { version = ">=1.23.0", optional = true }
 slack-sdk = { version = ">=3.35.0", optional = true }
+nvdataflow = { version = "*", optional = true }
 grpcio = "^1.76.0"
 grpcio-tools = "^1.76.0"
 protobuf = ">=4.22.0"
 
 [tool.poetry.scripts]
 ft_launcher = "nvidia_resiliency_ext.fault_tolerance.launcher:main"
 nvrx-mcp-analysis = "nvidia_resiliency_ext.attribution.mcp_integration.server_launcher:main"
-nvrx-attrsvc = "nvidia_resiliency_ext.attribution.nvrx_attrsvc.app:main"
+nvrx-attrsvc = "nvidia_resiliency_ext.services.attrsvc.__main__:main"
+nvrx-smonsvc = "nvidia_resiliency_ext.services.smonsvc.__main__:main"
 
 [tool.poetry.extras]
 attribution = [
@@ -83,6 +84,9 @@ attribution = [
     "slack-bolt",
     "slack-sdk",
 ]
+dataflow = [
+    "nvdataflow",
+]
 
 [tool.poetry-dynamic-versioning]
 enable = true

services/README.md

@@ -6,8 +6,8 @@ Automated log analysis and failure attribution for distributed training jobs.
 
 | Component | Description | Documentation |
 |-----------|-------------|---------------|
-| **nvrx-attrsvc** | FastAPI server for LLM-based log analysis | [nvrx_attrsvc/README.md](nvrx_attrsvc/README.md) |
-| **nvrx-smonsvc** | SLURM job monitor for automatic log submission | [nvrx_smonsvc/README.md](nvrx_smonsvc/README.md) |
+| **nvrx-attrsvc** | FastAPI server for LLM-based log analysis | [attrsvc/README.md](attrsvc/README.md) |
+| **nvrx-smonsvc** | SLURM job monitor for automatic log submission | [smonsvc/README.md](smonsvc/README.md) |
 
 See component READMEs for quick start, configuration, and API details.
 
@@ -43,8 +43,8 @@ Periodically snapshot service endpoints for debugging:
 ./scripts/snapshot_services.sh hostname
 
 # Individual services (in respective directories)
-./nvrx_attrsvc/snapshot_attrsvc.sh hostname 8000
-./nvrx_smonsvc/deploy/snapshot_smonsvc.sh hostname 8100
+./attrsvc/deploy/snapshot_attrsvc.sh hostname 8000
+./smonsvc/deploy/snapshot_smonsvc.sh hostname 8100
 ```
 
 Configure via environment: `SNAPSHOT_INTERVAL`, `SNAPSHOT_OUTPUT_DIR`.
@@ -53,12 +53,11 @@ Configure via environment: `SNAPSHOT_INTERVAL`, `SNAPSHOT_OUTPUT_DIR`.
 
 | Path | Description |
 |------|-------------|
-| `nvrx_attrsvc/` | Attribution service package |
-| `nvrx_smonsvc/` | SLURM monitor package |
+| `attrsvc/` | Attribution service deployment docs and assets |
+| `smonsvc/` | SLURM monitor deployment docs and assets |
 | `scripts/` | Shell scripts ([README](scripts/README.md)) |
-| `pyproject.toml` | Python package configuration |
 
 ## Library Layer
 
 The core analysis functionality in `nvidia_resiliency_ext.attribution` can be used without HTTP.
-See [nvrx_attrsvc/README.md](nvrx_attrsvc/README.md#python-api) for the Python API.
+See [attrsvc/README.md](attrsvc/README.md#python-api) for the Python API.

services/nvrx_attrsvc/ATTRSVC_SPEC.md services/attrsvc/ATTRSVC_SPEC.mdservices/nvrx_attrsvc/ATTRSVC_SPEC.md renamed to services/attrsvc/ATTRSVC_SPEC.md

@@ -79,7 +79,7 @@ processed-files ledger (`CACHE_FILE`) — see §3.7. **README.md** for runbooks.
 --------------------------------------------------------------------------------
 
 Two layers: **library** (`nvidia_resiliency_ext.attribution`) and **service**
-(`services/nvrx_attrsvc/` — FastAPI, `Settings`, rate limits, `AttributionHttpAdapter`).
+(`services/attrsvc/` — FastAPI, `Settings`, rate limits, `AttributionHttpAdapter`).
 
 | Layer | See |
 |-------|-----|
@@ -391,7 +391,7 @@ implementation.
 Structured text default; levels per `LOG_LEVEL` (`DEBUG` / `INFO` / `WARNING`). No duplicated template table.
 
 23. DEPENDENCIES  
-**services/pyproject.toml**
+The root package owns service entry points and extras in **pyproject.toml**.
 
 24. TESTING  
 Layout and strategies are not normative for the HTTP contract; see repo **`tests/`**

services/nvrx_attrsvc/README.md services/attrsvc/README.mdservices/nvrx_attrsvc/README.md renamed to services/attrsvc/README.md

@@ -6,7 +6,7 @@ FastAPI server that exposes log analysis over HTTP. It wraps the **`nvidia_resil
 
 ## Library vs this service
 
-| | **Library** (`nvidia_resiliency_ext.attribution`) | **This package** (`nvrx_attrsvc`) |
+| | **Library** (`nvidia_resiliency_ext.attribution`) | **This package** (`nvidia_resiliency_ext.services.attrsvc`) |
 |---|--------------------------------------------------|-----------------------------------|
 | **Role** | **`AttributionController`** (config, cache persistence, health/status, dataflow/Slack stats) over **`Analyzer`** (→ `LogAnalyzer`, pipelines, MCP/lib LogSage, FR analysis, jobs/splitlog) | HTTP API, env-based `Settings`, rate limits, ledger file |
 | **Docs** | [`src/nvidia_resiliency_ext/attribution/ARCHITECTURE.md`](../../src/nvidia_resiliency_ext/attribution/ARCHITECTURE.md), [`README.md`](../../src/nvidia_resiliency_ext/attribution/README.md) | This file, [`ATTRSVC_SPEC.md`](ATTRSVC_SPEC.md) |
@@ -238,7 +238,7 @@ For combined deployment with monitor, see `../scripts/nvrx_services.sbatch`
 
 ```python
 import asyncio
-from nvrx_attrsvc import AttributionHttpAdapter, setup
+from nvidia_resiliency_ext.services.attrsvc import AttributionHttpAdapter, setup
 
 async def main():
     cfg = setup()

services/nvrx_attrsvc/deploy/Dockerfile services/attrsvc/deploy/Dockerfileservices/nvrx_attrsvc/deploy/Dockerfile renamed to services/attrsvc/deploy/Dockerfile

@@ -1,7 +1,7 @@
 # NVRX Attribution Service - Docker Image
 #
 # Build (from repo root):
-#   docker build -t nvrx-attrsvc -f services/nvrx_attrsvc/deploy/Dockerfile .
+#   docker build -t nvrx-attrsvc -f services/attrsvc/deploy/Dockerfile .
 #
 # Run:
 #   docker run -d \
@@ -25,17 +25,12 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends curl && \
     rm -rf /var/lib/apt/lists/*
 
-# Copy and install main library (no CUDA extensions needed for attribution)
-COPY pyproject.toml ./
+# Copy and install root package with attribution extras. The root package owns
+# both service entry points.
+COPY pyproject.toml README.md build.py ./
 COPY src/ ./src/
 ENV STRAGGLER_DET_SKIP_CUPTI_EXT_BUILD=1
-RUN pip install --no-cache-dir --no-deps .
-
-# Copy and install attribution service
-COPY services/pyproject.toml ./services/
-COPY services/nvrx_attrsvc/ ./services/nvrx_attrsvc/
-COPY services/nvrx_smonsvc/ ./services/nvrx_smonsvc/
-RUN pip install --no-cache-dir ./services
+RUN pip install --no-cache-dir ".[attribution]"
 
 # Verify commands are available
 RUN command -v nvrx-attrsvc && command -v nvrx-smonsvc

…ices/nvrx_attrsvc/deploy/kubernetes.yaml services/attrsvc/deploy/kubernetes.yamlservices/nvrx_attrsvc/deploy/kubernetes.yaml renamed to services/attrsvc/deploy/kubernetes.yaml services/nvrx_attrsvc/deploy/kubernetes.yaml renamed to services/attrsvc/deploy/kubernetes.yaml

@@ -1,7 +1,7 @@
 # NVRX Attribution Service - Kubernetes Deployment
 #
 # Deploy:
-#   kubectl apply -f services/nvrx_attrsvc/deploy/kubernetes.yaml
+#   kubectl apply -f services/attrsvc/deploy/kubernetes.yaml
 #
 # Prerequisites:
 #   - Create secret: kubectl create secret generic llm-api-key --from-literal=api-key=your-llm-api-key-here