Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@
# editable install's .pth file (which points to a build-stage-only path).
image=flyte.Image.from_debian_base()
.with_uv_project(pyproject_file=MY_APP_ROOT / "pyproject.toml")
.with_source_folder(MY_LIB_PKG)
.with_local_v2()
.with_code_bundle(),
)
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from my_app.tasks import compute_stats, summarize

MY_APP_ROOT = pathlib.Path(__file__).parent.parent.parent # -> my_app/
SRC_DIR = MY_APP_ROOT / "src" # -> my_app/src/
PROJECT_ROOT = MY_APP_ROOT.parent # -> 02_sibling_packages/


@env.task
Expand All @@ -16,7 +16,7 @@ async def stats_pipeline(values: list[float]) -> str:

if __name__ == "__main__":
# my_lib is installed in the image; root_dir only needs to cover my_app source
flyte.init_from_config(root_dir=SRC_DIR)
flyte.init_from_config(root_dir=PROJECT_ROOT)

# Development -- run a task directly, code bundle handles source delivery
run = flyte.run(stats_pipeline, values=[1.0, 2.0, 3.0, 4.0, 5.0])
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from my_lib.stats import mean, std

from my_app.env import env


@env.task
async def compute_stats(values: list[float]) -> dict:
"""Compute basic statistics using the my_lib utility library."""
from my_lib.stats import mean, std

return {
"mean": mean(values),
"std": std(values),
Expand Down
166 changes: 165 additions & 1 deletion examples/uv_monorepo_guide/02_sibling_packages/my_app/uv.lock

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name = "my-lib"
version = "0.1.0"
description = "Pure utility library — no Flyte dependency"
requires-python = ">=3.11"
dependencies = []
dependencies = ["mypy", "ty"]

[build-system]
requires = ["uv_build>=0.9,<0.10"]
Expand Down
7 changes: 7 additions & 0 deletions src/flyte/_internal/imagebuild/docker_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@
copy_files_to_context,
get_and_list_dockerignore,
get_uv_editable_install_mounts,
get_uv_project_editable_dependencies,
get_uv_project_editable_package_names,
)
from flyte._logging import logger
from flyte._utils.asyncify import run_sync_with_loop
Expand Down Expand Up @@ -299,6 +301,10 @@ async def handle(
pip_install_args = " ".join(layer.get_pip_install_args())
if "--no-install-project" not in pip_install_args:
pip_install_args += " --no-install-project"
editable_deps = get_uv_project_editable_dependencies(layer.pyproject.parent)
# Skip building/installing editable deps as wheels too
for pkg_name in get_uv_project_editable_package_names(editable_deps):
pip_install_args += f" --no-install-package {pkg_name}"
# Only Copy pyproject.yaml and uv.lock (if provided) from the project root.
pyproject_dst = copy_files_to_context(layer.pyproject, context_path)
# Apply any editable install mounts to the template.
Expand All @@ -309,6 +315,7 @@ async def handle(
*STANDARD_IGNORE_PATTERNS,
*docker_ignore_patterns,
],
editable_deps=editable_deps,
)
if layer.uvlock is not None:
uvlock_dst = copy_files_to_context(layer.uvlock, context_path)
Expand Down
9 changes: 7 additions & 2 deletions src/flyte/_internal/imagebuild/remote_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
copy_files_to_context,
get_and_list_dockerignore,
get_uv_project_editable_dependencies,
get_uv_project_editable_package_names,
)
from flyte._internal.runtime.task_serde import get_security_context
from flyte._logging import logger
Expand Down Expand Up @@ -342,10 +343,14 @@ def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2
# We copy the entire dependency directory (not just pyproject.toml) so that uv_build
# can find the source files when building the package during uv sync.
standard_ignore_patterns = STANDARD_IGNORE_PATTERNS.copy()
for editable_dep in get_uv_project_editable_dependencies(layer.pyproject.parent):
editable_deps = get_uv_project_editable_dependencies(layer.pyproject.parent)
# Skip building/installing editable deps as wheels too
for pkg_name in get_uv_project_editable_package_names(editable_deps):
pip_options.extra_args += f" --no-install-package {pkg_name}"
for editable_dep in editable_deps:
pyproject_dir_dsts.append(
copy_files_to_context(
editable_dep,
editable_dep / "pyproject.toml",
context_path,
ignore_patterns=[*standard_ignore_patterns, *docker_ignore_patterns],
)
Expand Down
32 changes: 30 additions & 2 deletions src/flyte/_internal/imagebuild/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
from pathlib import Path, PurePath
from typing import List, Optional

import toml

from flyte._code_bundle._ignore import STANDARD_IGNORE_PATTERNS
from flyte._image import DockerIgnore, Image
from flyte._logging import logger
Expand Down Expand Up @@ -131,8 +133,31 @@ def get_uv_project_editable_dependencies(project_root: Path) -> list[Path]:
return paths


def get_uv_project_editable_package_names(editable_deps: list[Path]) -> list[str]:
"""Returns package names for editable dependencies by reading their pyproject.toml.

Args:
editable_deps: Paths to editable dependency directories.

Returns:
A list of package names for editable dependencies.
"""
names = []
for dep_path in editable_deps:
pyproject_path = dep_path / "pyproject.toml"
if pyproject_path.exists():
data = toml.load(pyproject_path)
name = data.get("project", {}).get("name")
if name:
names.append(name)
return names


def get_uv_editable_install_mounts(
project_root: Path, context_path: Path, ignore_patterns: list[str] | None = None
project_root: Path,
context_path: Path,
ignore_patterns: list[str] | None = None,
editable_deps: list[Path] | None = None,
) -> str:
"""Builds Docker bind mounts for uv editable path dependencies.

Expand All @@ -141,12 +166,15 @@ def get_uv_editable_install_mounts(
context_path: Build context directory for Docker.
ignore_patterns: A list of ignore patterns to apply when copying editable dependency contents.
If None, the standard ignore patterns of 'StandardIgnore' will be used.
editable_deps: Pre-fetched editable dependency paths. If None, they will be resolved via uv export.
Returns:
A string of Docker bind-mount arguments for editable dependencies.
"""
ignore_patterns = ignore_patterns or STANDARD_IGNORE_PATTERNS.copy()
if editable_deps is None:
editable_deps = get_uv_project_editable_dependencies(project_root)
mounts = []
for editable_dep in get_uv_project_editable_dependencies(project_root):
for editable_dep in editable_deps:
# Copy the contents of the editable install by applying ignores
editable_dep_within_context = copy_files_to_context(editable_dep, context_path, ignore_patterns=ignore_patterns)
mounts.append(
Expand Down
38 changes: 37 additions & 1 deletion src/flyte/_module.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,39 @@
from typing import Tuple


def _find_package_root(file_path: pathlib.Path, source_dir: pathlib.Path) -> pathlib.Path:
"""Find the best Python package root for a file by checking sys.path.

In src-layout projects (e.g., my_app/src/my_app/main.py), the importable
package root is my_app/src/, not source_dir. This function looks for
the deepest sys.path entry between source_dir and the file that still
produces a valid module path (i.e., not just the bare filename).

Example:
source_dir = /repo
file_path = /repo/my_app/src/my_app/main.py
sys.path = [..., /repo/my_app/src, ...]
returns /repo/my_app/src (so module = my_app.main)
"""
file_parent = file_path.parent.resolve()
best = source_dir
for p in sys.path:
if "site-packages" in p or "dist-packages" in p:
continue
candidate = pathlib.Path(p).resolve()
# candidate must be: between source_dir and file_path, but not the
# file's own directory (that would give just the filename, e.g. "main")
if (
candidate != source_dir
and candidate != file_parent
and candidate.is_relative_to(source_dir)
and file_path.is_relative_to(candidate)
and len(candidate.parts) > len(best.parts)
):
best = candidate
return best


def extract_obj_module(obj: object, /, source_dir: pathlib.Path | None = None) -> Tuple[str, ModuleType]:
"""
Extract the module from the given object. If source_dir is provided, the module will be relative to the source_dir.
Expand Down Expand Up @@ -34,13 +67,16 @@ def extract_obj_module(obj: object, /, source_dir: pathlib.Path | None = None) -
try:
# Get the relative path to the current directory
# Will raise ValueError if the file is not in the source directory
relative_path = file_path.relative_to(str(pathlib.Path(source_dir).absolute()))
source_dir_abs = pathlib.Path(source_dir).absolute()
relative_path = file_path.relative_to(str(source_dir_abs))

if relative_path == pathlib.Path("_internal/resolvers"):
entity_module_name = entity_module.__name__
elif "site-packages" in str(file_path) or "dist-packages" in str(file_path):
raise ValueError("Object from a library")
else:
package_root = _find_package_root(file_path, source_dir_abs)
relative_path = file_path.relative_to(str(package_root))
# Replace file separators with dots and remove the '.py' extension
dotted_path = os.path.splitext(str(relative_path))[0].replace(os.sep, ".")
entity_module_name = dotted_path
Expand Down
17 changes: 15 additions & 2 deletions src/flyte/_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,9 +300,22 @@ async def _run_remote(self, obj: TaskTemplate[P, R, F] | LazyEntity, *args: P.ar
added_paths = [
f"./{pathlib.Path(p).relative_to(root_dir_abs)}"
for p in sys.path
if pathlib.Path(p).is_relative_to(root_dir_abs)
if pathlib.Path(p).is_relative_to(root_dir_abs) and "site-packages" not in pathlib.Path(p).parts
]
env[FLYTE_SYS_PATH] = ":".join(added_paths)
# Remove redundant subdirectory paths. Python auto-adds the script's
# directory (e.g., ./my_app/src/my_app) which shadows the real package
# at its parent (./my_app/src). Keep only the shallowest paths.
normalized = {p: os.path.normpath(p) for p in added_paths}
filtered_paths = [
p for p in added_paths
if not any(
normalized[p] != normalized[other]
and normalized[other] != "."
and pathlib.PurePosixPath(normalized[p]).is_relative_to(normalized[other])
for other in added_paths
)
]
env[FLYTE_SYS_PATH] = ":".join(filtered_paths)

# TODO: Remove once the actions service is the default and this env var is no longer needed.
if os.getenv("_U_USE_ACTIONS") == "1":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def _generate_request_id() -> str:

def with_metadata(call_details: ClientCallDetails, new_metadata: Metadata) -> ClientCallDetails:
metadata = Metadata()
for k, v in call_details.metadata.keys():
for k, v in call_details.metadata:
# Add existing metadata to the new metadata object
metadata.add(key=k, value=v)
for k, v in new_metadata.keys():
Expand Down
37 changes: 37 additions & 0 deletions tests/flyte/imagebuild/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
get_and_list_dockerignore,
get_uv_editable_install_mounts,
get_uv_project_editable_dependencies,
get_uv_project_editable_package_names,
)


Expand Down Expand Up @@ -134,6 +135,42 @@ def test_get_uv_editable_install_mounts():
assert mounts == " ".join(expected_mounts)


def test_get_uv_project_editable_package_names():
with tempfile.TemporaryDirectory() as tmp_context:
project_root = Path(tmp_context)

# Create two editable deps with pyproject.toml files
dep_a = project_root / "dep_a"
dep_a.mkdir()
(dep_a / "pyproject.toml").write_text('[project]\nname = "my-lib-a"\nversion = "0.1.0"\n')

dep_b = project_root / "dep_b"
dep_b.mkdir()
(dep_b / "pyproject.toml").write_text('[project]\nname = "my-lib-b"\nversion = "0.2.0"\n')

# Create a dep without a [project].name (should be skipped)
dep_c = project_root / "dep_c"
dep_c.mkdir()
(dep_c / "pyproject.toml").write_text('[tool.something]\nkey = "value"\n')

names = get_uv_project_editable_package_names([dep_a, dep_b, dep_c])

assert names == ["my-lib-a", "my-lib-b"]


def test_get_uv_project_editable_package_names_no_pyproject():
"""Test that deps without pyproject.toml are skipped."""
with tempfile.TemporaryDirectory() as tmp_context:
project_root = Path(tmp_context)
dep = project_root / "dep_missing"
dep.mkdir()
# No pyproject.toml created

names = get_uv_project_editable_package_names([dep])

assert names == []


def test_copy_files_to_context_ignores_egg_info():
"""Test that copy_files_to_context ignores .egg-info directories."""
with tempfile.TemporaryDirectory() as tmp_context:
Expand Down
Loading