diff --git a/CHANGELOG.md b/CHANGELOG.md index dc5cd08b..9a565484 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,28 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang +## [1.13.5](https://github.com/opsmill/infrahub-sdk-python/tree/v1.13.5) - 2025-07-23 + +### Fixed + +- Respect ordering when loading files from a directory + +## [1.13.4](https://github.com/opsmill/infrahub-sdk-python/tree/v1.13.4) - 2025-07-22 + +### Fixed + +- Fix processing of relationshhip during nodes retrieval using the Sync Client, when prefecthing related_nodes. ([#461](https://github.com/opsmill/infrahub-sdk-python/issues/461)) +- Fix schema loading to ignore non-YAML files in folders. ([#462](https://github.com/opsmill/infrahub-sdk-python/issues/462)) +- Fix ignored node variable in filters(). ([#469](https://github.com/opsmill/infrahub-sdk-python/issues/469)) +- Fix use of parallel with filters for Infrahub Client Sync. +- Avoid sending empty list to infrahub if no valids schemas are found. + +## [1.13.3](https://github.com/opsmill/infrahub-sdk-python/tree/v1.13.3) - 2025-06-30 + +### Fixed + +- Update InfrahubNode creation to include __typename, display_label, and kind from a RelatedNode ([#455](https://github.com/opsmill/infrahub-sdk-python/issues/455)) + ## [1.13.2](https://github.com/opsmill/infrahub-sdk-python/tree/v1.13.2) - 2025-06-27 ### Fixed diff --git a/changelog/+batch.fixed.md b/changelog/+batch.fixed.md new file mode 100644 index 00000000..635d6b62 --- /dev/null +++ b/changelog/+batch.fixed.md @@ -0,0 +1 @@ +Create a new batch while fetching relationships instead of using the reusing the same one. \ No newline at end of file diff --git a/changelog/+branch-in-count.fixed.md b/changelog/+branch-in-count.fixed.md new file mode 100644 index 00000000..b4227386 --- /dev/null +++ b/changelog/+branch-in-count.fixed.md @@ -0,0 +1 @@ +Update internal calls to `count` to include the branch parameter so that the query is performed on the correct branch \ No newline at end of file diff --git a/infrahub_sdk/client.py b/infrahub_sdk/client.py index dc1f539f..16c1c73a 100644 --- a/infrahub_sdk/client.py +++ b/infrahub_sdk/client.py @@ -784,7 +784,6 @@ async def filters( if at: at = Timestamp(at) - node = InfrahubNode(client=self, schema=schema, branch=branch) filters = kwargs pagination_size = self.pagination_size @@ -825,12 +824,12 @@ async def process_batch() -> tuple[list[InfrahubNode], list[InfrahubNode]]: nodes = [] related_nodes = [] batch_process = await self.create_batch() - count = await self.count(kind=schema.kind, partial_match=partial_match, **filters) + count = await self.count(kind=schema.kind, branch=branch, partial_match=partial_match, **filters) total_pages = (count + pagination_size - 1) // pagination_size for page_number in range(1, total_pages + 1): page_offset = (page_number - 1) * pagination_size - batch_process.add(task=process_page, node=node, page_offset=page_offset, page_number=page_number) + batch_process.add(task=process_page, page_offset=page_offset, page_number=page_number) async for _, response in batch_process.execute(): nodes.extend(response[1]["nodes"]) @@ -847,7 +846,7 @@ async def process_non_batch() -> tuple[list[InfrahubNode], list[InfrahubNode]]: while has_remaining_items: page_offset = (page_number - 1) * pagination_size - response, process_result = await process_page(page_offset, page_number) + response, process_result = await process_page(page_offset=page_offset, page_number=page_number) nodes.extend(process_result["nodes"]) related_nodes.extend(process_result["related_nodes"]) @@ -1946,9 +1945,9 @@ def filters( """ branch = branch or self.default_branch schema = self.schema.get(kind=kind, branch=branch) - node = InfrahubNodeSync(client=self, schema=schema, branch=branch) if at: at = Timestamp(at) + filters = kwargs pagination_size = self.pagination_size @@ -1990,12 +1989,12 @@ def process_batch() -> tuple[list[InfrahubNodeSync], list[InfrahubNodeSync]]: related_nodes = [] batch_process = self.create_batch() - count = self.count(kind=schema.kind, partial_match=partial_match, **filters) + count = self.count(kind=schema.kind, branch=branch, partial_match=partial_match, **filters) total_pages = (count + pagination_size - 1) // pagination_size for page_number in range(1, total_pages + 1): page_offset = (page_number - 1) * pagination_size - batch_process.add(task=process_page, node=node, page_offset=page_offset, page_number=page_number) + batch_process.add(task=process_page, page_offset=page_offset, page_number=page_number) for _, response in batch_process.execute(): nodes.extend(response[1]["nodes"]) @@ -2012,7 +2011,7 @@ def process_non_batch() -> tuple[list[InfrahubNodeSync], list[InfrahubNodeSync]] while has_remaining_items: page_offset = (page_number - 1) * pagination_size - response, process_result = process_page(page_offset, page_number) + response, process_result = process_page(page_offset=page_offset, page_number=page_number) nodes.extend(process_result["nodes"]) related_nodes.extend(process_result["related_nodes"]) diff --git a/infrahub_sdk/ctl/utils.py b/infrahub_sdk/ctl/utils.py index 63d7dfb8..66f86865 100644 --- a/infrahub_sdk/ctl/utils.py +++ b/infrahub_sdk/ctl/utils.py @@ -187,6 +187,9 @@ def load_yamlfile_from_disk_and_exit( has_error = False try: data_files = file_type.load_from_disk(paths=paths) + if not data_files: + console.print("[red]No valid files found to load.") + raise typer.Exit(1) except FileNotValidError as exc: console.print(f"[red]{exc.message}") raise typer.Exit(1) from exc diff --git a/infrahub_sdk/node/node.py b/infrahub_sdk/node/node.py index 7a095586..f69c6231 100644 --- a/infrahub_sdk/node/node.py +++ b/infrahub_sdk/node/node.py @@ -402,10 +402,10 @@ def generate_query_data_init( if order: data["@filters"]["order"] = order - if offset: + if offset is not None: data["@filters"]["offset"] = offset - if limit: + if limit is not None: data["@filters"]["limit"] = limit if include and exclude: @@ -507,11 +507,17 @@ def _init_relationships(self, data: dict | RelatedNode | None = None) -> None: if rel_schema.cardinality == "one": if isinstance(rel_data, RelatedNode): - peer_id_data: dict[str, Any] = {} - if rel_data.id: - peer_id_data["id"] = rel_data.id - if rel_data.hfid: - peer_id_data["hfid"] = rel_data.hfid + peer_id_data: dict[str, Any] = { + key: value + for key, value in ( + ("id", rel_data.id), + ("hfid", rel_data.hfid), + ("__typename", rel_data.typename), + ("kind", rel_data.kind), + ("display_label", rel_data.display_label), + ) + if value is not None + } if peer_id_data: rel_data = peer_id_data else: @@ -1090,11 +1096,17 @@ def _init_relationships(self, data: dict | None = None) -> None: if rel_schema.cardinality == "one": if isinstance(rel_data, RelatedNodeSync): - peer_id_data: dict[str, Any] = {} - if rel_data.id: - peer_id_data["id"] = rel_data.id - if rel_data.hfid: - peer_id_data["hfid"] = rel_data.hfid + peer_id_data: dict[str, Any] = { + key: value + for key, value in ( + ("id", rel_data.id), + ("hfid", rel_data.hfid), + ("__typename", rel_data.typename), + ("kind", rel_data.kind), + ("display_label", rel_data.display_label), + ) + if value is not None + } if peer_id_data: rel_data = peer_id_data else: @@ -1481,15 +1493,15 @@ def _process_relationships( for rel_name in self._relationships: rel = getattr(self, rel_name) if rel and isinstance(rel, RelatedNodeSync): - relation = node_data["node"].get(rel_name) - if relation.get("node", None): + relation = node_data["node"].get(rel_name, None) + if relation and relation.get("node", None): related_node = InfrahubNodeSync.from_graphql( client=self._client, branch=branch, data=relation, timeout=timeout ) related_nodes.append(related_node) elif rel and isinstance(rel, RelationshipManagerSync): - peers = node_data["node"].get(rel_name) - if peers: + peers = node_data["node"].get(rel_name, None) + if peers and peers["edges"]: for peer in peers["edges"]: related_node = InfrahubNodeSync.from_graphql( client=self._client, branch=branch, data=peer, timeout=timeout diff --git a/infrahub_sdk/node/related_node.py b/infrahub_sdk/node/related_node.py index 60d46ca9..bf6cb532 100644 --- a/infrahub_sdk/node/related_node.py +++ b/infrahub_sdk/node/related_node.py @@ -39,6 +39,7 @@ def __init__(self, branch: str, schema: RelationshipSchemaAPI, data: Any | dict, self._hfid: list[str] | None = None self._display_label: str | None = None self._typename: str | None = None + self._kind: str | None = None if isinstance(data, (CoreNodeBase)): self._peer = data @@ -118,6 +119,12 @@ def typename(self) -> str | None: return self._peer.typename return self._typename + @property + def kind(self) -> str | None: + if self._peer: + return self._peer.get_kind() + return self._kind + def _generate_input_data(self, allocate_from_pool: bool = False) -> dict[str, Any]: data: dict[str, Any] = {} diff --git a/infrahub_sdk/node/relationship.py b/infrahub_sdk/node/relationship.py index c527dc50..8473a1cb 100644 --- a/infrahub_sdk/node/relationship.py +++ b/infrahub_sdk/node/relationship.py @@ -1,11 +1,15 @@ from __future__ import annotations +from collections import defaultdict from collections.abc import Iterable from typing import TYPE_CHECKING, Any +from ..batch import InfrahubBatch from ..exceptions import ( + Error, UninitializedError, ) +from ..types import Order from .constants import PROPERTIES_FLAG, PROPERTIES_OBJECT from .related_node import RelatedNode, RelatedNodeSync @@ -156,8 +160,26 @@ async def fetch(self) -> None: self.peers = rm.peers self.initialized = True + ids_per_kind_map = defaultdict(list) for peer in self.peers: - await peer.fetch() # type: ignore[misc] + if not peer.id or not peer.typename: + raise Error("Unable to fetch the peer, id and/or typename are not defined") + ids_per_kind_map[peer.typename].append(peer.id) + + batch = InfrahubBatch(max_concurrent_execution=self.client.max_concurrent_execution) + for kind, ids in ids_per_kind_map.items(): + batch.add( + task=self.client.filters, + kind=kind, + ids=ids, + populate_store=True, + branch=self.branch, + parallel=True, + order=Order(disable=True), + ) + + async for _ in batch.execute(): + pass def add(self, data: str | RelatedNode | dict) -> None: """Add a new peer to this relationship.""" @@ -261,8 +283,27 @@ def fetch(self) -> None: self.peers = rm.peers self.initialized = True + ids_per_kind_map = defaultdict(list) for peer in self.peers: - peer.fetch() + if not peer.id or not peer.typename: + raise Error("Unable to fetch the peer, id and/or typename are not defined") + ids_per_kind_map[peer.typename].append(peer.id) + + # Unlike Async, no need to create a new batch from scratch because we are not using a semaphore + batch = self.client.create_batch() + for kind, ids in ids_per_kind_map.items(): + batch.add( + task=self.client.filters, + kind=kind, + ids=ids, + populate_store=True, + branch=self.branch, + parallel=True, + order=Order(disable=True), + ) + + for _ in batch.execute(): + pass def add(self, data: str | RelatedNodeSync | dict) -> None: """Add a new peer to this relationship.""" diff --git a/infrahub_sdk/yaml.py b/infrahub_sdk/yaml.py index 8e3f2f73..6b764081 100644 --- a/infrahub_sdk/yaml.py +++ b/infrahub_sdk/yaml.py @@ -120,16 +120,22 @@ def load_file_from_disk(cls, path: Path) -> list[Self]: @classmethod def load_from_disk(cls, paths: list[Path]) -> list[Self]: yaml_files: list[Self] = [] + file_extensions = {".yaml", ".yml", ".json"} # FIXME: .json is not a YAML file, should be removed + for file_path in paths: - if file_path.is_file() and file_path.suffix in [".yaml", ".yml", ".json"]: - yaml_files.extend(cls.load_file_from_disk(path=file_path)) + if not file_path.exists(): + # Check if the provided path exists, relevant for the first call coming from the user + raise FileNotValidError(name=str(file_path), message=f"{file_path} does not exist!") + if file_path.is_file(): + if file_path.suffix in file_extensions: + yaml_files.extend(cls.load_file_from_disk(path=file_path)) + # else: silently skip files with unrelevant extensions (e.g. .md, .py...) elif file_path.is_dir(): + # Introduce recursion to handle sub-folders sub_paths = [Path(sub_file_path) for sub_file_path in file_path.glob("*")] - sub_files = cls.load_from_disk(paths=sub_paths) - sorted_sub_files = sorted(sub_files, key=lambda x: x.location) - yaml_files.extend(sorted_sub_files) - else: - raise FileNotValidError(name=str(file_path), message=f"{file_path} does not exist!") + sub_paths = sorted(sub_paths, key=lambda p: p.name) + yaml_files.extend(cls.load_from_disk(paths=sub_paths)) + # else: skip non-file, non-dir (e.g., symlink...) return yaml_files diff --git a/pyproject.toml b/pyproject.toml index f6b4f7e3..2465500f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "infrahub-sdk" -version = "1.13.2" +version = "1.13.5" description = "Python Client to interact with Infrahub" authors = ["OpsMill "] readme = "README.md" diff --git a/tasks.py b/tasks.py index 62ad2299..e10ca2c1 100644 --- a/tasks.py +++ b/tasks.py @@ -201,7 +201,7 @@ def lint_vale(context: Context) -> None: return print(" - Check documentation style with vale") - exec_cmd = r'vale $(find ./docs -type f \( -name "*.mdx" -o -name "*.md" \))' + exec_cmd = r'vale $(find ./docs -type f \( -name "*.mdx" -o -name "*.md" \) -not -path "*/node_modules/*")' with context.cd(MAIN_DIRECTORY_PATH): context.run(exec_cmd) diff --git a/tests/fixtures/nested_spec_objects/0_folder/4_subfolder/to_be_ignored.py b/tests/fixtures/nested_spec_objects/0_folder/4_subfolder/to_be_ignored.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/nested_spec_objects/2_folder/to_be_ignored.md b/tests/fixtures/nested_spec_objects/2_folder/to_be_ignored.md new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/test_node.py b/tests/integration/test_node.py index c398d00a..9cc2ed75 100644 --- a/tests/integration/test_node.py +++ b/tests/integration/test_node.py @@ -83,6 +83,7 @@ async def test_node_create_with_relationships_using_related_node( assert node_after.name.value == node.name.value assert node_after.manufacturer.peer.id == manufacturer_mercedes.id assert node_after.owner.peer.id == person_joe.id + assert node_after.owner.peer.typename == "TestingPerson" async def test_node_update_with_original_data( self, diff --git a/tests/unit/sdk/test_node.py b/tests/unit/sdk/test_node.py index 2938d4a5..b7f5eb38 100644 --- a/tests/unit/sdk/test_node.py +++ b/tests/unit/sdk/test_node.py @@ -196,7 +196,13 @@ async def test_init_node_data_user_with_relationships(client, location_schema: N @pytest.mark.parametrize("client_type", client_types) -@pytest.mark.parametrize("rel_data", [{"id": "pppppppp"}, {"hfid": ["pppp", "pppp"]}]) +@pytest.mark.parametrize( + "rel_data", + [ + {"id": "pppppppp", "__typename": "BuiltinTag"}, + {"hfid": ["pppp", "pppp"], "display_label": "mmmm", "kind": "BuiltinTag"}, + ], +) async def test_init_node_data_user_with_relationships_using_related_node( client, location_schema: NodeSchemaAPI, client_type, rel_data ): @@ -231,6 +237,9 @@ async def test_init_node_data_user_with_relationships_using_related_node( assert isinstance(node.primary_tag, RelatedNodeBase) assert node.primary_tag.id == rel_data.get("id") assert node.primary_tag.hfid == rel_data.get("hfid") + assert node.primary_tag.typename == rel_data.get("__typename") + assert node.primary_tag.kind == rel_data.get("kind") + assert node.primary_tag.display_label == rel_data.get("display_label") keys = dir(node) assert "name" in keys @@ -1874,6 +1883,19 @@ async def test_node_fetch_relationship( ) response2 = { + "data": { + "BuiltinTag": { + "count": 1, + } + } + } + + httpx_mock.add_response( + method="POST", + json=response2, + ) + + response3 = { "data": { "BuiltinTag": { "count": 1, @@ -1886,7 +1908,7 @@ async def test_node_fetch_relationship( httpx_mock.add_response( method="POST", - json=response2, + json=response3, match_headers={"X-Infrahub-Tracker": "query-builtintag-page1"}, ) diff --git a/tests/unit/sdk/test_yaml.py b/tests/unit/sdk/test_yaml.py index 3532265c..af5a8591 100644 --- a/tests/unit/sdk/test_yaml.py +++ b/tests/unit/sdk/test_yaml.py @@ -1,5 +1,9 @@ from pathlib import Path +import pytest + +from infrahub_sdk.exceptions import FileNotValidError +from infrahub_sdk.utils import get_fixtures_dir from infrahub_sdk.yaml import YamlFile here = Path(__file__).parent.resolve() @@ -42,3 +46,9 @@ def test_read_multiple_files_invalid() -> None: assert yaml_files[0].valid is True assert yaml_files[1].document_position == 2 assert yaml_files[1].valid is False + + +def test_load_non_existing_folder(): + with pytest.raises(FileNotValidError) as exc: + YamlFile.load_from_disk(paths=[get_fixtures_dir() / "does_not_exist"]) + assert "does not exist" in str(exc.value)