From a514ff74415c2222226faf8b744ccaaf8ab49fd7 Mon Sep 17 00:00:00 2001 From: Stoffel Date: Mon, 7 Jul 2025 23:09:27 +0200 Subject: [PATCH 01/12] Add first iteration fix -> Update once checked with code owner! --- plugins/inventory/nb_inventory.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/inventory/nb_inventory.py b/plugins/inventory/nb_inventory.py index ac4998da..2ef25358 100644 --- a/plugins/inventory/nb_inventory.py +++ b/plugins/inventory/nb_inventory.py @@ -1331,12 +1331,12 @@ def refresh_services(self): else: device_services = self.get_resource_list_chunked( api_url=url, - query_key="device_id", + query_key="parent_object_id", query_values=self.devices_lookup.keys(), ) vm_services = self.get_resource_list_chunked( api_url=url, - query_key="virtual_machine_id", + query_key="parent_object_id", query_values=self.vms_lookup.keys(), ) services = chain(device_services, vm_services) @@ -1349,13 +1349,13 @@ def refresh_services(self): for service in services: service_id = service["id"] - if service.get("device"): - self.device_services_lookup[service["device"]["id"]][ + if service.get("parent_object_type") == "dcim.device": + self.device_services_lookup[service["parent_object_id"]][ service_id ] = service - if service.get("virtual_machine"): - self.vm_services_lookup[service["virtual_machine"]["id"]][ + if service.get("parent_object_type") == "virtualization.virtualmachine": + self.vm_services_lookup[service["parent_object_id"]][ service_id ] = service From 310665e654e5d7d60cea13ec9ecc783734ec3d71 Mon Sep 17 00:00:00 2001 From: Stoffel Date: Tue, 8 Jul 2025 20:27:29 +0200 Subject: [PATCH 02/12] Code cleanup and optimization. --- plugins/inventory/nb_inventory.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/plugins/inventory/nb_inventory.py b/plugins/inventory/nb_inventory.py index 2ef25358..3917cf4b 100644 --- a/plugins/inventory/nb_inventory.py +++ b/plugins/inventory/nb_inventory.py @@ -1329,17 +1329,12 @@ def refresh_services(self): if self.fetch_all: services = self.get_resource_list(url) else: - device_services = self.get_resource_list_chunked( + services = self.get_resource_list_chunked( api_url=url, query_key="parent_object_id", - query_values=self.devices_lookup.keys(), - ) - vm_services = self.get_resource_list_chunked( - api_url=url, - query_key="parent_object_id", - query_values=self.vms_lookup.keys(), + # Query only affected devices and vms and sanitize the list to only contain every ID once + query_values=set(chain(self.vms_lookup.keys(), self.devices_lookup.keys())) ) - services = chain(device_services, vm_services) # Construct a dictionary of dictionaries, separately for devices and vms. # Allows looking up services by device id or vm id From e41da76c54bd82c83d5e066be9235d6403cddde2 Mon Sep 17 00:00:00 2001 From: Stoffel Date: Tue, 8 Jul 2025 20:31:07 +0200 Subject: [PATCH 03/12] Code linting fix --- plugins/inventory/nb_inventory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/inventory/nb_inventory.py b/plugins/inventory/nb_inventory.py index 3917cf4b..8bbb8a86 100644 --- a/plugins/inventory/nb_inventory.py +++ b/plugins/inventory/nb_inventory.py @@ -1333,7 +1333,9 @@ def refresh_services(self): api_url=url, query_key="parent_object_id", # Query only affected devices and vms and sanitize the list to only contain every ID once - query_values=set(chain(self.vms_lookup.keys(), self.devices_lookup.keys())) + query_values=set( + chain(self.vms_lookup.keys(), self.devices_lookup.keys()) + ), ) # Construct a dictionary of dictionaries, separately for devices and vms. From f4fe55c9e47d69ca5887f66d429eaeed8749fcfa Mon Sep 17 00:00:00 2001 From: Stoffel Date: Tue, 8 Jul 2025 21:06:03 +0200 Subject: [PATCH 04/12] Adding debug print to false green tests --- .../integration/targets/inventory-v4.2/compare_inventory_json.py | 1 + .../integration/targets/inventory-v4.3/compare_inventory_json.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/integration/targets/inventory-v4.2/compare_inventory_json.py b/tests/integration/targets/inventory-v4.2/compare_inventory_json.py index ccdf8bd5..817aa8f8 100755 --- a/tests/integration/targets/inventory-v4.2/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.2/compare_inventory_json.py @@ -141,6 +141,7 @@ def main(): if result: # Dictionary is not empty - print differences + print(result) print(json.dumps(result, sort_keys=True, indent=4)) sys.exit(1) else: diff --git a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py index ccdf8bd5..817aa8f8 100755 --- a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py @@ -141,6 +141,7 @@ def main(): if result: # Dictionary is not empty - print differences + print(result) print(json.dumps(result, sort_keys=True, indent=4)) sys.exit(1) else: From 869d61b3ec72ead8242c522126740cce8f8619b8 Mon Sep 17 00:00:00 2001 From: Stoffel Date: Tue, 8 Jul 2025 21:21:25 +0200 Subject: [PATCH 05/12] turn DeepDiff object into standard dict to enable serialization --- .../targets/inventory-v4.2/compare_inventory_json.py | 3 +-- .../targets/inventory-v4.3/compare_inventory_json.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration/targets/inventory-v4.2/compare_inventory_json.py b/tests/integration/targets/inventory-v4.2/compare_inventory_json.py index 817aa8f8..c2363aae 100755 --- a/tests/integration/targets/inventory-v4.2/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.2/compare_inventory_json.py @@ -141,8 +141,7 @@ def main(): if result: # Dictionary is not empty - print differences - print(result) - print(json.dumps(result, sort_keys=True, indent=4)) + print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) sys.exit(1) else: # Success, no differences diff --git a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py index 817aa8f8..c2363aae 100755 --- a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py @@ -141,8 +141,7 @@ def main(): if result: # Dictionary is not empty - print differences - print(result) - print(json.dumps(result, sort_keys=True, indent=4)) + print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) sys.exit(1) else: # Success, no differences From 2ae9fda1a76af7bcfc5444ef613bcef1b9e6d6b5 Mon Sep 17 00:00:00 2001 From: Stoffel Date: Tue, 8 Jul 2025 21:40:44 +0200 Subject: [PATCH 06/12] Using DeepDiff native to_json() --- .../targets/inventory-v4.2/compare_inventory_json.py | 3 ++- .../targets/inventory-v4.3/compare_inventory_json.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/targets/inventory-v4.2/compare_inventory_json.py b/tests/integration/targets/inventory-v4.2/compare_inventory_json.py index c2363aae..d9ba77dd 100755 --- a/tests/integration/targets/inventory-v4.2/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.2/compare_inventory_json.py @@ -141,7 +141,8 @@ def main(): if result: # Dictionary is not empty - print differences - print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) + print(result.to_json()) + #print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) sys.exit(1) else: # Success, no differences diff --git a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py index c2363aae..82453dfe 100755 --- a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py @@ -141,7 +141,8 @@ def main(): if result: # Dictionary is not empty - print differences - print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) + print(result.to_json(indent=4)) + #print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) sys.exit(1) else: # Success, no differences From 7b325e0c03b72ae92df1ca28608013b69817d5c6 Mon Sep 17 00:00:00 2001 From: Stoffel Date: Tue, 8 Jul 2025 21:44:05 +0200 Subject: [PATCH 07/12] Linting fixes... --- .../targets/inventory-v4.2/compare_inventory_json.py | 2 +- .../targets/inventory-v4.3/compare_inventory_json.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/targets/inventory-v4.2/compare_inventory_json.py b/tests/integration/targets/inventory-v4.2/compare_inventory_json.py index d9ba77dd..ab3e8568 100755 --- a/tests/integration/targets/inventory-v4.2/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.2/compare_inventory_json.py @@ -142,7 +142,7 @@ def main(): if result: # Dictionary is not empty - print differences print(result.to_json()) - #print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) + # print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) sys.exit(1) else: # Success, no differences diff --git a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py index 82453dfe..a50ea199 100755 --- a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py @@ -142,7 +142,7 @@ def main(): if result: # Dictionary is not empty - print differences print(result.to_json(indent=4)) - #print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) + # print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) sys.exit(1) else: # Success, no differences From c70764450a9b44469f689389ae191df6b0a708cc Mon Sep 17 00:00:00 2001 From: Stoffel Date: Tue, 8 Jul 2025 22:03:34 +0200 Subject: [PATCH 08/12] enabling json output during inventory tests --- .../targets/inventory-v4.1/compare_inventory_json.py | 2 +- .../targets/inventory-v4.2/compare_inventory_json.py | 3 +-- .../targets/inventory-v4.3/compare_inventory_json.py | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration/targets/inventory-v4.1/compare_inventory_json.py b/tests/integration/targets/inventory-v4.1/compare_inventory_json.py index ccdf8bd5..20b39e83 100755 --- a/tests/integration/targets/inventory-v4.1/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.1/compare_inventory_json.py @@ -141,7 +141,7 @@ def main(): if result: # Dictionary is not empty - print differences - print(json.dumps(result, sort_keys=True, indent=4)) + print(result.to_json(indent=4)) sys.exit(1) else: # Success, no differences diff --git a/tests/integration/targets/inventory-v4.2/compare_inventory_json.py b/tests/integration/targets/inventory-v4.2/compare_inventory_json.py index ab3e8568..20b39e83 100755 --- a/tests/integration/targets/inventory-v4.2/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.2/compare_inventory_json.py @@ -141,8 +141,7 @@ def main(): if result: # Dictionary is not empty - print differences - print(result.to_json()) - # print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) + print(result.to_json(indent=4)) sys.exit(1) else: # Success, no differences diff --git a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py index a50ea199..20b39e83 100755 --- a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py @@ -142,7 +142,6 @@ def main(): if result: # Dictionary is not empty - print differences print(result.to_json(indent=4)) - # print(json.dumps(result.to_dict(), sort_keys=True, indent=4)) sys.exit(1) else: # Success, no differences From a5495ec43e9588be25e2a7c1371b626387c52a57 Mon Sep 17 00:00:00 2001 From: Stoffel Date: Tue, 8 Jul 2025 22:23:37 +0200 Subject: [PATCH 09/12] Adding collected inventory to debug output to easier incorporate changes --- .../targets/inventory-v4.3/compare_inventory_json.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py index 20b39e83..aa293e0a 100755 --- a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py @@ -141,6 +141,8 @@ def main(): if result: # Dictionary is not empty - print differences + print(json.dumps(data_b, sort_keys=True, indent=4)) + print("===================== differences") print(result.to_json(indent=4)) sys.exit(1) else: From d14c87d7fa37168ea6c88bfefbf4a592bdccb8d2 Mon Sep 17 00:00:00 2001 From: Stoffel Date: Wed, 9 Jul 2025 22:42:48 +0200 Subject: [PATCH 10/12] Updating code to be backwards compatible --- changelogs/fragments/nb_inventory.yml | 1 + plugins/inventory/nb_inventory.py | 41 +++++++++++++++++++++------ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/changelogs/fragments/nb_inventory.yml b/changelogs/fragments/nb_inventory.yml index 8e5e470c..e473f7db 100644 --- a/changelogs/fragments/nb_inventory.yml +++ b/changelogs/fragments/nb_inventory.yml @@ -1,2 +1,3 @@ bugfixes: - nb_inventory - Fixed empty inventory results when netbox server URL is a non-root path + - 1438 nb_inventory - Fix service collection for version greater than 4.3 \ No newline at end of file diff --git a/plugins/inventory/nb_inventory.py b/plugins/inventory/nb_inventory.py index 8bbb8a86..cbac91ab 100644 --- a/plugins/inventory/nb_inventory.py +++ b/plugins/inventory/nb_inventory.py @@ -1328,7 +1328,7 @@ def refresh_services(self): if self.fetch_all: services = self.get_resource_list(url) - else: + elif self.api_version >= version.parse("4.3.0"): services = self.get_resource_list_chunked( api_url=url, query_key="parent_object_id", @@ -1337,6 +1337,18 @@ def refresh_services(self): chain(self.vms_lookup.keys(), self.devices_lookup.keys()) ), ) + else: + device_services = self.get_resource_list_chunked( + api_url=url, + query_key="device_id", + query_values=self.devices_lookup.keys(), + ) + vm_services = self.get_resource_list_chunked( + api_url=url, + query_key="virtual_machine_id", + query_values=self.vms_lookup.keys(), + ) + services = chain(device_services, vm_services) # Construct a dictionary of dictionaries, separately for devices and vms. # Allows looking up services by device id or vm id @@ -1346,15 +1358,26 @@ def refresh_services(self): for service in services: service_id = service["id"] - if service.get("parent_object_type") == "dcim.device": - self.device_services_lookup[service["parent_object_id"]][ - service_id - ] = service + if self.api_version >= version.parse("4.3.0"): + if service.get("parent_object_type") == "dcim.device": + self.device_services_lookup[service["parent_object_id"]][ + service_id + ] = service - if service.get("parent_object_type") == "virtualization.virtualmachine": - self.vm_services_lookup[service["parent_object_id"]][ - service_id - ] = service + if service.get("parent_object_type") == "virtualization.virtualmachine": + self.vm_services_lookup[service["parent_object_id"]][ + service_id + ] = service + else: + if service.get("device"): + self.device_services_lookup[service["device"]["id"]][ + service_id + ] = service + + if service.get("virtual_machine"): + self.vm_services_lookup[service["virtual_machine"]["id"]][ + service_id + ] = service def refresh_virtual_disks(self): url_vm_virtual_disks = ( From 841e00d24a5fb89f7c82c8ca74c630c74769c433 Mon Sep 17 00:00:00 2001 From: Stoffel Date: Wed, 9 Jul 2025 23:38:10 +0200 Subject: [PATCH 11/12] Removing debug output from tests --- changelogs/fragments/1438-fix-inventory-service-collection.yml | 2 ++ changelogs/fragments/nb_inventory.yml | 3 +-- .../targets/inventory-v4.3/compare_inventory_json.py | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 changelogs/fragments/1438-fix-inventory-service-collection.yml diff --git a/changelogs/fragments/1438-fix-inventory-service-collection.yml b/changelogs/fragments/1438-fix-inventory-service-collection.yml new file mode 100644 index 00000000..5316fb05 --- /dev/null +++ b/changelogs/fragments/1438-fix-inventory-service-collection.yml @@ -0,0 +1,2 @@ +bugfixes: + - nb_inventory - Fix service collection for version greater than 4.3 \ No newline at end of file diff --git a/changelogs/fragments/nb_inventory.yml b/changelogs/fragments/nb_inventory.yml index e473f7db..6342c3b2 100644 --- a/changelogs/fragments/nb_inventory.yml +++ b/changelogs/fragments/nb_inventory.yml @@ -1,3 +1,2 @@ bugfixes: - - nb_inventory - Fixed empty inventory results when netbox server URL is a non-root path - - 1438 nb_inventory - Fix service collection for version greater than 4.3 \ No newline at end of file + - nb_inventory - Fixed empty inventory results when netbox server URL is a non-root path \ No newline at end of file diff --git a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py index aa293e0a..20b39e83 100755 --- a/tests/integration/targets/inventory-v4.3/compare_inventory_json.py +++ b/tests/integration/targets/inventory-v4.3/compare_inventory_json.py @@ -141,8 +141,6 @@ def main(): if result: # Dictionary is not empty - print differences - print(json.dumps(data_b, sort_keys=True, indent=4)) - print("===================== differences") print(result.to_json(indent=4)) sys.exit(1) else: From 7419bc097d9649bd5edc47d4c95ee3ee4c4b0ff2 Mon Sep 17 00:00:00 2001 From: Stoffel Date: Wed, 9 Jul 2025 23:43:41 +0200 Subject: [PATCH 12/12] Linting fixes --- changelogs/fragments/1438-fix-inventory-service-collection.yml | 2 +- changelogs/fragments/nb_inventory.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changelogs/fragments/1438-fix-inventory-service-collection.yml b/changelogs/fragments/1438-fix-inventory-service-collection.yml index 5316fb05..a908e39b 100644 --- a/changelogs/fragments/1438-fix-inventory-service-collection.yml +++ b/changelogs/fragments/1438-fix-inventory-service-collection.yml @@ -1,2 +1,2 @@ bugfixes: - - nb_inventory - Fix service collection for version greater than 4.3 \ No newline at end of file + - nb_inventory - Fix service collection for version greater than 4.3 diff --git a/changelogs/fragments/nb_inventory.yml b/changelogs/fragments/nb_inventory.yml index 6342c3b2..8e5e470c 100644 --- a/changelogs/fragments/nb_inventory.yml +++ b/changelogs/fragments/nb_inventory.yml @@ -1,2 +1,2 @@ bugfixes: - - nb_inventory - Fixed empty inventory results when netbox server URL is a non-root path \ No newline at end of file + - nb_inventory - Fixed empty inventory results when netbox server URL is a non-root path