Merge pull request #430 from linguini-dev/feature/gcp-general-label-handling

Adds general label handling for GCP

Author: asmorodskyi

ocw/lib/gce.py

@@ -56,6 +56,16 @@ def _delete_resource(self, api_call, resource_name, *_, **kwargs) -> None:
             self.compute_client().routes: "route",
             self.compute_client().subnetworks: "subnetwork",
         }.get(api_call, "resource")
+
+        # Get object details / metadata and check the labels
+        resource_details = api_call().get(**kwargs).execute()
+        labels = resource_details.get('labels', {})
+        if labels:
+            self.log_dbg(f"Resource {resource_type}/{resource_name} has these labels: {labels}")
+            if 'pcw_ignore' in labels:
+                self.log_info(f"Skipping deletion of {resource_type} {resource_name} due to 'pcw_ignore' label set on resource")
+                return
+
         if self.dry_run:
             self.log_info(f"Deletion of {resource_type} {resource_name} skipped due to dry run mode")
             return
@@ -177,29 +187,19 @@ def cleanup_disks(self) -> None:
                 self.log_dbg(f"{len(disks)} disks found")
                 for disk in disks:
                     if self.is_outdated(parse(disk["creationTimestamp"]).astimezone(timezone.utc)):
-                        labels = disk.get('labels', [])
-                        pcw_ignore_tag = 'pcw_ignore' in labels
-                        if pcw_ignore_tag:
-                            self.log_dbg(f"Ignoring {disk['name']} due to 'pcw_ignore' label set to '1'")
-                        else:
-                            self._delete_resource(
-                                self.compute_client().disks, disk["name"], project=self.project, zone=zone, disk=disk["name"]
-                            )
+                        self._delete_resource(
+                            self.compute_client().disks, disk["name"], project=self.project, zone=zone, disk=disk["name"]
+                        )
 
     def cleanup_images(self) -> None:
         self.log_dbg("Images cleanup")
         images = self._paginated(self.compute_client().images, project=self.project)
         self.log_dbg(f"{len(images)} images found")
         for image in images:
             if self.is_outdated(parse(image["creationTimestamp"]).astimezone(timezone.utc)):
-                labels = image.get('labels', [])
-                pcw_ignore_tag = 'pcw_ignore' in labels
-                if pcw_ignore_tag:
-                    self.log_dbg(f"Ignoring {image['name']} due to 'pcw_ignore' label set to '1'")
-                else:
-                    self._delete_resource(
-                        self.compute_client().images, image["name"], project=self.project, image=image["name"]
-                    )
+                self._delete_resource(
+                    self.compute_client().images, image["name"], project=self.project, image=image["name"]
+                )
 
     def cleanup_firewalls(self) -> None:
         self.log_dbg("Firewalls cleanup")

tests/test_gce.py

@@ -50,11 +50,8 @@ def delete(self, *args, **kwargs):
                 return MockRequest(None)
         raise ValueError("Unexpected delete request")
 
-    def get(self, project, region):
-        def something():
-            pass
-        something.execute = lambda: self.responses
-        return something
+    def get(self, *args, **kwargs):
+        return MockRequest()
 
 
 class MockClient:
@@ -111,6 +108,13 @@ def mocked_resource():
             ], 'id': "id"}),
         MockRequest({'error': {'errors': [{'message': 'err message'}]},
                     'warnings': [{'message': 'warning message'}]}),
+        MockRequest({
+            'items': [
+                {'name': 'pcw_ignore', 'creationTimestamp': older_than_max_age, 'timeCreated': older_than_max_age,
+                 'network': 'mynetwork', 'labels': {'pcw_ignore': '1'}}
+            ], 'id': "id"
+        }),
+        MockRequest(),  # on images().delete()
         None   # on images().list_next()
     ])
 
@@ -142,7 +146,14 @@ def test_list_regions(gce):
 def test_list_zones(gce):
     gce.compute_client.regions = MockResource({'zones': ['somethingthatIdonotknow/RabbitHole']})
     gce.compute_client.list_zones = {'zones': ['somethingthatIdonotknow/RabbitHole']}
-    assert gce.list_zones('Oxfordshire') == ['RabbitHole']
+
+    mock_region_response = {
+        'zones': ['somethingthatIdonotknow/RabbitHole']
+    }
+
+    # Mock the regions().get() method to return the mock response
+    with patch.object(gce.compute_client.regions(), 'get', return_value=MockRequest(mock_region_response)):
+        assert gce.list_zones('Oxfordshire') == ['RabbitHole']
 
 
 def _test_cleanup(gce, resource_type, cleanup_call, resources):
@@ -156,8 +167,10 @@ def _test_cleanup(gce, resource_type, cleanup_call, resources):
         cleanup_call()
         if gce.dry_run:
             assert resources.deleted_resources == []
-        else:
+        elif resources.get("labels") == "pcw_ignore":
             assert resources.deleted_resources == ['delete1', 'delete2']
+        else:
+            assert resources.deleted_resources == ['delete1', 'delete2', 'pcw_ignore']
 
 
 @mark.parametrize("dry_run", [True, False])
@@ -222,6 +235,12 @@ def test_cleanup_networks(gce, mocked_resource, dry_run):
     _test_cleanup(gce, "networks", gce.cleanup_networks, mocked_resource)
 
 
+@mark.parametrize("dry_run", [True, False])
+def test_pcw_ignore_label(gce, mocked_resource, dry_run):
+    gce.dry_run = dry_run
+    _test_cleanup(gce, "images", gce.cleanup_images, mocked_resource)
+
+
 def test_cleanup_all(gce):
     gce.cleanup_blobs = MagicMock()
     gce.cleanup_disks = MagicMock()