From c8376b74e957785d5da7c0ad38b83f77605aeb26 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Mon, 11 Mar 2024 12:46:14 +0200 Subject: [PATCH] Speed up resource updates when there are no matching webhooks (#7553) The signal handling code in `cvat.apps.webhooks.signals` always fetches the old version of the resource from the database and serializes it. But if no webhook is defined for the current resource, this work is pointless. Select the webhooks first, and only then (if necessary) serialize the old resource. Also, move the diffing of the old and new versions to the `post_save` signal handler, to avoid serializing the new version a second time. This leads to a considerable speedup for resources with complicated serializers, such as tasks and jobs, when no webhooks are attached. In my experiments, saving of annotations was sped up by 20-25% with this patch. This patch also makes the signal handlers more hygienic with the use of temporary attributes. Both the attributes it uses are now prefixed with `_webhooks` to avoid collisions, and removed after they are used. --- ...05_171241_roman_webhooks_lazy_rendering.md | 4 ++ cvat/apps/webhooks/signals.py | 68 +++++++++++-------- 2 files changed, 45 insertions(+), 27 deletions(-) create mode 100644 changelog.d/20240305_171241_roman_webhooks_lazy_rendering.md diff --git a/changelog.d/20240305_171241_roman_webhooks_lazy_rendering.md b/changelog.d/20240305_171241_roman_webhooks_lazy_rendering.md new file mode 100644 index 000000000000..35e6eaf5f7a6 --- /dev/null +++ b/changelog.d/20240305_171241_roman_webhooks_lazy_rendering.md @@ -0,0 +1,4 @@ +### Changed + +- Sped up resource updates when there are no matching webhooks + () diff --git a/cvat/apps/webhooks/signals.py b/cvat/apps/webhooks/signals.py index 0d34950cf6ff..3e17e8f3d8f6 100644 --- a/cvat/apps/webhooks/signals.py +++ b/cvat/apps/webhooks/signals.py @@ -136,25 +136,32 @@ def get_sender(instance): @receiver(pre_save, sender=Invitation, dispatch_uid=__name__ + ":invitation:pre_save") @receiver(pre_save, sender=Membership, dispatch_uid=__name__ + ":membership:pre_save") def pre_save_resource_event(sender, instance, **kwargs): - try: - old_instance = sender.objects.get(pk=instance.pk) - except ObjectDoesNotExist: - return + instance._webhooks_selected_webhooks = [] - old_serializer = get_serializer(instance=old_instance) - serializer = get_serializer(instance=instance) - diff = get_instance_diff(old_data=old_serializer.data, data=serializer.data) + if instance.pk is None: + created = True + else: + try: + old_instance = sender.objects.get(pk=instance.pk) + created = False + except ObjectDoesNotExist: + created = True - if not diff: - return + resource_name = instance.__class__.__name__.lower() - before_update = { - attr: value["old_value"] - for attr, value in diff.items() - } + event_type = event_name("create" if created else "update", resource_name) + if event_type not in map(lambda a: a[0], EventTypeChoice.choices()): + return - instance._before_update = before_update + instance._webhooks_selected_webhooks = select_webhooks(instance, event_type) + if not instance._webhooks_selected_webhooks: + return + if created: + instance._webhooks_old_data = None + else: + old_serializer = get_serializer(instance=old_instance) + instance._webhooks_old_data = old_serializer.data @receiver(post_save, sender=Project, dispatch_uid=__name__ + ":project:post_save") @receiver(post_save, sender=Task, dispatch_uid=__name__ + ":task:post_save") @@ -164,31 +171,38 @@ def pre_save_resource_event(sender, instance, **kwargs): @receiver(post_save, sender=Organization, dispatch_uid=__name__ + ":organization:post_save") @receiver(post_save, sender=Invitation, dispatch_uid=__name__ + ":invitation:post_save") @receiver(post_save, sender=Membership, dispatch_uid=__name__ + ":membership:post_save") -def post_save_resource_event(sender, instance, created, **kwargs): - resource_name = instance.__class__.__name__.lower() +def post_save_resource_event(sender, instance, **kwargs): + selected_webhooks = instance._webhooks_selected_webhooks + del instance._webhooks_selected_webhooks - event_type = event_name("create" if created else "update", resource_name) - if event_type not in map(lambda a: a[0], EventTypeChoice.choices()): + if not selected_webhooks: return - filtered_webhooks = select_webhooks(instance, event_type) - if not filtered_webhooks: - return + old_data = instance._webhooks_old_data + del instance._webhooks_old_data + + created = old_data is None + + resource_name = instance.__class__.__name__.lower() + event_type = event_name("create" if created else "update", resource_name) + + serializer = get_serializer(instance=instance) data = { "event": event_type, - resource_name: get_serializer(instance=instance).data, + resource_name: serializer.data, "sender": get_sender(instance), } if not created: - if before_update := getattr(instance, "_before_update", None): - data["before_update"] = before_update - else: - return + if diff := get_instance_diff(old_data=old_data, data=serializer.data): + data["before_update"] = { + attr: value["old_value"] + for attr, value in diff.items() + } transaction.on_commit( - lambda: batch_add_to_queue(filtered_webhooks, data), + lambda: batch_add_to_queue(selected_webhooks, data), robust=True, )