Add numerical comparator base class and L1 comparator

Differential Revision: D76746854

Pull Request resolved: #11751

Author: Juntian777

devtools/inspector/numerical_comparator/TARGETS

@@ -0,0 +1,30 @@
+load("@fbcode_macros//build_defs:python_library.bzl", "python_library")
+
+oncall("executorch")
+
+
+python_library(
+    name = "numerical_comparator_base",
+    srcs = ["numerical_comparator_base.py"],
+    deps = [],
+)
+
+
+python_library(
+    name = "l1_numerical_comparator",
+    srcs = ["l1_numerical_comparator.py"],
+    deps = [
+        "//executorch/devtools/inspector/numerical_comparator:numerical_comparator_base",
+        "//executorch/devtools/inspector:lib",
+    ],
+)
+
+
+
+python_library(
+    name = "lib",
+    srcs = ["__init__.py"],
+    deps = [
+        ":l1_numerical_comparator",
+    ],
+)

devtools/inspector/numerical_comparator/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+from executorch.devtools.inspector.numerical_comparator.l1_numerical_comparator import (
+    L1Comparator,
+)
+
+
+__all__ = ["L1Comparator"]

devtools/inspector/numerical_comparator/inspector_numerical_comparator_base.py

@@ -0,0 +1,28 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# pyre-unsafe
+
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class InspectorNumericalComparatorBase(ABC):
+    @abstractmethod
+    def compare(self, a: Any, b: Any) -> float:
+        """Compare two intermediate output and return a result.
+
+        This method should be overridden by subclasses to provide custom comparison logic.
+
+        Args:
+            a: The first intermediate output to compare.
+            b: The second intermediate output to compare.
+
+        Returns:
+            A numerical result indicating the comparison outcome.
+        """
+        pass

devtools/inspector/numerical_comparator/l1_numerical_comparator.py

@@ -0,0 +1,30 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Any
+
+import torch
+from executorch.devtools.inspector._inspector_utils import convert_to_float_tensor
+from executorch.devtools.inspector.numerical_comparator.numerical_comparator_base import (
+    NumericalComparatorBase,
+)
+
+
+class L1Comparator(NumericalComparatorBase):
+    def compare(self, a: Any, b: Any) -> float:
+        """Sum up all these element-wise absolute differences between two tensors."""
+
+        t_a = convert_to_float_tensor(a)
+        t_b = convert_to_float_tensor(b)
+        if torch.isnan(t_a).any() or torch.isnan(t_b).any():
+            t_a = torch.nan_to_num(t_a)
+            t_b = torch.nan_to_num(t_b)
+
+        try:
+            res = torch.abs(t_a - t_b).sum().item()
+        except Exception as e:
+            raise ValueError(f"Error computing L1 difference between tensors: {str(e)}")
+        return res

devtools/inspector/numerical_comparator/numerical_comparator_base.py

@@ -0,0 +1,26 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class NumericalComparatorBase(ABC):
+    @abstractmethod
+    def compare(self, a: Any, b: Any) -> float:
+        """Compare two intermediate output and return a result.
+
+        This method should be overridden by subclasses to provide custom comparison logic.
+
+        Args:
+            a: The first intermediate output to compare.
+            b: The second intermediate output to compare.
+
+        Returns:
+            A numerical result indicating the comparison outcome.
+        """
+        pass

devtools/inspector/tests/TARGETS

@@ -54,6 +54,14 @@ python_unittest(
     ],
 )
 
+python_unittest(
+    name = "l1_comparator_test",
+    srcs = ["l1_comparator_test.py"],
+    deps = [
+        "//executorch/devtools/inspector/numerical_comparator:lib",
+    ],
+)
+
 python_library(
     name = "inspector_test_utils",
     srcs = [

devtools/inspector/tests/l1_comparator_test.py

@@ -0,0 +1,56 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import unittest
+
+import torch
+
+from executorch.devtools.inspector.numerical_comparator import L1Comparator
+
+
+class TestL1Comparator(unittest.TestCase):
+    l1_comparator = L1Comparator()
+
+    def test_identical_tensors(self):
+        a = torch.tensor([[1, 2], [3, 4]])
+        b = torch.tensor([[1, 2], [3, 4]])
+        expected = 0.0
+        result = self.l1_comparator.compare(a, b)
+        self.assertAlmostEqual(result, expected)
+
+    def test_scalar(self):
+        a = 1
+        b = 2
+        expected = 1.0
+        result = self.l1_comparator.compare(a, b)
+        self.assertAlmostEqual(result, expected)
+
+    def test_with_nans_replaced_with_zero(self):
+        a = torch.tensor([3, 2, -1, float("nan")])
+        b = torch.tensor([float("nan"), 0, -3, 1])
+        expected = 8.0
+        result = self.l1_comparator.compare(a, b)
+        self.assertAlmostEqual(result, expected)
+
+    def test_shape_mismatch_raises_exception(self):
+        a = torch.tensor([0, 2, -1])
+        b = torch.tensor([1, 0, -3, 4])
+        with self.assertRaises(ValueError):
+            self.l1_comparator.compare(a, b)
+
+    def test_2D_tensors(self):
+        a = torch.tensor([[4, 9], [6, 4]])
+        b = torch.tensor([[1, 2], [3, 5]])
+        expected = 14.0
+        result = self.l1_comparator.compare(a, b)
+        self.assertAlmostEqual(result, expected)
+
+    def test_list_of_tensors(self):
+        a = [torch.tensor([2, 4]), torch.tensor([5, 2])]
+        b = [torch.tensor([1, 2]), torch.tensor([3, 5])]
+        expected = 8.0
+        result = self.l1_comparator.compare(a, b)
+        self.assertAlmostEqual(result, expected)