Fix: Enforce deterministic SDFG graph traversal and code generation

dace/codegen/codegen.py

@@ -245,6 +245,9 @@ def generate_code(sdfg: SDFG, validate=True) -> List[CodeObject]:
         for k, v in frame._dispatcher.instrumentation.items()
     }
 
+    # Sort SDFG dictionaries for deterministic pattern matching.
+    sdfg.sort_sdfg_alphabetically()
+
     # NOTE: THE SDFG IS ASSUMED TO BE FROZEN (not change) FROM THIS POINT ONWARDS
 
     # Generate frame code (and the rest of the code)

dace/config_schema.yml

@@ -251,6 +251,17 @@ required:
                     If set, specifies additional arguments to the initial invocation
                     of ``cmake``.
 
+            sdfg_alphabetical_sorting:
+                type: bool
+                default: false
+                title: SDFG alphabetical sorting
+                description: >
+                    When enabled, sorts all internal SDFG dictionaries
+                    (nodes, edges, arrays, symbols) into a canonical
+                    alphabetical order before code generation. This
+                    guarantees reproducible output across consecutive
+                    compilations at the cost of additional sorting overhead.
+
             #############################################
             # CPU compiler
             cpu:

dace/sdfg/sdfg.py

@@ -3006,3 +3006,76 @@ def recheck_using_explicit_control_flow(self) -> bool:
                 break
         self.root_sdfg.using_explicit_control_flow = found_explicit_cf_block
         return found_explicit_cf_block
+
+    def sort_sdfg_alphabetically(self, rebuild_nx: bool = False, visited: Optional[Set[int]] = None) -> None:
+        """
+        Forces all internal dictionaries, graph structures, and metadata registries
+        into a deterministic, semantically-aware order to guarantee stable code generation.
+
+        In DaCe, the code generators rely heavily on the iteration order
+        of internal dictionaries. This method performs a deep, in-place stabilization
+        of the entire SDFG hierarchy to eliminate stochastic compilation jitter
+        caused by memory addresses or volatile UUIDs.
+
+        The stabilization process executes in four phases:
+            1. Global Metadata: Alphabetizes arrays, symbols, and constants.
+            2. State Machine: Sorts the top-level SDFG (States and Interstate Edges)
+               using semantic topological keys.
+            3. Dataflow: Sorts the internal nodes and memlet edges within every State.
+            4. Recursion: Recursively applies this stabilization to all Nested SDFGs.
+
+        This method is a no-op unless the ``compiler.sdfg_alphabetical_sorting`` configuration
+        option is set to ``True``.
+
+        :param rebuild_nx: If True, rebuilds the internal NetworkX graph in each
+                           sorted graph. Default is False for performance, since
+                           DaCe's codegen and pattern matching do not rely on the
+                           internal _nx iteration order.
+        :param visited: A set of memory addresses (IDs) of already processed SDFGs.
+                        Used internally to prevent infinite recursion in the event
+                        of cyclic nested SDFG references. Also serves as the signal
+                        for the top-level entry point: the node key cache is only
+                        cleared when ``visited`` is ``None`` (i.e., on the first call),
+                        not on recursive calls into nested SDFGs where parent keys
+                        are still valid.
+        """
+        # Only perform sorting when deterministic code generation is enabled.
+        if not Config.get_bool('compiler', 'sdfg_alphabetical_sorting'):
+            return
+
+        # Avoid import loops
+        from dace.sdfg.utils import sort_graph_dicts_alphabetically, _node_key_cache
+
+        if visited is None:
+            visited = set()
+            # Only clear the cache at the top-level entry point, not on
+            # recursive calls into nested SDFGs where parent keys are
+            # still valid.
+            _node_key_cache.clear()
+
+        # Cycle prevention for recursive nested SDFGs
+        if id(self) in visited:
+            return
+        visited.add(id(self))
+
+        # 1. Stabilize Global Metadata (Arrays, Symbols, Constants)
+        for attr in ['_arrays', 'symbols', 'constants_prop']:
+            if hasattr(self, attr):
+                val = getattr(self, attr)
+                # Ensure the attribute exists, is dict-like, and supports clear/update
+                if val and hasattr(val, 'keys') and hasattr(val, 'clear'):
+                    sorted_items = sorted(val.items(), key=lambda item: item[0])
+                    val.clear()
+                    val.update(sorted_items)
+
+        # 2. Stabilize the top-level State Machine (States and Interstate edges)
+        sort_graph_dicts_alphabetically(self, rebuild_nx=rebuild_nx)
+
+        # 3. Stabilize the Dataflow inside each State and recurse into Nested SDFGs
+        for state in self.nodes():
+            sort_graph_dicts_alphabetically(state, rebuild_nx=rebuild_nx)
+
+            # 4. Recurse into Nested SDFGs
+            for node in state.nodes():
+                if hasattr(node, 'sdfg') and node.sdfg is not None:
+                    node.sdfg.sort_sdfg_alphabetically(rebuild_nx=rebuild_nx, visited=visited)

dace/sdfg/utils.py

@@ -7,6 +7,8 @@
 import warnings
 import networkx as nx
 import time
+import re
+import hashlib
 
 import dace.sdfg.nodes
 from dace.codegen import compiled_sdfg as csdfg
@@ -2762,3 +2764,201 @@ def expand_nodes(sdfg: SDFG, predicate: Callable[[nd.Node], bool]):
 
         if expanded_something:
             states.append(state)
+
+
+# Module-level cache for node keys, cleared at the start of each
+# sort_sdfg_alphabetically() call to prevent stale entries after
+# transformations modify the graph structure. Keyed by id(node).
+_node_key_cache: Dict[int, str] = {}
+
+
+def _resolve_degree(node: Any, attr: str) -> int:
+    """
+    Safely resolves in_degree or out_degree from a node, handling both
+    the case where it is a plain attribute (int) and where it is a bound
+    method (as on NetworkX nodes or DaCe Graph objects).
+
+    :param node: The graph node to inspect.
+    :param attr: The attribute name ('in_degree' or 'out_degree').
+    :return: The integer degree value, or 0 if the attribute is absent.
+    """
+    val = getattr(node, attr, None)
+    if val is None:
+        return 0
+    if callable(val):
+        try:
+            return val()
+        except TypeError:
+            return 0
+    return val
+
+
+def get_deterministic_node_key(node: Any) -> str:
+    """
+    Generates a highly stable, deterministic string key for DaCe graph nodes
+    based on their semantic properties rather than memory locations.
+
+    During SDFG compilation, relying on memory addresses or volatile UUIDs
+    for sorting leads to non-deterministic code generation. This function
+    extracts the intrinsic semantic identity of a node to ensure structural
+    collisions are resolved deterministically.
+
+    The generated key incorporates:
+        - Node type and label
+        - Graph topology (in-degree and out-degree)
+        - Interface semantics (in/out connectors)
+        - Loop and memory semantics (map parameters, schedules, access types)
+        - Internal code logic (via a stable MD5 hash of Tasklet code)
+        - Nested SDFG structural size
+
+    Results are cached per node identity to avoid redundant recomputation
+    during sort comparisons. The cache is invalidated at the start of each
+    sort_sdfg_alphabetically() call.
+
+    :param node: The DaCe graph node (e.g., Tasklet, AccessNode, MapEntry)
+                 to be evaluated.
+    :return: A stable string representing the node's semantic identity.
+    """
+    node_id = id(node)
+    if node_id in _node_key_cache:
+        return _node_key_cache[node_id]
+
+    node_type = type(node).__name__
+
+    # Extract core identifier
+    raw_label = getattr(node, 'data', getattr(node, 'label', str(node)))
+
+    parts = [node_type, str(raw_label)]
+
+    # 1. Topological Context
+    in_deg = _resolve_degree(node, 'in_degree')
+    out_deg = _resolve_degree(node, 'out_degree')
+    parts.append(f"i{in_deg}o{out_deg}")
+
+    # 2. Interface Semantics (Connectors for Tasklets / NestedSDFGs)
+    if hasattr(node, 'in_connectors') and node.in_connectors:
+        parts.append("inC:" + "-".join(sorted(node.in_connectors.keys())))
+    if hasattr(node, 'out_connectors') and node.out_connectors:
+        parts.append("outC:" + "-".join(sorted(node.out_connectors.keys())))
+
+    # 3. Loop Semantics (Map Parameters & Schedules)
+    if hasattr(node, 'map') and hasattr(node.map, 'params'):
+        parts.append("map:" + "-".join(node.map.params))
+        if hasattr(node.map, 'schedule'):
+            parts.append(f"sch:{str(node.map.schedule)}")
+
+    # 4. Memory Semantics (Access Types for AccessNodes)
+    if hasattr(node, 'access'):
+        parts.append(f"acc:{str(node.access)}")
+
+    # 5. Internal Code Semantics (Tasklets)
+    if hasattr(node, 'code') and hasattr(node.code, 'as_string'):
+        # Hash the code to prevent massive strings while guaranteeing uniqueness.
+        # MD5 is used because Python's built-in hash() is non-deterministic across runs.
+        code_str = str(node.code.as_string).strip()
+        code_hash = hashlib.md5(code_str.encode('utf-8')).hexdigest()[:8]
+        parts.append(f"code:{code_hash}")
+
+    # 6. Nested SDFG Differentiation
+    if hasattr(node, 'sdfg') and node.sdfg:
+        parts.append(f"nsdfg_states:{len(node.sdfg.nodes())}")
+
+    result = "_".join(parts)
+    _node_key_cache[node_id] = result
+    return result
+
+
+def get_deterministic_edge_key(edge: Any) -> str:
+    """
+    Generates a highly stable string key for graph edges to ensure
+    deterministic sorting.
+
+    This function extracts the semantic connection points (connectors)
+    and the data payload (Memlets or Interstate conditions) to prevent
+    non-deterministic compiler graph traversals.
+
+    :param edge: The DaCe graph edge (or InterstateEdge) to be evaluated.
+    :return: A stable string representation of the edge's routing and payload.
+    """
+    # 1. Extract raw strings
+    raw_src_conn = str(getattr(edge, 'src_conn', ''))
+    raw_dst_conn = str(getattr(edge, 'dst_conn', ''))
+    raw_data_str = str(getattr(edge, 'data', ''))
+
+    # 2. Retrieve the stabilized keys for the source and destination nodes
+    src_key = get_deterministic_node_key(edge.src)
+    dst_key = get_deterministic_node_key(edge.dst)
+
+    return f"{src_key}[{raw_src_conn}]->{dst_key}[{raw_dst_conn}]({raw_data_str})"
+
+
+def sort_graph_dicts_alphabetically(graph: Any, rebuild_nx: bool = False) -> None:
+    """
+    Sorts internal graph nodes, edge dictionaries, and NetworkX backends in-place
+    using semantically-aware deterministic keys.
+
+    In DaCe, the order in which code is generated heavily depends on
+    the iteration order of the graph's internal dictionaries. This function performs
+    a deep, in-place sort of these structures to guarantee that graph traversal
+    and code generation are perfectly deterministic across different executions.
+
+    The stabilization process occurs in four phases:
+        1. Alphabetizes the master `_nodes` dictionary based on semantic node keys.
+        2. Alphabetizes the nested adjacency lists (`in_edges`, `out_edges`) for
+           every node based on semantic edge keys.
+        3. Alphabetizes the master `_edges` dictionary.
+        4. (Optional) Tears down and sequentially rebuilds the underlying NetworkX
+           graph (`_nx`) so its internal node/edge registries match the newly
+           stabilized order. Skipped by default since pattern matching builds
+           its own NetworkX digraph via collapse_multigraph_to_nx.
+
+    :param graph: The DaCe graph structure (e.g., SDFGState or generic Graph)
+                  whose internal dictionaries need to be stabilized.
+    :param rebuild_nx: If True, rebuilds the internal NetworkX graph to match
+                       the new order. Default is False for performance, since
+                       DaCe's codegen and pattern matching do not rely on the
+                       internal _nx iteration order.
+    """
+    # 1. Sort the master Nodes dictionary
+    if hasattr(graph, '_nodes'):
+        sorted_node_items = sorted(graph._nodes.items(), key=lambda item: get_deterministic_node_key(item[0]))
+        graph._nodes.clear()
+        graph._nodes.update(sorted_node_items)
+
+        # 2. Sort the nested adjacency lists (In/Out Edges) within each node
+        for node, (in_edges, out_edges) in graph._nodes.items():
+            sorted_in_items = sorted(in_edges.items(), key=lambda item: get_deterministic_edge_key(item[1]))
+            in_edges.clear()
+            in_edges.update(sorted_in_items)
+
+            sorted_out_items = sorted(out_edges.items(), key=lambda item: get_deterministic_edge_key(item[1]))
+            out_edges.clear()
+            out_edges.update(sorted_out_items)
+
+    # 3. Sort the master Edges dictionary
+    if hasattr(graph, '_edges'):
+        sorted_edge_items = sorted(graph._edges.items(), key=lambda item: get_deterministic_edge_key(item[1]))
+        graph._edges.clear()
+        graph._edges.update(sorted_edge_items)
+
+    # 4. Optionally rebuild the NetworkX backend to match the new deterministic order
+    if rebuild_nx and hasattr(graph, '_nx'):
+        old_nx = graph._nx
+        graph._nx = type(old_nx)()
+
+        # Add nodes sequentially
+        for n in graph._nodes.keys():
+            graph._nx.add_node(n, **old_nx.nodes.get(n, {}))
+
+        # Add edges sequentially using the newly sorted master edge list
+        for e_obj in graph._edges.values():
+            edge_attrs = {'data': e_obj.data}
+
+            if hasattr(e_obj, 'src_conn'):
+                edge_attrs['src_conn'] = e_obj.src_conn
+                edge_attrs['dst_conn'] = e_obj.dst_conn
+
+            if hasattr(e_obj, 'key'):
+                graph._nx.add_edge(e_obj.src, e_obj.dst, key=e_obj.key, **edge_attrs)
+            else:
+                graph._nx.add_edge(e_obj.src, e_obj.dst, **edge_attrs)