Fix Ubuntu arp settings

roles/arp/defaults/main.yml

@@ -0,0 +1,31 @@
+---
+# Don't reboot for arp config changes
+arp_arp_apply_config: false
+
+# Permissable values for arp_ignore (https://sysctl-explorer.net/net/ipv4/arp_ignore/):
+#   0 - (default): reply for any local target IP address, configured on any interface
+#   1 - reply only if the target IP address is local address configured on the incoming interface
+#   2 - reply only if the target IP address is local address configured on the incoming interface
+#       and both with the sender’s IP address are part from same subnet on this interface
+#   3 - do not reply for local addresses configured with scope host, only resolutions for global
+#       and link addresses are replied
+#   4-7 - reserved
+#   8 - do not reply for all local addresses
+arp_arp_ignore_default: 1
+
+# Permissable values for arp_announce (https://sysctl-explorer.net/net/ipv4/arp_announce/):
+# 0 - (ubuntu default) Use any local address, configured on any interface
+# 1 - Try to avoid local addresses that are not in the target’s subnet for this interface.
+#     This mode is useful when target hosts reachable via this interface require the source
+#     IP address in ARP requests to be part of their logical network configured on the
+#     receiving interface. When we generate the request we will check all our subnets that
+#     include the target IP and will preserve the source address if it is from such subnet.
+#     If there is no such subnet we select source address according to the rules for level 2.
+# 2 - Always use the best local address for this target. In this mode we ignore the source
+#     address in the IP packet and try to select local address that we prefer for talks with
+#     the target host. Such local address is selected by looking for primary IP addresses on
+#     all our subnets on the outgoing interface that include the target IP address.
+#     If no suitable local address is found we select the first local address we have on the
+#     outgoing interface or on all other interfaces, with the hope we will receive reply for
+#     our request and even sometimes no matter the source IP address we announce.
+arp_arp_announce_default: 2

roles/arp/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+# handlers file for stackhpc.linux.arp
+- name: Restart for arp change
+  ansible.builtin.reboot:
+    msg: "Rebooting on arp change"
+  when: arp_arp_apply_config

roles/arp/tasks/main.yml

@@ -0,0 +1,19 @@
+---
+# By default, Ubuntu's configuration permits:
+#   - Responding to ARP requests with addresses from other interfaces (arp_filter=0)
+#   - Reply for any local target IP address, configured on any interface (arp_ignore=0)
+# This behaviour breaks RoCE when a host has multiple interfaces in the same network.
+- name: Configure sysctl with ARP settings
+  ansible.builtin.copy:
+    dest: /etc/sysctl.d/15-fixarp.conf
+    content: |
+      net.ipv4.conf.all.arp_ignore={{ arp_arp_ignore | default(arp_arp_ignore_default, true) }}
+      net.ipv4.conf.all.arp_announce={{ arp_arp_announce | default(arp_arp_announce_default, true) }}
+    mode: "0644"
+    owner: root
+    group: root
+  when:
+    - ansible_facts.os_family == "Debian"
+    - ansible_facts.distribution == "Ubuntu"
+  notify:
+    - Restart for arp change