[FROM-ML] IOMMU Performance Optimization support

drivers/gpu/drm/amd/amdgpu/amdgpu.h

@@ -156,6 +156,7 @@ struct amdgpu_watchdog_timer {
  * Modules parameters.
  */
 extern int amdgpu_modeset;
+extern int amdgpu_iommu_perfopt;
 extern unsigned int amdgpu_vram_limit;
 extern int amdgpu_vis_vram_limit;
 extern int amdgpu_gart_size;

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

@@ -33,6 +33,7 @@
 #include <linux/console.h>
 #include <linux/slab.h>
 #include <linux/iommu.h>
+#include <linux/amd-iommu.h>
 #include <linux/pci.h>
 #include <linux/pci-p2pdma.h>
 #include <linux/apple-gmux.h>
@@ -3752,6 +3753,17 @@ amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px)
 		       apple_gmux_detect(NULL, NULL)));
 }
 
+static inline bool amdgpu_device_identity(struct amdgpu_device *adev)
+{
+	struct pci_dev *pdev = adev->pdev;
+	struct iommu_domain *domain = iommu_get_domain_for_dev(&pdev->dev);
+
+	if (!domain)
+		return false;
+
+	return domain->type == IOMMU_DOMAIN_IDENTITY;
+}
+
 /**
  * amdgpu_device_init - initialize the driver
  *
@@ -3958,6 +3970,18 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	if (r)
 		return r;
 
+	if (amdgpu_iommu_perfopt != 0 &&
+	    amdgpu_device_identity(adev) &&
+	    adev->flags & AMD_IS_APU) {
+		int perfopt_ret = amd_iommu_enable_perfopt(pdev);
+
+		/* Optional optimization; a failure to arm it must not abort probe. */
+		if (perfopt_ret)
+			dev_warn(adev->dev,
+				 "Failed to enable IOMMU PerfOpt (%d); continuing without it\n",
+				 perfopt_ret);
+	}
+
 	/*
 	 * No need to remove conflicting FBs for non-display class devices.
 	 * This prevents the sysfb from being freed accidently.
@@ -4327,6 +4351,9 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
 
 	amdgpu_gart_dummy_page_fini(adev);
 
+	if (amdgpu_iommu_perfopt != 0)
+		amd_iommu_disable_perfopt(adev->pdev);
+
 	if (pci_dev_is_disconnected(adev->pdev))
 		amdgpu_device_unmap_mmio(adev);
 
@@ -4692,6 +4719,20 @@ int amdgpu_device_resume(struct drm_device *dev, bool notify_clients)
 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
+	if (amdgpu_iommu_perfopt != 0 && amdgpu_device_identity(adev)) {
+		int perfopt_ret = amd_iommu_enable_perfopt(adev->pdev);
+
+		/*
+		 * Must not return on failure: a bare return would leak the
+		 * SR-IOV VF exclusive-mode acquisition taken above (released
+		 * via the exit: path).
+		 */
+		if (perfopt_ret)
+			dev_warn(adev->dev,
+				 "Failed to enable IOMMU PerfOpt (%d); continuing without it\n",
+				 perfopt_ret);
+	}
+
 	if (adev->in_s0ix)
 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
 

drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c

@@ -185,6 +185,7 @@ char *amdgpu_disable_cu;
 char *amdgpu_virtual_display;
 int amdgpu_enforce_isolation = -1;
 int amdgpu_modeset = -1;
+int amdgpu_iommu_perfopt = 1;
 
 /* Specifies the default granularity for SVM, used in buffer
  * migration and restoration of backing memory when handling
@@ -392,6 +393,17 @@ module_param_named(fw_load_type, amdgpu_fw_load_type, int, 0444);
 MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(aspm, amdgpu_aspm, int, 0444);
 
+/**
+ * DOC: iommu_perfopt (int)
+ * Control the AMD IOMMU PerfOpt DMA-latency optimization
+ * (0 = disable; 1 = enable on APU devices in identity domain).
+ * This arms the IOMMU PerfOpt control (IOMMU spec, MMIO Offset 016Ch, EFR PerfOptSup / PerfOptEn)
+ * Arming it disables ATS, PRI, PASID and SVA for the GPU and removes IOMMU DMA containment for it,
+ * trading isolation for lower DMA latency.
+ */
+MODULE_PARM_DESC(iommu_perfopt, "Control IOMMU PerfOpt DMA-latency optimization (1 = enable on APU devices in identity domain, 0 = disable)");
+module_param_named(iommu_perfopt, amdgpu_iommu_perfopt, int, 0444);
+
 /**
  * DOC: runpm (int)
  * Override for runtime power management control for dGPUs. The amdgpu driver can dynamically power down

drivers/iommu/amd/amd_iommu.h

@@ -48,6 +48,9 @@ extern u8 amd_iommu_hpt_vasize;
 extern unsigned long amd_iommu_pgsize_bitmap;
 extern bool amd_iommu_hatdis;
 
+int amd_iommu_perfopt_clear(struct amd_iommu *iommu);
+int amd_iommu_perfopt_restore(struct amd_iommu *iommu);
+
 /* Protection domain ops */
 void amd_iommu_init_identity_domain(void);
 struct protection_domain *protection_domain_alloc(void);

drivers/iommu/amd/amd_iommu_types.h

@@ -65,6 +65,7 @@
 #define MMIO_MSI_ADDR_LO_OFFSET	0x015C
 #define MMIO_MSI_ADDR_HI_OFFSET	0x0160
 #define MMIO_MSI_DATA_OFFSET	0x0164
+#define MMIO_PERF_OPT_OFFSET	0x016C
 #define MMIO_INTCAPXT_EVT_OFFSET	0x0170
 #define MMIO_INTCAPXT_PPR_OFFSET	0x0178
 #define MMIO_INTCAPXT_GALOG_OFFSET	0x0180
@@ -99,6 +100,8 @@
 #define FEATURE_GLX		GENMASK_ULL(15, 14)
 #define FEATURE_GAM_VAPIC	BIT_ULL(21)
 #define FEATURE_PASMAX		GENMASK_ULL(36, 32)
+#define FEATURE_PERF_OPT	BIT_ULL(45)
+#define PERF_OPT_EN		BIT(13)
 #define FEATURE_GIOSUP		BIT_ULL(48)
 #define FEATURE_HASUP		BIT_ULL(49)
 #define FEATURE_EPHSUP		BIT_ULL(50)
@@ -670,6 +673,9 @@ struct amd_iommu {
 	/* Extended features 2 */
 	u64 features2;
 
+	/* Devices requesting PerfOpt; the shared PERF_OPT_EN bit is on while >0. Protected by @lock. */
+	int perfopt_refcount;
+
 	/* PCI device id of the IOMMU device */
 	u16 devid;
 
@@ -831,6 +837,7 @@ struct iommu_dev_data {
 	u8 ppr          :1;		  /* Enable device PPR support */
 	bool use_vapic;			  /* Enable device to use vapic mode */
 	bool defer_attach;
+	bool perfopt;
 
 	struct ratelimit_state rs;        /* Ratelimit IOPF messages */
 };

drivers/iommu/amd/init.c

@@ -1942,6 +1942,9 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
 	if (!iommu->mmio_base)
 		return -ENOMEM;
 
+	if (amd_iommu_perfopt_clear(iommu))
+		pr_err("IOMMU%d: failed to clear PerfOpt\n", iommu->index);
+
 	return init_iommu_from_acpi(iommu, h);
 }
 
@@ -3032,10 +3035,46 @@ static void enable_iommus_vapic(void)
 #endif
 }
 
+static int clear_perfopt_all(void)
+{
+	struct amd_iommu *iommu;
+	int err, ret = 0;
+
+	for_each_iommu(iommu) {
+		err = amd_iommu_perfopt_clear(iommu);
+		if (err)
+			ret = err;
+	}
+
+	return ret;
+}
+
+static int restore_perfopt_all(void)
+{
+	struct amd_iommu *iommu;
+	int err, ret = 0;
+
+	for_each_iommu(iommu) {
+		err = amd_iommu_perfopt_restore(iommu);
+		if (err)
+			ret = err;
+	}
+
+	return ret;
+}
+
 static void disable_iommus(void)
 {
 	struct amd_iommu *iommu;
 
+	/*
+	 * PerfOpt is an optional performance bit, so a failure to clear it must
+	 * not skip the mandatory disable below. This also runs from the void
+	 * amd_iommu_disable() shutdown/kexec path, which cannot report an error.
+	 */
+	if (clear_perfopt_all())
+		pr_err("Failed to clear PerfOpt while disabling IOMMUs\n");
+
 	for_each_iommu(iommu)
 		iommu_disable(iommu);
 
@@ -3061,6 +3100,10 @@ static void amd_iommu_resume(void *data)
 	for_each_iommu(iommu)
 		early_enable_iommu(iommu);
 
+	/* early_enable_iommu() cleared PERF_OPT_EN; re-assert it from the refcount. */
+	if (restore_perfopt_all())
+		pr_err("Failed to restore PerfOpt after IOMMU resume\n");
+
 	iommu_enable_event_buffer();
 	amd_iommu_enable_interrupts();
 }