FROMLIST: SQUASH: i965: SIMD32 selection heuristics

(cover letter https://patchwork.freedesktop.org/series/51006/)

FROMLIST: i965: SIMD32 heuristics debug flag

Added a new DEBUG_HEUR32 flag to INTEL_DEBUG flags for enabling SIMD32
selection heuristics.

(am from https://patchwork.freedesktop.org/patch/256764/)

FROMLIST: i965: SIMD32 heuristics control data

Added a new structure for holding SIMD32 heuristics control data. The
control data itself will be fetched from drirc.

(am from https://patchwork.freedesktop.org/patch/256806/)

FROMLIST: i965: SIMD32 heuristics control data from drirc

To be able to test the heuristics with different parameters, they can be
controlled via environment variables through drirc.

(am from https://patchwork.freedesktop.org/patch/256788/)

FROMLIST: mesa: Helper functions for counting set bits in a mask

(am from https://patchwork.freedesktop.org/patch/256765/)

FROMLIST: i965/fs: Save the instruction count of each dispatch width

The SIMD32 selection heuristics will use this information for deciding whether
SIMD32 shaders should be used.

(am from https://patchwork.freedesktop.org/patch/256793/)

FROMLIST: i965/fs: SIMD32 selection heuristic based on grouped texture fetches

The function goes through the compiled shader and checks how many grouped
texture fetches there are. This is a simple heuristic which gets rid of most
of the regressions when enabling SIMD32 shaders but still retains some of
the benefits.

(am from https://patchwork.freedesktop.org/patch/256798/)

FROMLIST: i965/fs: Enable all SIMD32 heuristics

There are three simple heuristics for SIMD32 shader enabling:

- How many MRTs does the shader write into?
- How many grouped texture fetches does the shader have?
- How many instructions does the SIMD32 shader have compared to the SIMD16
   shader?

For testing purposes, the heuristics can be controlled via these environment
variables:

simd32_heuristic_mrt_check
- Enables MRT write check
- Default: true

simd32_heuristic_max_mrts
- How many MRT writes the heuristic allows
- Default: 1

simd32_heuristic_grouped_check
- Enables grouped texture fetch check
- Default: true

simd32_heuristic_grouped_sends
- How many grouped texture fetches the heuristic allows
- Default: 6

simd32_heuristic_inst_check
- Enables SIMD32 vs. SIMD16 instruction count check
- Default: true

simd32_heuristic_inst_ratio
- SIMD32 vs. SIMD16 instruction count ratio the heuristic allows
- Default: 2.3

SIMD32 shaders will not be compiled also when SIMD16 compilation fails or
spills.

(am from https://patchwork.freedesktop.org/patch/256766/)

Author: Toni Lönnberg

src/intel/compiler/brw_compiler.h

@@ -38,6 +38,15 @@ struct ra_regs;
 struct nir_shader;
 struct brw_program;
 
+struct brw_simd32_heuristics_control {
+   bool grouped_sends_check;
+   int max_grouped_sends;
+   bool inst_count_check;
+   float inst_count_ratio;
+   bool mrt_check;
+   int max_mrts;
+};
+
 struct brw_compiler {
    const struct gen_device_info *devinfo;
 
@@ -119,6 +128,8 @@ struct brw_compiler {
     * whether nir_opt_large_constants will be run.
     */
    bool supports_shader_constants;
+
+   struct brw_simd32_heuristics_control simd32_heuristics_control;
 };
 
 /**

src/intel/compiler/brw_fs.cpp

@@ -8240,6 +8240,8 @@ brw_compile_fs(const struct brw_compiler *compiler, void *log_data,
                char **error_str)
 {
    const struct gen_device_info *devinfo = compiler->devinfo;
+   bool simd16_failed = false;
+   bool simd16_spilled = false;
 
    unsigned max_subgroup_size = unlikely(INTEL_DEBUG & DEBUG_DO32) ? 32 : 16;
 
@@ -8323,20 +8325,30 @@ brw_compile_fs(const struct brw_compiler *compiler, void *log_data,
                      shader_time_index16);
       v16.import_uniforms(&v8);
       if (!v16.run_fs(allow_spilling, use_rep_send)) {
+         simd16_failed = true;
          compiler->shader_perf_log(log_data,
                                    "SIMD16 shader failed to compile: %s",
                                    v16.fail_msg);
       } else {
+         simd16_spilled = v16.spilled_any_registers;
          simd16_cfg = v16.cfg;
          prog_data->dispatch_grf_start_reg_16 = v16.payload.num_regs;
          prog_data->reg_blocks_16 = brw_register_blocks(v16.grf_used);
       }
    }
 
    /* Currently, the compiler only supports SIMD32 on SNB+ */
+   const brw_simd32_heuristics_control *ctrl = &compiler->simd32_heuristics_control;
+   uint64_t mrts = shader->info.outputs_written << FRAG_RESULT_DATA0;
+
    if (v8.max_dispatch_width >= 32 && !use_rep_send &&
        compiler->devinfo->gen >= 6 &&
-       unlikely(INTEL_DEBUG & DEBUG_DO32)) {
+       (unlikely(INTEL_DEBUG & DEBUG_DO32) ||
+        (unlikely(INTEL_DEBUG & DEBUG_HEUR32) &&
+         !simd16_failed && !simd16_spilled &&
+         (!ctrl->mrt_check ||
+          (ctrl->mrt_check &&
+          u_count_bits64(&mrts) <= ctrl->max_mrts))))) {
       /* Try a SIMD32 compile */
       fs_visitor v32(compiler, log_data, mem_ctx, &key->base,
                      &prog_data->base, shader, 32,
@@ -8347,9 +8359,12 @@ brw_compile_fs(const struct brw_compiler *compiler, void *log_data,
                                    "SIMD32 shader failed to compile: %s",
                                    v32.fail_msg);
       } else {
-         simd32_cfg = v32.cfg;
-         prog_data->dispatch_grf_start_reg_32 = v32.payload.num_regs;
-         prog_data->reg_blocks_32 = brw_register_blocks(v32.grf_used);
+         if (likely(!(INTEL_DEBUG & DEBUG_HEUR32)) ||
+              v32.run_heuristic(ctrl)) {
+            simd32_cfg = v32.cfg;
+            prog_data->dispatch_grf_start_reg_32 = v32.payload.num_regs;
+            prog_data->reg_blocks_32 = brw_register_blocks(v32.grf_used);
+         }
       }
    }
 
@@ -8424,14 +8439,51 @@ brw_compile_fs(const struct brw_compiler *compiler, void *log_data,
    }
 
    if (simd32_cfg) {
-      prog_data->dispatch_32 = true;
-      prog_data->prog_offset_32 = g.generate_code(simd32_cfg, 32, stats);
-      stats = stats ? stats + 1 : NULL;
+      uint32_t offset = g.generate_code(simd32_cfg, 32);
+
+      if (unlikely(INTEL_DEBUG & DEBUG_DO32) ||
+          (unlikely(INTEL_DEBUG & DEBUG_HEUR32) &&
+           (!simd16_cfg ||
+            (simd16_cfg &&
+             (!ctrl->inst_count_check ||
+             (ctrl->inst_count_check &&
+             (float)g.get_inst_count(32) / (float)g.get_inst_count(16) <= ctrl->inst_count_ratio)))))) {
+         prog_data->dispatch_32 = true;
+         prog_data->prog_offset_32 = offset;
+         uint32_t offset = g.generate_code(simd32_cfg, 32);
+      }
+
    }
 
    return g.get_assembly();
 }
 
+bool
+fs_visitor::run_heuristic(const struct brw_simd32_heuristics_control *ctrl) {
+   int grouped_sends = 0;
+   int max_grouped_sends = 0;
+   bool pass = true;
+
+   foreach_block_and_inst(block, fs_inst, inst, cfg) {
+      if (inst->opcode >= SHADER_OPCODE_TEX && inst->opcode <= SHADER_OPCODE_SAMPLEINFO_LOGICAL) {
+         ++grouped_sends;
+      } else if (grouped_sends > 0) {
+         if (grouped_sends > max_grouped_sends) {
+            max_grouped_sends = grouped_sends;
+         }
+         grouped_sends = 0;
+      }
+   }
+
+   if (ctrl->grouped_sends_check) {
+      if (max_grouped_sends > ctrl->max_grouped_sends) {
+         pass = false;
+      }
+   }
+
+   return pass;
+}
+
 fs_reg *
 fs_visitor::emit_cs_work_group_id_setup()
 {

src/intel/compiler/brw_fs.h

@@ -306,6 +306,8 @@ class fs_visitor : public backend_shader
    virtual void dump_instructions(const char *name);
    void dump_instruction(backend_instruction *inst);
    void dump_instruction(backend_instruction *inst, FILE *file);
+   
+   bool run_heuristic(const struct brw_simd32_heuristics_control *ctrl);
 
    const brw_base_prog_key *const key;
    const struct brw_sampler_prog_key_data *key_tex;
@@ -430,6 +432,7 @@ class fs_generator
    void enable_debug(const char *shader_name);
    int generate_code(const cfg_t *cfg, int dispatch_width,
                      struct brw_compile_stats *stats);
+   int get_inst_count(int dispatch_width);
    const unsigned *get_assembly();
 
 private:
@@ -525,6 +528,7 @@ class fs_generator
    struct brw_stage_prog_data * const prog_data;
 
    unsigned dispatch_width; /**< 8, 16 or 32 */
+   int inst_count[3]; /* for 8, 16 and 32 */
 
    exec_list discard_halt_patches;
    struct shader_stats shader_stats;

src/intel/compiler/brw_fs.h.rej

@@ -0,0 +1,26 @@
+diff a/src/intel/compiler/brw_fs.h b/src/intel/compiler/brw_fs.h	(rejected hunks)
+@@ -300,6 +300,8 @@ public:
+    void dump_instruction(backend_instruction *inst);
+    void dump_instruction(backend_instruction *inst, FILE *file);
+ 
++   bool run_heuristic(const struct brw_simd32_heuristics_control *ctrl);
++
+    const void *const key;
+    const struct brw_sampler_prog_key_data *key_tex;
+ 
+@@ -420,6 +422,7 @@ public:
+ 
+    void enable_debug(const char *shader_name);
+    int generate_code(const cfg_t *cfg, int dispatch_width);
++   int get_inst_count(int dispatch_width);
+    const unsigned *get_assembly();
+ 
+ private:
+@@ -515,6 +518,7 @@ private:
+    struct brw_stage_prog_data * const prog_data;
+ 
+    unsigned dispatch_width; /**< 8, 16 or 32 */
++   int inst_count[3]; /* for 8, 16 and 32 */
+ 
+    exec_list discard_halt_patches;
+    unsigned promoted_constants;

src/intel/compiler/brw_fs_generator.cpp

@@ -2430,6 +2430,8 @@ fs_generator::generate_code(const cfg_t *cfg, int dispatch_width,
       stats->fills = fill_count;
    }
 
+   inst_count[ffs(dispatch_width) - 4] = before_size / 16;
+
    return start_offset;
 }
 
@@ -2438,3 +2440,13 @@ fs_generator::get_assembly()
 {
    return brw_get_program(p, &prog_data->program_size);
 }
+
+int
+fs_generator::get_inst_count(int dispatch_width)
+{
+   if (dispatch_width == 8 || dispatch_width == 16 || dispatch_width == 32) {
+      return inst_count[ffs(dispatch_width) - 4];
+   } else {
+      return 0;
+   }
+}

src/intel/compiler/brw_fs_generator.cpp.rej

@@ -0,0 +1,10 @@
+diff a/src/intel/compiler/brw_fs_generator.cpp b/src/intel/compiler/brw_fs_generator.cpp	(rejected hunks)
+@@ -2297,6 +2297,8 @@ fs_generator::generate_code(const cfg_t *cfg, int dispatch_width)
+                               fill_count, promoted_constants, before_size,
+                               after_size);
+ 
++   inst_count[ffs(dispatch_width) - 4] = before_size / 16;
++
+    return start_offset;
+ }
+ 

src/intel/dev/gen_debug.c

@@ -86,6 +86,7 @@ static const struct debug_control debug_control[] = {
    { "color",       DEBUG_COLOR },
    { "reemit",      DEBUG_REEMIT },
    { "soft64",      DEBUG_SOFT64 },
+   { "heur32",      DEBUG_HEUR32 },
    { "tcs8",        DEBUG_TCS_EIGHT_PATCH },
    { "bt",          DEBUG_BT },
    { "pc",          DEBUG_PIPE_CONTROL },

src/intel/dev/gen_debug.c.rej

@@ -0,0 +1,9 @@
+diff a/src/intel/dev/gen_debug.c b/src/intel/dev/gen_debug.c	(rejected hunks)
+@@ -86,6 +86,7 @@ static const struct debug_control debug_control[] = {
+    { "color",       DEBUG_COLOR },
+    { "reemit",      DEBUG_REEMIT },
+    { "soft64",      DEBUG_SOFT64 },
++   { "heur32",      DEBUG_HEUR32 },
+    { NULL,    0 }
+ };
+ 

src/intel/dev/gen_debug.h

@@ -88,6 +88,7 @@ extern uint64_t INTEL_DEBUG;
 #define DEBUG_BT                  (1ull << 44)
 #define DEBUG_PIPE_CONTROL        (1ull << 45)
 #define DEBUG_NO_FAST_CLEAR       (1ull << 46)
+#define DEBUG_HEUR32              (1ull << 47)
 
 /* These flags are not compatible with the disk shader cache */
 #define DEBUG_DISK_CACHE_DISABLE_MASK DEBUG_SHADER_TIME
@@ -96,7 +97,7 @@ extern uint64_t INTEL_DEBUG;
 #define DEBUG_DISK_CACHE_MASK \
    (DEBUG_NO16 | DEBUG_NO_DUAL_OBJECT_GS | DEBUG_NO8 |  DEBUG_SPILL_FS | \
    DEBUG_SPILL_VEC4 | DEBUG_NO_COMPACTION | DEBUG_DO32 | DEBUG_SOFT64 | \
-   DEBUG_TCS_EIGHT_PATCH)
+   DEBUG_TCS_EIGHT_PATCH | DEBUG_HEUR32)
 
 #ifdef HAVE_ANDROID_PLATFORM
 #define LOG_TAG "INTEL-MESA"

src/intel/dev/gen_debug.h.rej

@@ -0,0 +1,18 @@
+diff a/src/intel/dev/gen_debug.h b/src/intel/dev/gen_debug.h	(rejected hunks)
+@@ -84,6 +84,7 @@ extern uint64_t INTEL_DEBUG;
+ #define DEBUG_COLOR               (1ull << 40)
+ #define DEBUG_REEMIT              (1ull << 41)
+ #define DEBUG_SOFT64              (1ull << 42)
++#define DEBUG_HEUR32              (1ull << 43)
+ 
+ /* These flags are not compatible with the disk shader cache */
+ #define DEBUG_DISK_CACHE_DISABLE_MASK DEBUG_SHADER_TIME
+@@ -91,7 +92,7 @@ extern uint64_t INTEL_DEBUG;
+ /* These flags may affect program generation */
+ #define DEBUG_DISK_CACHE_MASK \
+    (DEBUG_NO16 | DEBUG_NO_DUAL_OBJECT_GS | DEBUG_NO8 |  DEBUG_SPILL_FS | \
+-   DEBUG_SPILL_VEC4 | DEBUG_NO_COMPACTION | DEBUG_DO32 | DEBUG_SOFT64)
++   DEBUG_SPILL_VEC4 | DEBUG_NO_COMPACTION | DEBUG_DO32 | DEBUG_SOFT64 | DEBUG_HEUR32)
+ 
+ #ifdef HAVE_ANDROID_PLATFORM
+ #define LOG_TAG "INTEL-MESA"