xref: /aosp_15_r20/external/mesa3d/src/intel/vulkan/genX_pipeline.c (revision 6104692788411f58d303aa86923a9ff6ecaded22)

/*
 * Copyright © 2015 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "anv_private.h"

#include "genxml/gen_macros.h"
#include "genxml/genX_pack.h"
#include "genxml/genX_rt_pack.h"

#include "common/intel_compute_slm.h"
#include "common/intel_genX_state_brw.h"
#include "common/intel_l3_config.h"
#include "common/intel_sample_positions.h"
#include "nir/nir_xfb_info.h"
#include "vk_util.h"
#include "vk_format.h"
#include "vk_log.h"
#include "vk_render_pass.h"

static inline struct anv_batch *
anv_gfx_pipeline_add(struct anv_graphics_pipeline *pipeline,
                     struct anv_gfx_state_ptr *ptr,
                     uint32_t n_dwords)
{
   struct anv_batch *batch = &pipeline->base.base.batch;

   assert(ptr->len == 0 ||
          (batch->next - batch->start) / 4 == (ptr->offset + ptr->len));
   if (ptr->len == 0)
      ptr->offset = (batch->next - batch->start) / 4;
   ptr->len += n_dwords;

   return batch;
}

#define anv_pipeline_emit_tmp(pipeline, field, cmd, name)               \
   for (struct cmd name = { __anv_cmd_header(cmd) },                    \
           *_dst = (void *) field;                                      \
        __builtin_expect(_dst != NULL, 1);                              \
        ({ __anv_cmd_pack(cmd)(&(pipeline)->base.base.batch,            \
                               _dst, &name);                            \
           VG(VALGRIND_CHECK_MEM_IS_DEFINED(_dst, __anv_cmd_length(cmd) * 4)); \
           _dst = NULL;                                                 \
        }))

#define anv_pipeline_emit(pipeline, state, cmd, name)                   \
   for (struct cmd name = { __anv_cmd_header(cmd) },                    \
           *_dst = anv_batch_emit_dwords(                               \
              anv_gfx_pipeline_add(pipeline,                            \
                                   &(pipeline)->state,                  \
                                   __anv_cmd_length(cmd)),              \
              __anv_cmd_length(cmd));                                   \
        __builtin_expect(_dst != NULL, 1);                              \
        ({ __anv_cmd_pack(cmd)(&(pipeline)->base.base.batch,            \
                               _dst, &name);                            \
           VG(VALGRIND_CHECK_MEM_IS_DEFINED(_dst, __anv_cmd_length(cmd) * 4)); \
           _dst = NULL;                                                 \
        }))

#define anv_pipeline_emit_merge(pipeline, state, dwords, cmd, name) \
   for (struct cmd name = { 0 },                                        \
           *_dst = anv_batch_emit_dwords(                               \
              anv_gfx_pipeline_add(pipeline,                            \
                                   &(pipeline)->state,                  \
                                   __anv_cmd_length(cmd)),              \
              __anv_cmd_length(cmd));                                   \
        __builtin_expect(_dst != NULL, 1);                              \
        ({ uint32_t _partial[__anv_cmd_length(cmd)];                    \
           assert((pipeline)->state.len == __anv_cmd_length(cmd));      \
           __anv_cmd_pack(cmd)(&(pipeline)->base.base.batch,            \
                               _partial, &name);                        \
           for (uint32_t i = 0; i < __anv_cmd_length(cmd); i++) {       \
              ((uint32_t *)_dst)[i] = _partial[i] | dwords[i];          \
           }                                                            \
           VG(VALGRIND_CHECK_MEM_IS_DEFINED(_dst, __anv_cmd_length(cmd) * 4)); \
           _dst = NULL;                                                 \
         }))

#define anv_pipeline_emitn(pipeline, state, n, cmd, ...) ({             \
   void *__dst = anv_batch_emit_dwords(                                 \
      anv_gfx_pipeline_add(pipeline, &(pipeline)->state, n), n);        \
   if (__dst) {                                                         \
      struct cmd __template = {                                         \
         __anv_cmd_header(cmd),                                         \
         .DWordLength = n - __anv_cmd_length_bias(cmd),                 \
         __VA_ARGS__                                                    \
      };                                                                \
      __anv_cmd_pack(cmd)(&pipeline->base.base.batch,                   \
                          __dst, &__template);                          \
   }                                                                    \
   __dst;                                                               \
   })

#define pipeline_needs_protected(pipeline) \
   ((pipeline)->device->vk.enabled_features.protectedMemory)

static uint32_t
vertex_element_comp_control(enum isl_format format, unsigned comp)
{
   uint8_t bits;
   switch (comp) {
   case 0: bits = isl_format_layouts[format].channels.r.bits; break;
   case 1: bits = isl_format_layouts[format].channels.g.bits; break;
   case 2: bits = isl_format_layouts[format].channels.b.bits; break;
   case 3: bits = isl_format_layouts[format].channels.a.bits; break;
   default: unreachable("Invalid component");
   }

   /*
    * Take in account hardware restrictions when dealing with 64-bit floats.
    *
    * From Broadwell spec, command reference structures, page 586:
    *  "When SourceElementFormat is set to one of the *64*_PASSTHRU formats,
    *   64-bit components are stored * in the URB without any conversion. In
    *   this case, vertex elements must be written as 128 or 256 bits, with
    *   VFCOMP_STORE_0 being used to pad the output as required. E.g., if
    *   R64_PASSTHRU is used to copy a 64-bit Red component into the URB,
    *   Component 1 must be specified as VFCOMP_STORE_0 (with Components 2,3
    *   set to VFCOMP_NOSTORE) in order to output a 128-bit vertex element, or
    *   Components 1-3 must be specified as VFCOMP_STORE_0 in order to output
    *   a 256-bit vertex element. Likewise, use of R64G64B64_PASSTHRU requires
    *   Component 3 to be specified as VFCOMP_STORE_0 in order to output a
    *   256-bit vertex element."
    */
   if (bits) {
      return VFCOMP_STORE_SRC;
   } else if (comp >= 2 &&
              !isl_format_layouts[format].channels.b.bits &&
              isl_format_layouts[format].channels.r.type == ISL_RAW) {
      /* When emitting 64-bit attributes, we need to write either 128 or 256
       * bit chunks, using VFCOMP_NOSTORE when not writing the chunk, and
       * VFCOMP_STORE_0 to pad the written chunk */
      return VFCOMP_NOSTORE;
   } else if (comp < 3 ||
              isl_format_layouts[format].channels.r.type == ISL_RAW) {
      /* Note we need to pad with value 0, not 1, due hardware restrictions
       * (see comment above) */
      return VFCOMP_STORE_0;
   } else if (isl_format_layouts[format].channels.r.type == ISL_UINT ||
            isl_format_layouts[format].channels.r.type == ISL_SINT) {
      assert(comp == 3);
      return VFCOMP_STORE_1_INT;
   } else {
      assert(comp == 3);
      return VFCOMP_STORE_1_FP;
   }
}

void
genX(emit_vertex_input)(struct anv_batch *batch,
                        uint32_t *vertex_element_dws,
                        struct anv_graphics_pipeline *pipeline,
                        const struct vk_vertex_input_state *vi,
                        bool emit_in_pipeline)
{
   const struct anv_device *device = pipeline->base.base.device;
   const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
   const uint64_t inputs_read = vs_prog_data->inputs_read;
   const uint64_t double_inputs_read =
      vs_prog_data->double_inputs_read & inputs_read;
   assert((inputs_read & ((1 << VERT_ATTRIB_GENERIC0) - 1)) == 0);
   const uint32_t elements = inputs_read >> VERT_ATTRIB_GENERIC0;
   const uint32_t elements_double = double_inputs_read >> VERT_ATTRIB_GENERIC0;

   for (uint32_t i = 0; i < pipeline->vs_input_elements; i++) {
      /* The SKL docs for VERTEX_ELEMENT_STATE say:
       *
       *    "All elements must be valid from Element[0] to the last valid
       *    element. (I.e. if Element[2] is valid then Element[1] and
       *    Element[0] must also be valid)."
       *
       * The SKL docs for 3D_Vertex_Component_Control say:
       *
       *    "Don't store this component. (Not valid for Component 0, but can
       *    be used for Component 1-3)."
       *
       * So we can't just leave a vertex element blank and hope for the best.
       * We have to tell the VF hardware to put something in it; so we just
       * store a bunch of zero.
       *
       * TODO: Compact vertex elements so we never end up with holes.
       */
      struct GENX(VERTEX_ELEMENT_STATE) element = {
         .Valid = true,
         .Component0Control = VFCOMP_STORE_0,
         .Component1Control = VFCOMP_STORE_0,
         .Component2Control = VFCOMP_STORE_0,
         .Component3Control = VFCOMP_STORE_0,
      };
      GENX(VERTEX_ELEMENT_STATE_pack)(NULL,
                                      &vertex_element_dws[i * 2],
                                      &element);
   }

   u_foreach_bit(a, vi->attributes_valid) {
      enum isl_format format = anv_get_isl_format(device->info,
                                                  vi->attributes[a].format,
                                                  VK_IMAGE_ASPECT_COLOR_BIT,
                                                  VK_IMAGE_TILING_LINEAR);
      assume(format < ISL_NUM_FORMATS);

      uint32_t binding = vi->attributes[a].binding;
      assert(binding < MAX_VBS);

      if ((elements & (1 << a)) == 0)
         continue; /* Binding unused */

      uint32_t slot =
         __builtin_popcount(elements & ((1 << a) - 1)) -
         DIV_ROUND_UP(__builtin_popcount(elements_double &
                                        ((1 << a) -1)), 2);

      struct GENX(VERTEX_ELEMENT_STATE) element = {
         .VertexBufferIndex = vi->attributes[a].binding,
         .Valid = true,
         .SourceElementFormat = format,
         .EdgeFlagEnable = false,
         .SourceElementOffset = vi->attributes[a].offset,
         .Component0Control = vertex_element_comp_control(format, 0),
         .Component1Control = vertex_element_comp_control(format, 1),
         .Component2Control = vertex_element_comp_control(format, 2),
         .Component3Control = vertex_element_comp_control(format, 3),
      };
      GENX(VERTEX_ELEMENT_STATE_pack)(NULL,
                                      &vertex_element_dws[slot * 2],
                                      &element);

      /* On Broadwell and later, we have a separate VF_INSTANCING packet
       * that controls instancing.  On Haswell and prior, that's part of
       * VERTEX_BUFFER_STATE which we emit later.
       */
      if (emit_in_pipeline) {
         anv_pipeline_emit(pipeline, final.vf_instancing, GENX(3DSTATE_VF_INSTANCING), vfi) {
            bool per_instance = vi->bindings[binding].input_rate ==
               VK_VERTEX_INPUT_RATE_INSTANCE;
            uint32_t divisor = vi->bindings[binding].divisor *
               pipeline->instance_multiplier;

            vfi.InstancingEnable = per_instance;
            vfi.VertexElementIndex = slot;
            vfi.InstanceDataStepRate = per_instance ? divisor : 1;
         }
      } else {
         anv_batch_emit(batch, GENX(3DSTATE_VF_INSTANCING), vfi) {
            bool per_instance = vi->bindings[binding].input_rate ==
               VK_VERTEX_INPUT_RATE_INSTANCE;
            uint32_t divisor = vi->bindings[binding].divisor *
               pipeline->instance_multiplier;

            vfi.InstancingEnable = per_instance;
            vfi.VertexElementIndex = slot;
            vfi.InstanceDataStepRate = per_instance ? divisor : 1;
         }
      }
   }
}

static void
emit_vertex_input(struct anv_graphics_pipeline *pipeline,
                  const struct vk_graphics_pipeline_state *state,
                  const struct vk_vertex_input_state *vi)
{
   /* Only pack the VERTEX_ELEMENT_STATE if not dynamic so we can just memcpy
    * everything in gfx8_cmd_buffer.c
    */
   if (!BITSET_TEST(state->dynamic, MESA_VK_DYNAMIC_VI)) {
      genX(emit_vertex_input)(NULL,
                              pipeline->vertex_input_data,
                              pipeline, vi, true /* emit_in_pipeline */);
   }

   const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
   const bool needs_svgs_elem = pipeline->svgs_count > 1 ||
                                !vs_prog_data->uses_drawid;
   const uint32_t id_slot = pipeline->vs_input_elements;
   const uint32_t drawid_slot = id_slot + needs_svgs_elem;
   if (pipeline->svgs_count > 0) {
      assert(pipeline->vertex_input_elems >= pipeline->svgs_count);
      uint32_t slot_offset =
         pipeline->vertex_input_elems - pipeline->svgs_count;

      if (needs_svgs_elem) {
#if GFX_VER < 11
         /* From the Broadwell PRM for the 3D_Vertex_Component_Control enum:
          *    "Within a VERTEX_ELEMENT_STATE structure, if a Component
          *    Control field is set to something other than VFCOMP_STORE_SRC,
          *    no higher-numbered Component Control fields may be set to
          *    VFCOMP_STORE_SRC"
          *
          * This means, that if we have BaseInstance, we need BaseVertex as
          * well.  Just do all or nothing.
          */
         uint32_t base_ctrl = (vs_prog_data->uses_firstvertex ||
                               vs_prog_data->uses_baseinstance) ?
                              VFCOMP_STORE_SRC : VFCOMP_STORE_0;
#endif

         struct GENX(VERTEX_ELEMENT_STATE) element = {
            .VertexBufferIndex = ANV_SVGS_VB_INDEX,
            .Valid = true,
            .SourceElementFormat = ISL_FORMAT_R32G32_UINT,
#if GFX_VER >= 11
            /* On gen11, these are taken care of by extra parameter slots */
            .Component0Control = VFCOMP_STORE_0,
            .Component1Control = VFCOMP_STORE_0,
#else
            .Component0Control = base_ctrl,
            .Component1Control = base_ctrl,
#endif
            .Component2Control = VFCOMP_STORE_0,
            .Component3Control = VFCOMP_STORE_0,
         };
         GENX(VERTEX_ELEMENT_STATE_pack)(NULL,
                                         &pipeline->vertex_input_data[slot_offset * 2],
                                         &element);
         slot_offset++;

         anv_pipeline_emit(pipeline, final.vf_sgvs_instancing,
                           GENX(3DSTATE_VF_INSTANCING), vfi) {
            vfi.VertexElementIndex = id_slot;
         }
      }

      if (vs_prog_data->uses_drawid) {
         struct GENX(VERTEX_ELEMENT_STATE) element = {
            .VertexBufferIndex = ANV_DRAWID_VB_INDEX,
            .Valid = true,
            .SourceElementFormat = ISL_FORMAT_R32_UINT,
#if GFX_VER >= 11
            /* On gen11, this is taken care of by extra parameter slots */
            .Component0Control = VFCOMP_STORE_0,
#else
            .Component0Control = VFCOMP_STORE_SRC,
#endif
            .Component1Control = VFCOMP_STORE_0,
            .Component2Control = VFCOMP_STORE_0,
            .Component3Control = VFCOMP_STORE_0,
         };
         GENX(VERTEX_ELEMENT_STATE_pack)(NULL,
                                         &pipeline->vertex_input_data[slot_offset * 2],
                                         &element);
         slot_offset++;

         anv_pipeline_emit(pipeline, final.vf_sgvs_instancing,
                           GENX(3DSTATE_VF_INSTANCING), vfi) {
            vfi.VertexElementIndex = drawid_slot;
         }
      }
   }

   anv_pipeline_emit(pipeline, final.vf_sgvs, GENX(3DSTATE_VF_SGVS), sgvs) {
      sgvs.VertexIDEnable              = vs_prog_data->uses_vertexid;
      sgvs.VertexIDComponentNumber     = 2;
      sgvs.VertexIDElementOffset       = id_slot;
      sgvs.InstanceIDEnable            = vs_prog_data->uses_instanceid;
      sgvs.InstanceIDComponentNumber   = 3;
      sgvs.InstanceIDElementOffset     = id_slot;
   }

#if GFX_VER >= 11
   anv_pipeline_emit(pipeline, final.vf_sgvs_2, GENX(3DSTATE_VF_SGVS_2), sgvs) {
      /* gl_BaseVertex */
      sgvs.XP0Enable                   = vs_prog_data->uses_firstvertex;
      sgvs.XP0SourceSelect             = XP0_PARAMETER;
      sgvs.XP0ComponentNumber          = 0;
      sgvs.XP0ElementOffset            = id_slot;

      /* gl_BaseInstance */
      sgvs.XP1Enable                   = vs_prog_data->uses_baseinstance;
      sgvs.XP1SourceSelect             = StartingInstanceLocation;
      sgvs.XP1ComponentNumber          = 1;
      sgvs.XP1ElementOffset            = id_slot;

      /* gl_DrawID */
      sgvs.XP2Enable                   = vs_prog_data->uses_drawid;
      sgvs.XP2ComponentNumber          = 0;
      sgvs.XP2ElementOffset            = drawid_slot;
   }
#endif
}

void
genX(emit_urb_setup)(struct anv_device *device, struct anv_batch *batch,
                     const struct intel_l3_config *l3_config,
                     VkShaderStageFlags active_stages,
                     const struct intel_urb_config *urb_cfg_in,
                     struct intel_urb_config *urb_cfg_out,
                     enum intel_urb_deref_block_size *deref_block_size)
{
   const struct intel_device_info *devinfo = device->info;

   bool constrained;
   intel_get_urb_config(devinfo, l3_config,
                        active_stages &
                           VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT,
                        active_stages & VK_SHADER_STAGE_GEOMETRY_BIT,
                        urb_cfg_out, deref_block_size,
                        &constrained);

#if INTEL_NEEDS_WA_16014912113
      if (intel_urb_setup_changed(urb_cfg_in, urb_cfg_out,
          MESA_SHADER_TESS_EVAL) && urb_cfg_in->size[0] != 0) {
         for (int i = 0; i <= MESA_SHADER_GEOMETRY; i++) {
            anv_batch_emit(batch, GENX(3DSTATE_URB_VS), urb) {
               urb._3DCommandSubOpcode      += i;
               urb.VSURBStartingAddress      = urb_cfg_in->start[i];
               urb.VSURBEntryAllocationSize  = urb_cfg_in->size[i] - 1;
               urb.VSNumberofURBEntries      = i == 0 ? 256 : 0;
            }
         }
         genx_batch_emit_pipe_control(batch, device->info, _3D,
                                      ANV_PIPE_HDC_PIPELINE_FLUSH_BIT);
      }
#endif

   for (int i = 0; i <= MESA_SHADER_GEOMETRY; i++) {
      anv_batch_emit(batch, GENX(3DSTATE_URB_VS), urb) {
         urb._3DCommandSubOpcode      += i;
         urb.VSURBStartingAddress      = urb_cfg_out->start[i];
         urb.VSURBEntryAllocationSize  = urb_cfg_out->size[i] - 1;
         urb.VSNumberofURBEntries      = urb_cfg_out->entries[i];
      }
   }
#if GFX_VERx10 >= 125
   if (device->vk.enabled_extensions.EXT_mesh_shader) {
      anv_batch_emit(batch, GENX(3DSTATE_URB_ALLOC_MESH), zero);
      anv_batch_emit(batch, GENX(3DSTATE_URB_ALLOC_TASK), zero);
   }
#endif
}

#if GFX_VERx10 >= 125
static void
emit_urb_setup_mesh(struct anv_graphics_pipeline *pipeline,
                    enum intel_urb_deref_block_size *deref_block_size)
{
   const struct intel_device_info *devinfo = pipeline->base.base.device->info;

   const struct brw_task_prog_data *task_prog_data =
      anv_pipeline_has_stage(pipeline, MESA_SHADER_TASK) ?
      get_task_prog_data(pipeline) : NULL;
   const struct brw_mesh_prog_data *mesh_prog_data = get_mesh_prog_data(pipeline);

   const struct intel_mesh_urb_allocation alloc =
      intel_get_mesh_urb_config(devinfo, pipeline->base.base.l3_config,
                                task_prog_data ? task_prog_data->map.size_dw : 0,
                                mesh_prog_data->map.size_dw);

   /* Zero out the primitive pipeline URB allocations. */
   for (int i = 0; i <= MESA_SHADER_GEOMETRY; i++) {
      anv_pipeline_emit(pipeline, final.urb, GENX(3DSTATE_URB_VS), urb) {
         urb._3DCommandSubOpcode += i;
      }
   }

   anv_pipeline_emit(pipeline, final.urb, GENX(3DSTATE_URB_ALLOC_TASK), urb) {
      if (task_prog_data) {
         urb.TASKURBEntryAllocationSize   = alloc.task_entry_size_64b - 1;
         urb.TASKNumberofURBEntriesSlice0 = alloc.task_entries;
         urb.TASKNumberofURBEntriesSliceN = alloc.task_entries;
         urb.TASKURBStartingAddressSlice0 = alloc.task_starting_address_8kb;
         urb.TASKURBStartingAddressSliceN = alloc.task_starting_address_8kb;
      }
   }

   anv_pipeline_emit(pipeline, final.urb, GENX(3DSTATE_URB_ALLOC_MESH), urb) {
      urb.MESHURBEntryAllocationSize   = alloc.mesh_entry_size_64b - 1;
      urb.MESHNumberofURBEntriesSlice0 = alloc.mesh_entries;
      urb.MESHNumberofURBEntriesSliceN = alloc.mesh_entries;
      urb.MESHURBStartingAddressSlice0 = alloc.mesh_starting_address_8kb;
      urb.MESHURBStartingAddressSliceN = alloc.mesh_starting_address_8kb;
   }

   *deref_block_size = alloc.deref_block_size;
}
#endif

static void
emit_urb_setup(struct anv_graphics_pipeline *pipeline,
               enum intel_urb_deref_block_size *deref_block_size)
{
#if GFX_VERx10 >= 125
   if (anv_pipeline_is_mesh(pipeline)) {
      emit_urb_setup_mesh(pipeline, deref_block_size);
      return;
   }
#endif
   for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++) {
      const struct brw_vue_prog_data *prog_data =
         !anv_pipeline_has_stage(pipeline, i) ? NULL :
         (const struct brw_vue_prog_data *) pipeline->base.shaders[i]->prog_data;

      pipeline->urb_cfg.size[i] = prog_data ? prog_data->urb_entry_size : 1;
   }

   struct anv_device *device = pipeline->base.base.device;
   const struct intel_device_info *devinfo = device->info;


   bool constrained;
   intel_get_urb_config(devinfo,
                        pipeline->base.base.l3_config,
                        pipeline->base.base.active_stages &
                           VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT,
                        pipeline->base.base.active_stages &
                           VK_SHADER_STAGE_GEOMETRY_BIT,
                        &pipeline->urb_cfg, deref_block_size,
                        &constrained);

   for (int i = 0; i <= MESA_SHADER_GEOMETRY; i++) {
      anv_pipeline_emit(pipeline, final.urb, GENX(3DSTATE_URB_VS), urb) {
         urb._3DCommandSubOpcode      += i;
         urb.VSURBStartingAddress      = pipeline->urb_cfg.start[i];
         urb.VSURBEntryAllocationSize  = pipeline->urb_cfg.size[i] - 1;
         urb.VSNumberofURBEntries      = pipeline->urb_cfg.entries[i];
      }
   }

#if GFX_VERx10 >= 125
   if (device->vk.enabled_extensions.EXT_mesh_shader) {
      anv_pipeline_emit(pipeline, final.urb, GENX(3DSTATE_URB_ALLOC_TASK), zero);
      anv_pipeline_emit(pipeline, final.urb, GENX(3DSTATE_URB_ALLOC_MESH), zero);
   }
#endif

}

static bool
sbe_primitive_id_override(struct anv_graphics_pipeline *pipeline)
{
   const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);
   if (!wm_prog_data)
      return false;

   const struct intel_vue_map *fs_input_map =
      &anv_pipeline_get_last_vue_prog_data(pipeline)->vue_map;

   return (wm_prog_data->inputs & VARYING_BIT_PRIMITIVE_ID) &&
          fs_input_map->varying_to_slot[VARYING_SLOT_PRIMITIVE_ID] == -1;
}

static void
emit_3dstate_sbe(struct anv_graphics_pipeline *pipeline)
{
   const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);

   if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT)) {
      anv_pipeline_emit(pipeline, final.sbe, GENX(3DSTATE_SBE), sbe);
      anv_pipeline_emit(pipeline, final.sbe_swiz, GENX(3DSTATE_SBE_SWIZ), sbe);
#if GFX_VERx10 >= 125
      if (anv_pipeline_is_mesh(pipeline))
         anv_pipeline_emit(pipeline, final.sbe_mesh, GENX(3DSTATE_SBE_MESH), sbe);
#endif
      return;
   }

   anv_pipeline_emit(pipeline, final.sbe, GENX(3DSTATE_SBE), sbe) {
   anv_pipeline_emit(pipeline, final.sbe_swiz, GENX(3DSTATE_SBE_SWIZ), swiz) {

      /* TODO(mesh): Figure out cases where we need attribute swizzling.  See also
       * calculate_urb_setup() and related functions.
       */
      sbe.AttributeSwizzleEnable = anv_pipeline_is_primitive(pipeline);
      sbe.PointSpriteTextureCoordinateOrigin = UPPERLEFT;
      sbe.NumberofSFOutputAttributes = wm_prog_data->num_varying_inputs;
      sbe.ConstantInterpolationEnable = wm_prog_data->flat_inputs;

      for (unsigned i = 0; i < 32; i++)
         sbe.AttributeActiveComponentFormat[i] = ACF_XYZW;

      if (anv_pipeline_is_primitive(pipeline)) {
         const struct intel_vue_map *fs_input_map =
            &anv_pipeline_get_last_vue_prog_data(pipeline)->vue_map;

         int first_slot =
            brw_compute_first_urb_slot_required(wm_prog_data->inputs,
                                                fs_input_map);
         assert(first_slot % 2 == 0);
         unsigned urb_entry_read_offset = first_slot / 2;
         int max_source_attr = 0;
         for (uint8_t idx = 0; idx < wm_prog_data->urb_setup_attribs_count; idx++) {
            uint8_t attr = wm_prog_data->urb_setup_attribs[idx];
            int input_index = wm_prog_data->urb_setup[attr];

            assert(0 <= input_index);

            /* gl_Viewport, gl_Layer and FragmentShadingRateKHR are stored in the
             * VUE header
             */
            if (attr == VARYING_SLOT_VIEWPORT ||
                attr == VARYING_SLOT_LAYER ||
                attr == VARYING_SLOT_PRIMITIVE_SHADING_RATE) {
               continue;
            }

            if (attr == VARYING_SLOT_PNTC) {
               sbe.PointSpriteTextureCoordinateEnable = 1 << input_index;
               continue;
            }

            const int slot = fs_input_map->varying_to_slot[attr];

            if (slot == -1) {
               /* This attribute does not exist in the VUE--that means that
                * the vertex shader did not write to it. It could be that it's
                * a regular varying read by the fragment shader but not
                * written by the vertex shader or it's gl_PrimitiveID. In the
                * first case the value is undefined, in the second it needs to
                * be gl_PrimitiveID.
                */
               swiz.Attribute[input_index].ConstantSource = PRIM_ID;
               swiz.Attribute[input_index].ComponentOverrideX = true;
               swiz.Attribute[input_index].ComponentOverrideY = true;
               swiz.Attribute[input_index].ComponentOverrideZ = true;
               swiz.Attribute[input_index].ComponentOverrideW = true;
               continue;
            }

            /* We have to subtract two slots to account for the URB entry
             * output read offset in the VS and GS stages.
             */
            const int source_attr = slot - 2 * urb_entry_read_offset;
            assert(source_attr >= 0 && source_attr < 32);
            max_source_attr = MAX2(max_source_attr, source_attr);
            /* The hardware can only do overrides on 16 overrides at a time,
             * and the other up to 16 have to be lined up so that the input
             * index = the output index. We'll need to do some tweaking to
             * make sure that's the case.
             */
            if (input_index < 16)
               swiz.Attribute[input_index].SourceAttribute = source_attr;
            else
               assert(source_attr == input_index);
         }

         sbe.VertexURBEntryReadOffset = urb_entry_read_offset;
         sbe.VertexURBEntryReadLength = DIV_ROUND_UP(max_source_attr + 1, 2);
         sbe.ForceVertexURBEntryReadOffset = true;
         sbe.ForceVertexURBEntryReadLength = true;

         /* Ask the hardware to supply PrimitiveID if the fragment shader
          * reads it but a previous stage didn't write one.
          */
         if (sbe_primitive_id_override(pipeline)) {
            sbe.PrimitiveIDOverrideAttributeSelect =
               wm_prog_data->urb_setup[VARYING_SLOT_PRIMITIVE_ID];
            sbe.PrimitiveIDOverrideComponentX = true;
            sbe.PrimitiveIDOverrideComponentY = true;
            sbe.PrimitiveIDOverrideComponentZ = true;
            sbe.PrimitiveIDOverrideComponentW = true;
         }
      } else {
         assert(anv_pipeline_is_mesh(pipeline));
#if GFX_VERx10 >= 125
         const struct brw_mesh_prog_data *mesh_prog_data = get_mesh_prog_data(pipeline);
         anv_pipeline_emit(pipeline, final.sbe_mesh,
                           GENX(3DSTATE_SBE_MESH), sbe_mesh) {
            const struct brw_mue_map *mue = &mesh_prog_data->map;

            assert(mue->per_vertex_header_size_dw % 8 == 0);
            sbe_mesh.PerVertexURBEntryOutputReadOffset = mue->per_vertex_header_size_dw / 8;
            sbe_mesh.PerVertexURBEntryOutputReadLength = DIV_ROUND_UP(mue->per_vertex_data_size_dw, 8);

            /* Clip distance array is passed in the per-vertex header so that
             * it can be consumed by the HW. If user wants to read it in the
             * FS, adjust the offset and length to cover it. Conveniently it
             * is at the end of the per-vertex header, right before per-vertex
             * attributes.
             *
             * Note that FS attribute reading must be aware that the clip
             * distances have fixed position.
             */
            if (mue->per_vertex_header_size_dw > 8 &&
                (wm_prog_data->urb_setup[VARYING_SLOT_CLIP_DIST0] >= 0 ||
                 wm_prog_data->urb_setup[VARYING_SLOT_CLIP_DIST1] >= 0)) {
               sbe_mesh.PerVertexURBEntryOutputReadOffset -= 1;
               sbe_mesh.PerVertexURBEntryOutputReadLength += 1;
            }

            if (mue->user_data_in_vertex_header) {
               sbe_mesh.PerVertexURBEntryOutputReadOffset -= 1;
               sbe_mesh.PerVertexURBEntryOutputReadLength += 1;
            }

            assert(mue->per_primitive_header_size_dw % 8 == 0);
            sbe_mesh.PerPrimitiveURBEntryOutputReadOffset =
               mue->per_primitive_header_size_dw / 8;
            sbe_mesh.PerPrimitiveURBEntryOutputReadLength =
               DIV_ROUND_UP(mue->per_primitive_data_size_dw, 8);

            /* Just like with clip distances, if Primitive Shading Rate,
             * Viewport Index or Layer is read back in the FS, adjust the
             * offset and length to cover the Primitive Header, where PSR,
             * Viewport Index & Layer are stored.
             */
            if (wm_prog_data->urb_setup[VARYING_SLOT_VIEWPORT] >= 0 ||
                wm_prog_data->urb_setup[VARYING_SLOT_PRIMITIVE_SHADING_RATE] >= 0 ||
                wm_prog_data->urb_setup[VARYING_SLOT_LAYER] >= 0 ||
                mue->user_data_in_primitive_header) {
               assert(sbe_mesh.PerPrimitiveURBEntryOutputReadOffset > 0);
               sbe_mesh.PerPrimitiveURBEntryOutputReadOffset -= 1;
               sbe_mesh.PerPrimitiveURBEntryOutputReadLength += 1;
            }
         }
#endif
      }
   }
   }
}

/** Returns the final polygon mode for rasterization
 *
 * This function takes into account polygon mode, primitive topology and the
 * different shader stages which might generate their own type of primitives.
 */
VkPolygonMode
genX(raster_polygon_mode)(const struct anv_graphics_pipeline *pipeline,
                          VkPolygonMode polygon_mode,
                          VkPrimitiveTopology primitive_topology)
{
   if (anv_pipeline_is_mesh(pipeline)) {
      switch (get_mesh_prog_data(pipeline)->primitive_type) {
      case MESA_PRIM_POINTS:
         return VK_POLYGON_MODE_POINT;
      case MESA_PRIM_LINES:
         return VK_POLYGON_MODE_LINE;
      case MESA_PRIM_TRIANGLES:
         return polygon_mode;
      default:
         unreachable("invalid primitive type for mesh");
      }
   } else if (anv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY)) {
      switch (get_gs_prog_data(pipeline)->output_topology) {
      case _3DPRIM_POINTLIST:
         return VK_POLYGON_MODE_POINT;

      case _3DPRIM_LINELIST:
      case _3DPRIM_LINESTRIP:
      case _3DPRIM_LINELOOP:
         return VK_POLYGON_MODE_LINE;

      case _3DPRIM_TRILIST:
      case _3DPRIM_TRIFAN:
      case _3DPRIM_TRISTRIP:
      case _3DPRIM_RECTLIST:
      case _3DPRIM_QUADLIST:
      case _3DPRIM_QUADSTRIP:
      case _3DPRIM_POLYGON:
         return polygon_mode;
      }
      unreachable("Unsupported GS output topology");
   } else if (anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL)) {
      switch (get_tes_prog_data(pipeline)->output_topology) {
      case INTEL_TESS_OUTPUT_TOPOLOGY_POINT:
         return VK_POLYGON_MODE_POINT;

      case INTEL_TESS_OUTPUT_TOPOLOGY_LINE:
         return VK_POLYGON_MODE_LINE;

      case INTEL_TESS_OUTPUT_TOPOLOGY_TRI_CW:
      case INTEL_TESS_OUTPUT_TOPOLOGY_TRI_CCW:
         return polygon_mode;
      }
      unreachable("Unsupported TCS output topology");
   } else {
      switch (primitive_topology) {
      case VK_PRIMITIVE_TOPOLOGY_POINT_LIST:
         return VK_POLYGON_MODE_POINT;

      case VK_PRIMITIVE_TOPOLOGY_LINE_LIST:
      case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP:
      case VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY:
      case VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY:
         return VK_POLYGON_MODE_LINE;

      case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST:
      case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP:
      case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN:
      case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST_WITH_ADJACENCY:
      case VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY:
         return polygon_mode;

      default:
         unreachable("Unsupported primitive topology");
      }
   }
}

const uint32_t genX(vk_to_intel_cullmode)[] = {
   [VK_CULL_MODE_NONE]                       = CULLMODE_NONE,
   [VK_CULL_MODE_FRONT_BIT]                  = CULLMODE_FRONT,
   [VK_CULL_MODE_BACK_BIT]                   = CULLMODE_BACK,
   [VK_CULL_MODE_FRONT_AND_BACK]             = CULLMODE_BOTH
};

const uint32_t genX(vk_to_intel_fillmode)[] = {
   [VK_POLYGON_MODE_FILL]                    = FILL_MODE_SOLID,
   [VK_POLYGON_MODE_LINE]                    = FILL_MODE_WIREFRAME,
   [VK_POLYGON_MODE_POINT]                   = FILL_MODE_POINT,
};

const uint32_t genX(vk_to_intel_front_face)[] = {
   [VK_FRONT_FACE_COUNTER_CLOCKWISE]         = 1,
   [VK_FRONT_FACE_CLOCKWISE]                 = 0
};

static void
emit_rs_state(struct anv_graphics_pipeline *pipeline,
              const struct vk_input_assembly_state *ia,
              const struct vk_rasterization_state *rs,
              const struct vk_multisample_state *ms,
              const struct vk_render_pass_state *rp,
              enum intel_urb_deref_block_size urb_deref_block_size)
{
   anv_pipeline_emit(pipeline, partial.sf, GENX(3DSTATE_SF), sf) {
      sf.ViewportTransformEnable = true;
      sf.StatisticsEnable = true;
      sf.VertexSubPixelPrecisionSelect = _8Bit;
      sf.AALineDistanceMode = true;

#if GFX_VER >= 12
      sf.DerefBlockSize = urb_deref_block_size;
#endif

      bool point_from_shader;
      if (anv_pipeline_is_primitive(pipeline)) {
         const struct brw_vue_prog_data *last_vue_prog_data =
            anv_pipeline_get_last_vue_prog_data(pipeline);
         point_from_shader = last_vue_prog_data->vue_map.slots_valid & VARYING_BIT_PSIZ;
      } else {
         assert(anv_pipeline_is_mesh(pipeline));
         const struct brw_mesh_prog_data *mesh_prog_data = get_mesh_prog_data(pipeline);
         point_from_shader = mesh_prog_data->map.start_dw[VARYING_SLOT_PSIZ] >= 0;
      }

      if (point_from_shader) {
         sf.PointWidthSource = Vertex;
      } else {
         sf.PointWidthSource = State;
         sf.PointWidth = 1.0;
      }
   }

   anv_pipeline_emit(pipeline, partial.raster, GENX(3DSTATE_RASTER), raster) {
      /* For details on 3DSTATE_RASTER multisample state, see the BSpec table
       * "Multisample Modes State".
       */
      /* NOTE: 3DSTATE_RASTER::ForcedSampleCount affects the BDW and SKL PMA fix
       * computations.  If we ever set this bit to a different value, they will
       * need to be updated accordingly.
       */
      raster.ForcedSampleCount = FSC_NUMRASTSAMPLES_0;
      raster.ForceMultisampling = false;

      raster.ScissorRectangleEnable = true;
   }
}

static void
emit_ms_state(struct anv_graphics_pipeline *pipeline,
              const struct vk_multisample_state *ms)
{
   anv_pipeline_emit(pipeline, partial.ms, GENX(3DSTATE_MULTISAMPLE), ms) {
      ms.PixelLocation              = CENTER;

      /* The PRM says that this bit is valid only for DX9:
       *
       *    SW can choose to set this bit only for DX9 API. DX10/OGL API's
       *    should not have any effect by setting or not setting this bit.
       */
      ms.PixelPositionOffsetEnable  = false;
   }
}

const uint32_t genX(vk_to_intel_logic_op)[] = {
   [VK_LOGIC_OP_COPY]                        = LOGICOP_COPY,
   [VK_LOGIC_OP_CLEAR]                       = LOGICOP_CLEAR,
   [VK_LOGIC_OP_AND]                         = LOGICOP_AND,
   [VK_LOGIC_OP_AND_REVERSE]                 = LOGICOP_AND_REVERSE,
   [VK_LOGIC_OP_AND_INVERTED]                = LOGICOP_AND_INVERTED,
   [VK_LOGIC_OP_NO_OP]                       = LOGICOP_NOOP,
   [VK_LOGIC_OP_XOR]                         = LOGICOP_XOR,
   [VK_LOGIC_OP_OR]                          = LOGICOP_OR,
   [VK_LOGIC_OP_NOR]                         = LOGICOP_NOR,
   [VK_LOGIC_OP_EQUIVALENT]                  = LOGICOP_EQUIV,
   [VK_LOGIC_OP_INVERT]                      = LOGICOP_INVERT,
   [VK_LOGIC_OP_OR_REVERSE]                  = LOGICOP_OR_REVERSE,
   [VK_LOGIC_OP_COPY_INVERTED]               = LOGICOP_COPY_INVERTED,
   [VK_LOGIC_OP_OR_INVERTED]                 = LOGICOP_OR_INVERTED,
   [VK_LOGIC_OP_NAND]                        = LOGICOP_NAND,
   [VK_LOGIC_OP_SET]                         = LOGICOP_SET,
};

const uint32_t genX(vk_to_intel_compare_op)[] = {
   [VK_COMPARE_OP_NEVER]                        = PREFILTEROP_NEVER,
   [VK_COMPARE_OP_LESS]                         = PREFILTEROP_LESS,
   [VK_COMPARE_OP_EQUAL]                        = PREFILTEROP_EQUAL,
   [VK_COMPARE_OP_LESS_OR_EQUAL]                = PREFILTEROP_LEQUAL,
   [VK_COMPARE_OP_GREATER]                      = PREFILTEROP_GREATER,
   [VK_COMPARE_OP_NOT_EQUAL]                    = PREFILTEROP_NOTEQUAL,
   [VK_COMPARE_OP_GREATER_OR_EQUAL]             = PREFILTEROP_GEQUAL,
   [VK_COMPARE_OP_ALWAYS]                       = PREFILTEROP_ALWAYS,
};

const uint32_t genX(vk_to_intel_stencil_op)[] = {
   [VK_STENCIL_OP_KEEP]                         = STENCILOP_KEEP,
   [VK_STENCIL_OP_ZERO]                         = STENCILOP_ZERO,
   [VK_STENCIL_OP_REPLACE]                      = STENCILOP_REPLACE,
   [VK_STENCIL_OP_INCREMENT_AND_CLAMP]          = STENCILOP_INCRSAT,
   [VK_STENCIL_OP_DECREMENT_AND_CLAMP]          = STENCILOP_DECRSAT,
   [VK_STENCIL_OP_INVERT]                       = STENCILOP_INVERT,
   [VK_STENCIL_OP_INCREMENT_AND_WRAP]           = STENCILOP_INCR,
   [VK_STENCIL_OP_DECREMENT_AND_WRAP]           = STENCILOP_DECR,
};

const uint32_t genX(vk_to_intel_primitive_type)[] = {
   [VK_PRIMITIVE_TOPOLOGY_POINT_LIST]                    = _3DPRIM_POINTLIST,
   [VK_PRIMITIVE_TOPOLOGY_LINE_LIST]                     = _3DPRIM_LINELIST,
   [VK_PRIMITIVE_TOPOLOGY_LINE_STRIP]                    = _3DPRIM_LINESTRIP,
   [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST]                 = _3DPRIM_TRILIST,
   [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP]                = _3DPRIM_TRISTRIP,
   [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN]                  = _3DPRIM_TRIFAN,
   [VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY]      = _3DPRIM_LINELIST_ADJ,
   [VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY]     = _3DPRIM_LINESTRIP_ADJ,
   [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST_WITH_ADJACENCY]  = _3DPRIM_TRILIST_ADJ,
   [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY] = _3DPRIM_TRISTRIP_ADJ,
};

static void
emit_3dstate_clip(struct anv_graphics_pipeline *pipeline,
                  const struct vk_input_assembly_state *ia,
                  const struct vk_viewport_state *vp,
                  const struct vk_rasterization_state *rs)
{
   const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);
   (void) wm_prog_data;

   anv_pipeline_emit(pipeline, partial.clip, GENX(3DSTATE_CLIP), clip) {
      clip.ClipEnable               = true;
      clip.StatisticsEnable         = true;
      clip.EarlyCullEnable          = true;
      clip.GuardbandClipTestEnable  = true;

      clip.VertexSubPixelPrecisionSelect = _8Bit;
      clip.ClipMode = CLIPMODE_NORMAL;

      clip.MinimumPointWidth = 0.125;
      clip.MaximumPointWidth = 255.875;

      /* TODO(mesh): Multiview. */
      if (anv_pipeline_is_primitive(pipeline)) {
         const struct brw_vue_prog_data *last =
            anv_pipeline_get_last_vue_prog_data(pipeline);

         /* From the Vulkan 1.0.45 spec:
          *
          *    "If the last active vertex processing stage shader entry point's
          *    interface does not include a variable decorated with Layer, then
          *    the first layer is used."
          */
         clip.ForceZeroRTAIndexEnable =
            !(last->vue_map.slots_valid & VARYING_BIT_LAYER);

      } else if (anv_pipeline_is_mesh(pipeline)) {
         const struct brw_mesh_prog_data *mesh_prog_data = get_mesh_prog_data(pipeline);

         clip.ForceZeroRTAIndexEnable =
            mesh_prog_data->map.start_dw[VARYING_SLOT_LAYER] < 0;
      }

      clip.NonPerspectiveBarycentricEnable = wm_prog_data ?
         wm_prog_data->uses_nonperspective_interp_modes : 0;
   }

#if GFX_VERx10 >= 125
   if (anv_pipeline_is_mesh(pipeline)) {
      const struct brw_mesh_prog_data *mesh_prog_data = get_mesh_prog_data(pipeline);
      anv_pipeline_emit(pipeline, final.clip_mesh,
                        GENX(3DSTATE_CLIP_MESH), clip_mesh) {
         clip_mesh.PrimitiveHeaderEnable = mesh_prog_data->map.per_primitive_header_size_dw > 0;
         clip_mesh.UserClipDistanceClipTestEnableBitmask = mesh_prog_data->clip_distance_mask;
         clip_mesh.UserClipDistanceCullTestEnableBitmask = mesh_prog_data->cull_distance_mask;
      }
   }
#endif
}

static void
emit_3dstate_streamout(struct anv_graphics_pipeline *pipeline,
                       const struct vk_rasterization_state *rs)
{
   const struct brw_vue_prog_data *prog_data =
      anv_pipeline_get_last_vue_prog_data(pipeline);
   const struct intel_vue_map *vue_map = &prog_data->vue_map;

   nir_xfb_info *xfb_info;
   if (anv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY))
      xfb_info = pipeline->base.shaders[MESA_SHADER_GEOMETRY]->xfb_info;
   else if (anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL))
      xfb_info = pipeline->base.shaders[MESA_SHADER_TESS_EVAL]->xfb_info;
   else
      xfb_info = pipeline->base.shaders[MESA_SHADER_VERTEX]->xfb_info;

   if (xfb_info) {
      struct GENX(SO_DECL) so_decl[MAX_XFB_STREAMS][128];
      int next_offset[MAX_XFB_BUFFERS] = {0, 0, 0, 0};
      int decls[MAX_XFB_STREAMS] = {0, 0, 0, 0};

      memset(so_decl, 0, sizeof(so_decl));

      for (unsigned i = 0; i < xfb_info->output_count; i++) {
         const nir_xfb_output_info *output = &xfb_info->outputs[i];
         unsigned buffer = output->buffer;
         unsigned stream = xfb_info->buffer_to_stream[buffer];

         /* Our hardware is unusual in that it requires us to program SO_DECLs
          * for fake "hole" components, rather than simply taking the offset
          * for each real varying.  Each hole can have size 1, 2, 3, or 4; we
          * program as many size = 4 holes as we can, then a final hole to
          * accommodate the final 1, 2, or 3 remaining.
          */
         int hole_dwords = (output->offset - next_offset[buffer]) / 4;
         while (hole_dwords > 0) {
            so_decl[stream][decls[stream]++] = (struct GENX(SO_DECL)) {
               .HoleFlag = 1,
               .OutputBufferSlot = buffer,
               .ComponentMask = (1 << MIN2(hole_dwords, 4)) - 1,
            };
            hole_dwords -= 4;
         }

         int varying = output->location;
         uint8_t component_mask = output->component_mask;
         /* VARYING_SLOT_PSIZ contains four scalar fields packed together:
          * - VARYING_SLOT_PRIMITIVE_SHADING_RATE in VARYING_SLOT_PSIZ.x
          * - VARYING_SLOT_LAYER                  in VARYING_SLOT_PSIZ.y
          * - VARYING_SLOT_VIEWPORT               in VARYING_SLOT_PSIZ.z
          * - VARYING_SLOT_PSIZ                   in VARYING_SLOT_PSIZ.w
          */
         if (varying == VARYING_SLOT_PRIMITIVE_SHADING_RATE) {
            varying = VARYING_SLOT_PSIZ;
            component_mask = 1 << 0; // SO_DECL_COMPMASK_X
         } else if (varying == VARYING_SLOT_LAYER) {
            varying = VARYING_SLOT_PSIZ;
            component_mask = 1 << 1; // SO_DECL_COMPMASK_Y
         } else if (varying == VARYING_SLOT_VIEWPORT) {
            varying = VARYING_SLOT_PSIZ;
            component_mask = 1 << 2; // SO_DECL_COMPMASK_Z
         } else if (varying == VARYING_SLOT_PSIZ) {
            component_mask = 1 << 3; // SO_DECL_COMPMASK_W
         }

         next_offset[buffer] = output->offset +
                               __builtin_popcount(component_mask) * 4;

         const int slot = vue_map->varying_to_slot[varying];
         if (slot < 0) {
            /* This can happen if the shader never writes to the varying.
             * Insert a hole instead of actual varying data.
             */
            so_decl[stream][decls[stream]++] = (struct GENX(SO_DECL)) {
               .HoleFlag = true,
               .OutputBufferSlot = buffer,
               .ComponentMask = component_mask,
            };
         } else {
            so_decl[stream][decls[stream]++] = (struct GENX(SO_DECL)) {
               .OutputBufferSlot = buffer,
               .RegisterIndex = slot,
               .ComponentMask = component_mask,
            };
         }
      }

      int max_decls = 0;
      for (unsigned s = 0; s < MAX_XFB_STREAMS; s++)
         max_decls = MAX2(max_decls, decls[s]);

      uint8_t sbs[MAX_XFB_STREAMS] = { };
      for (unsigned b = 0; b < MAX_XFB_BUFFERS; b++) {
         if (xfb_info->buffers_written & (1 << b))
            sbs[xfb_info->buffer_to_stream[b]] |= 1 << b;
      }

      uint32_t *dw = anv_pipeline_emitn(pipeline, final.so_decl_list,
                                        3 + 2 * max_decls,
                                        GENX(3DSTATE_SO_DECL_LIST),
                                        .StreamtoBufferSelects0 = sbs[0],
                                        .StreamtoBufferSelects1 = sbs[1],
                                        .StreamtoBufferSelects2 = sbs[2],
                                        .StreamtoBufferSelects3 = sbs[3],
                                        .NumEntries0 = decls[0],
                                        .NumEntries1 = decls[1],
                                        .NumEntries2 = decls[2],
                                        .NumEntries3 = decls[3]);

      for (int i = 0; i < max_decls; i++) {
         GENX(SO_DECL_ENTRY_pack)(NULL, dw + 3 + i * 2,
            &(struct GENX(SO_DECL_ENTRY)) {
               .Stream0Decl = so_decl[0][i],
               .Stream1Decl = so_decl[1][i],
               .Stream2Decl = so_decl[2][i],
               .Stream3Decl = so_decl[3][i],
            });
      }
   }

   anv_pipeline_emit(pipeline, partial.so, GENX(3DSTATE_STREAMOUT), so) {
      if (xfb_info) {
         pipeline->uses_xfb = true;

         so.SOFunctionEnable = true;
         so.SOStatisticsEnable = true;

         so.Buffer0SurfacePitch = xfb_info->buffers[0].stride;
         so.Buffer1SurfacePitch = xfb_info->buffers[1].stride;
         so.Buffer2SurfacePitch = xfb_info->buffers[2].stride;
         so.Buffer3SurfacePitch = xfb_info->buffers[3].stride;

         int urb_entry_read_offset = 0;
         int urb_entry_read_length =
            (prog_data->vue_map.num_slots + 1) / 2 - urb_entry_read_offset;

         /* We always read the whole vertex. This could be reduced at some
          * point by reading less and offsetting the register index in the
          * SO_DECLs.
          */
         so.Stream0VertexReadOffset = urb_entry_read_offset;
         so.Stream0VertexReadLength = urb_entry_read_length - 1;
         so.Stream1VertexReadOffset = urb_entry_read_offset;
         so.Stream1VertexReadLength = urb_entry_read_length - 1;
         so.Stream2VertexReadOffset = urb_entry_read_offset;
         so.Stream2VertexReadLength = urb_entry_read_length - 1;
         so.Stream3VertexReadOffset = urb_entry_read_offset;
         so.Stream3VertexReadLength = urb_entry_read_length - 1;
      }
   }
}

static inline uint32_t
get_sampler_count(const struct anv_shader_bin *bin)
{
   /* We can potentially have way more than 32 samplers and that's ok.
    * However, the 3DSTATE_XS packets only have 3 bits to specify how
    * many to pre-fetch and all values above 4 are marked reserved.
    */
   return DIV_ROUND_UP(CLAMP(bin->bind_map.sampler_count, 0, 16), 4);
}

static UNUSED struct anv_address
get_scratch_address(struct anv_pipeline *pipeline,
                    gl_shader_stage stage,
                    const struct anv_shader_bin *bin)
{
   return (struct anv_address) {
      .bo = anv_scratch_pool_alloc(pipeline->device,
                                   &pipeline->device->scratch_pool,
                                   stage, bin->prog_data->total_scratch),
      .offset = 0,
   };
}

static UNUSED uint32_t
get_scratch_space(const struct anv_shader_bin *bin)
{
   return ffs(bin->prog_data->total_scratch / 2048);
}

static UNUSED uint32_t
get_scratch_surf(struct anv_pipeline *pipeline,
                 gl_shader_stage stage,
                 const struct anv_shader_bin *bin,
                 bool protected)
{
   if (bin->prog_data->total_scratch == 0)
      return 0;

   struct anv_scratch_pool *pool = protected ?
      &pipeline->device->protected_scratch_pool :
      &pipeline->device->scratch_pool;
   struct anv_bo *bo =
      anv_scratch_pool_alloc(pipeline->device, pool,
                             stage, bin->prog_data->total_scratch);
   anv_reloc_list_add_bo(pipeline->batch.relocs, bo);
   return anv_scratch_pool_get_surf(pipeline->device, pool,
                                    bin->prog_data->total_scratch) >> ANV_SCRATCH_SPACE_SHIFT(GFX_VER);
}

static void
emit_3dstate_vs(struct anv_graphics_pipeline *pipeline)
{
   const struct intel_device_info *devinfo = pipeline->base.base.device->info;
   const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
   const struct anv_shader_bin *vs_bin =
      pipeline->base.shaders[MESA_SHADER_VERTEX];

   assert(anv_pipeline_has_stage(pipeline, MESA_SHADER_VERTEX));

   uint32_t vs_dwords[GENX(3DSTATE_VS_length)];
   anv_pipeline_emit_tmp(pipeline, vs_dwords, GENX(3DSTATE_VS), vs) {
      vs.Enable               = true;
      vs.StatisticsEnable     = true;
      vs.KernelStartPointer   = vs_bin->kernel.offset;
#if GFX_VER < 20
      vs.SIMD8DispatchEnable  =
         vs_prog_data->base.dispatch_mode == DISPATCH_MODE_SIMD8;
#endif

      assert(!vs_prog_data->base.base.use_alt_mode);
#if GFX_VER < 11
      vs.SingleVertexDispatch       = false;
#endif
      vs.VectorMaskEnable           = false;
      /* Wa_1606682166:
       * Incorrect TDL's SSP address shift in SARB for 16:6 & 18:8 modes.
       * Disable the Sampler state prefetch functionality in the SARB by
       * programming 0xB000[30] to '1'.
       */
      vs.SamplerCount               = GFX_VER == 11 ? 0 : get_sampler_count(vs_bin);
      vs.BindingTableEntryCount     = vs_bin->bind_map.surface_count;
      vs.FloatingPointMode          = IEEE754;
      vs.IllegalOpcodeExceptionEnable = false;
      vs.SoftwareExceptionEnable    = false;
      vs.MaximumNumberofThreads     = devinfo->max_vs_threads - 1;

      if (GFX_VER == 9 && devinfo->gt == 4 &&
          anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL)) {
         /* On Sky Lake GT4, we have experienced some hangs related to the VS
          * cache and tessellation.  It is unknown exactly what is happening
          * but the Haswell docs for the "VS Reference Count Full Force Miss
          * Enable" field of the "Thread Mode" register refer to a HSW bug in
          * which the VUE handle reference count would overflow resulting in
          * internal reference counting bugs.  My (Faith's) best guess is that
          * this bug cropped back up on SKL GT4 when we suddenly had more
          * threads in play than any previous gfx9 hardware.
          *
          * What we do know for sure is that setting this bit when
          * tessellation shaders are in use fixes a GPU hang in Batman: Arkham
          * City when playing with DXVK (https://bugs.freedesktop.org/107280).
          * Disabling the vertex cache with tessellation shaders should only
          * have a minor performance impact as the tessellation shaders are
          * likely generating and processing far more geometry than the vertex
          * stage.
          */
         vs.VertexCacheDisable = true;
      }

      vs.VertexURBEntryReadLength      = vs_prog_data->base.urb_read_length;
      vs.VertexURBEntryReadOffset      = 0;
      vs.DispatchGRFStartRegisterForURBData =
         vs_prog_data->base.base.dispatch_grf_start_reg;

      vs.UserClipDistanceClipTestEnableBitmask =
         vs_prog_data->base.clip_distance_mask;
      vs.UserClipDistanceCullTestEnableBitmask =
         vs_prog_data->base.cull_distance_mask;

#if GFX_VERx10 < 125
      vs.PerThreadScratchSpace   = get_scratch_space(vs_bin);
      vs.ScratchSpaceBasePointer =
         get_scratch_address(&pipeline->base.base, MESA_SHADER_VERTEX, vs_bin);
#endif
   }

   anv_pipeline_emit_merge(pipeline, final.vs, vs_dwords, GENX(3DSTATE_VS), vs) {
#if GFX_VERx10 >= 125
      vs.ScratchSpaceBuffer = get_scratch_surf(&pipeline->base.base,
                                               MESA_SHADER_VERTEX,
                                               vs_bin, false);
#endif
   }
   if (pipeline_needs_protected(&pipeline->base.base)) {
      anv_pipeline_emit_merge(pipeline, final.vs_protected,
                              vs_dwords, GENX(3DSTATE_VS), vs) {
#if GFX_VERx10 >= 125
         vs.ScratchSpaceBuffer = get_scratch_surf(&pipeline->base.base,
                                                  MESA_SHADER_VERTEX,
                                                  vs_bin, true);
#endif
      }
   }
}

static void
emit_3dstate_hs_ds(struct anv_graphics_pipeline *pipeline,
                   const struct vk_tessellation_state *ts)
{
   if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL)) {
      anv_pipeline_emit(pipeline, final.hs, GENX(3DSTATE_HS), hs);
      anv_pipeline_emit(pipeline, final.hs_protected, GENX(3DSTATE_HS), hs);
      anv_pipeline_emit(pipeline, final.ds, GENX(3DSTATE_DS), ds);
      anv_pipeline_emit(pipeline, final.ds_protected, GENX(3DSTATE_DS), ds);
      return;
   }

   const struct intel_device_info *devinfo = pipeline->base.base.device->info;
   const struct anv_shader_bin *tcs_bin =
      pipeline->base.shaders[MESA_SHADER_TESS_CTRL];
   const struct anv_shader_bin *tes_bin =
      pipeline->base.shaders[MESA_SHADER_TESS_EVAL];

   const struct brw_tcs_prog_data *tcs_prog_data = get_tcs_prog_data(pipeline);
   const struct brw_tes_prog_data *tes_prog_data = get_tes_prog_data(pipeline);

   uint32_t hs_dwords[GENX(3DSTATE_HS_length)];
   anv_pipeline_emit_tmp(pipeline, hs_dwords, GENX(3DSTATE_HS), hs) {
      hs.Enable = true;
      hs.StatisticsEnable = true;
      hs.KernelStartPointer = tcs_bin->kernel.offset;
      /* Wa_1606682166 */
      hs.SamplerCount = GFX_VER == 11 ? 0 : get_sampler_count(tcs_bin);
      hs.BindingTableEntryCount = tcs_bin->bind_map.surface_count;

#if GFX_VER >= 12
      /* Wa_1604578095:
       *
       *    Hang occurs when the number of max threads is less than 2 times
       *    the number of instance count. The number of max threads must be
       *    more than 2 times the number of instance count.
       */
      assert((devinfo->max_tcs_threads / 2) > tcs_prog_data->instances);
#endif

      hs.MaximumNumberofThreads = devinfo->max_tcs_threads - 1;
      hs.IncludeVertexHandles = true;
      hs.InstanceCount = tcs_prog_data->instances - 1;

      hs.VertexURBEntryReadLength = 0;
      hs.VertexURBEntryReadOffset = 0;
      hs.DispatchGRFStartRegisterForURBData =
         tcs_prog_data->base.base.dispatch_grf_start_reg & 0x1f;
#if GFX_VER >= 12
      hs.DispatchGRFStartRegisterForURBData5 =
         tcs_prog_data->base.base.dispatch_grf_start_reg >> 5;
#endif

#if GFX_VERx10 < 125
      hs.PerThreadScratchSpace = get_scratch_space(tcs_bin);
      hs.ScratchSpaceBasePointer =
         get_scratch_address(&pipeline->base.base, MESA_SHADER_TESS_CTRL, tcs_bin);
#endif

#if GFX_VER == 12
      /*  Patch Count threshold specifies the maximum number of patches that
       *  will be accumulated before a thread dispatch is forced.
       */
      hs.PatchCountThreshold = tcs_prog_data->patch_count_threshold;
#endif

#if GFX_VER < 20
      hs.DispatchMode = tcs_prog_data->base.dispatch_mode;
#endif
      hs.IncludePrimitiveID = tcs_prog_data->include_primitive_id;
   };

   uint32_t ds_dwords[GENX(3DSTATE_DS_length)];
   anv_pipeline_emit_tmp(pipeline, ds_dwords, GENX(3DSTATE_DS), ds) {
      ds.Enable = true;
      ds.StatisticsEnable = true;
      ds.KernelStartPointer = tes_bin->kernel.offset;
      /* Wa_1606682166 */
      ds.SamplerCount = GFX_VER == 11 ? 0 : get_sampler_count(tes_bin);
      ds.BindingTableEntryCount = tes_bin->bind_map.surface_count;
      ds.MaximumNumberofThreads = devinfo->max_tes_threads - 1;

      ds.ComputeWCoordinateEnable =
         tes_prog_data->domain == INTEL_TESS_DOMAIN_TRI;

      ds.PatchURBEntryReadLength = tes_prog_data->base.urb_read_length;
      ds.PatchURBEntryReadOffset = 0;
      ds.DispatchGRFStartRegisterForURBData =
         tes_prog_data->base.base.dispatch_grf_start_reg;

#if GFX_VER < 11
      ds.DispatchMode =
         tes_prog_data->base.dispatch_mode == DISPATCH_MODE_SIMD8 ?
         DISPATCH_MODE_SIMD8_SINGLE_PATCH :
         DISPATCH_MODE_SIMD4X2;
#else
      assert(tes_prog_data->base.dispatch_mode == INTEL_DISPATCH_MODE_SIMD8);
      ds.DispatchMode = DISPATCH_MODE_SIMD8_SINGLE_PATCH;
#endif

      ds.UserClipDistanceClipTestEnableBitmask =
         tes_prog_data->base.clip_distance_mask;
      ds.UserClipDistanceCullTestEnableBitmask =
         tes_prog_data->base.cull_distance_mask;

#if GFX_VER >= 12
      ds.PrimitiveIDNotRequired = !tes_prog_data->include_primitive_id;
#endif
#if GFX_VERx10 < 125
      ds.PerThreadScratchSpace = get_scratch_space(tes_bin);
      ds.ScratchSpaceBasePointer =
         get_scratch_address(&pipeline->base.base, MESA_SHADER_TESS_EVAL, tes_bin);
#endif
   }

   anv_pipeline_emit_merge(pipeline, final.hs, hs_dwords, GENX(3DSTATE_HS), hs) {
#if GFX_VERx10 >= 125
      hs.ScratchSpaceBuffer = get_scratch_surf(&pipeline->base.base,
                                               MESA_SHADER_TESS_CTRL,
                                               tcs_bin, false);
#endif
   }
   anv_pipeline_emit_merge(pipeline, final.ds, ds_dwords, GENX(3DSTATE_DS), ds) {
#if GFX_VERx10 >= 125
      ds.ScratchSpaceBuffer = get_scratch_surf(&pipeline->base.base,
                                               MESA_SHADER_TESS_EVAL,
                                               tes_bin, false);
#endif
   }
   if (pipeline_needs_protected(&pipeline->base.base)) {
      anv_pipeline_emit_merge(pipeline, final.hs_protected,
                              hs_dwords, GENX(3DSTATE_HS), hs) {
#if GFX_VERx10 >= 125
         hs.ScratchSpaceBuffer = get_scratch_surf(&pipeline->base.base,
                                                  MESA_SHADER_TESS_CTRL,
                                                  tcs_bin, true);
#endif
      }
      anv_pipeline_emit_merge(pipeline, final.ds_protected,
                              ds_dwords, GENX(3DSTATE_DS), ds) {
#if GFX_VERx10 >= 125
         ds.ScratchSpaceBuffer = get_scratch_surf(&pipeline->base.base,
                                                  MESA_SHADER_TESS_EVAL,
                                                  tes_bin, true);
#endif
      }
   }
}

static UNUSED bool
geom_or_tess_prim_id_used(struct anv_graphics_pipeline *pipeline)
{
   const struct brw_tcs_prog_data *tcs_prog_data =
      anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_CTRL) ?
      get_tcs_prog_data(pipeline) : NULL;
   const struct brw_tes_prog_data *tes_prog_data =
      anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL) ?
      get_tes_prog_data(pipeline) : NULL;
   const struct brw_gs_prog_data *gs_prog_data =
      anv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY) ?
      get_gs_prog_data(pipeline) : NULL;

   return (tcs_prog_data && tcs_prog_data->include_primitive_id) ||
          (tes_prog_data && tes_prog_data->include_primitive_id) ||
          (gs_prog_data && gs_prog_data->include_primitive_id);
}

static void
emit_3dstate_te(struct anv_graphics_pipeline *pipeline)
{
   anv_pipeline_emit(pipeline, partial.te, GENX(3DSTATE_TE), te) {
      if (anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL)) {
         const struct brw_tes_prog_data *tes_prog_data =
            get_tes_prog_data(pipeline);

         te.Partitioning = tes_prog_data->partitioning;
         te.TEDomain = tes_prog_data->domain;
         te.TEEnable = true;
         te.MaximumTessellationFactorOdd = 63.0;
         te.MaximumTessellationFactorNotOdd = 64.0;
#if GFX_VERx10 >= 125
         const struct anv_device *device = pipeline->base.base.device;
         if (intel_needs_workaround(device->info, 22012699309))
            te.TessellationDistributionMode = TEDMODE_RR_STRICT;
         else
            te.TessellationDistributionMode = TEDMODE_RR_FREE;

         if (intel_needs_workaround(device->info, 14015055625)) {
            /* Wa_14015055625:
             *
             * Disable Tessellation Distribution when primitive Id is enabled.
             */
            if (sbe_primitive_id_override(pipeline) ||
                geom_or_tess_prim_id_used(pipeline))
               te.TessellationDistributionMode = TEDMODE_OFF;
         }

#if GFX_VER >= 20
         te.TessellationDistributionLevel = TEDLEVEL_REGION;
#else
         te.TessellationDistributionLevel = TEDLEVEL_PATCH;
#endif
         /* 64_TRIANGLES */
         te.SmallPatchThreshold = 3;
         /* 1K_TRIANGLES */
         te.TargetBlockSize = 8;
         /* 1K_TRIANGLES */
         te.LocalBOPAccumulatorThreshold = 1;
#endif

#if GFX_VER >= 20
         te.NumberOfRegionsPerPatch = 2;
#endif
      }
   }
}

static void
emit_3dstate_gs(struct anv_graphics_pipeline *pipeline)
{
   if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY)) {
      anv_pipeline_emit(pipeline, partial.gs, GENX(3DSTATE_GS), gs);
      anv_pipeline_emit(pipeline, partial.gs_protected, GENX(3DSTATE_GS), gs);
      return;
   }

   const struct intel_device_info *devinfo = pipeline->base.base.device->info;
   const struct anv_shader_bin *gs_bin =
      pipeline->base.shaders[MESA_SHADER_GEOMETRY];
   const struct brw_gs_prog_data *gs_prog_data = get_gs_prog_data(pipeline);

   uint32_t gs_dwords[GENX(3DSTATE_GS_length)];
   anv_pipeline_emit_tmp(pipeline, gs_dwords, GENX(3DSTATE_GS), gs) {
      gs.Enable                  = true;
      gs.StatisticsEnable        = true;
      gs.KernelStartPointer      = gs_bin->kernel.offset;
#if GFX_VER < 20
      gs.DispatchMode            = gs_prog_data->base.dispatch_mode;
#endif

      gs.SingleProgramFlow       = false;
      gs.VectorMaskEnable        = false;
      /* Wa_1606682166 */
      gs.SamplerCount            = GFX_VER == 11 ? 0 : get_sampler_count(gs_bin);
      gs.BindingTableEntryCount  = gs_bin->bind_map.surface_count;
      gs.IncludeVertexHandles    = gs_prog_data->base.include_vue_handles;
      gs.IncludePrimitiveID      = gs_prog_data->include_primitive_id;

      gs.MaximumNumberofThreads = devinfo->max_gs_threads - 1;

      gs.OutputVertexSize        = gs_prog_data->output_vertex_size_hwords * 2 - 1;
      gs.OutputTopology          = gs_prog_data->output_topology;
      gs.ControlDataFormat       = gs_prog_data->control_data_format;
      gs.ControlDataHeaderSize   = gs_prog_data->control_data_header_size_hwords;
      gs.InstanceControl         = MAX2(gs_prog_data->invocations, 1) - 1;

      gs.ExpectedVertexCount     = gs_prog_data->vertices_in;
      gs.StaticOutput            = gs_prog_data->static_vertex_count >= 0;
      gs.StaticOutputVertexCount = gs_prog_data->static_vertex_count >= 0 ?
         gs_prog_data->static_vertex_count : 0;

      gs.VertexURBEntryReadOffset = 0;
      gs.VertexURBEntryReadLength = gs_prog_data->base.urb_read_length;
      gs.DispatchGRFStartRegisterForURBData =
         gs_prog_data->base.base.dispatch_grf_start_reg;

      gs.UserClipDistanceClipTestEnableBitmask =
         gs_prog_data->base.clip_distance_mask;
      gs.UserClipDistanceCullTestEnableBitmask =
         gs_prog_data->base.cull_distance_mask;

#if GFX_VERx10 < 125
      gs.PerThreadScratchSpace   = get_scratch_space(gs_bin);
      gs.ScratchSpaceBasePointer =
         get_scratch_address(&pipeline->base.base, MESA_SHADER_GEOMETRY, gs_bin);
#endif
   }

   anv_pipeline_emit_merge(pipeline, partial.gs, gs_dwords, GENX(3DSTATE_GS), gs) {
#if GFX_VERx10 >= 125
      gs.ScratchSpaceBuffer =
         get_scratch_surf(&pipeline->base.base, MESA_SHADER_GEOMETRY, gs_bin, false);
#endif
   }
   if (pipeline_needs_protected(&pipeline->base.base)) {
      anv_pipeline_emit_merge(pipeline, partial.gs_protected,
                              gs_dwords, GENX(3DSTATE_GS), gs) {
#if GFX_VERx10 >= 125
         gs.ScratchSpaceBuffer = get_scratch_surf(&pipeline->base.base,
                                                  MESA_SHADER_GEOMETRY,
                                                  gs_bin, true);
#endif
      }
   }
}

static void
emit_3dstate_wm(struct anv_graphics_pipeline *pipeline,
                const struct vk_input_assembly_state *ia,
                const struct vk_rasterization_state *rs,
                const struct vk_multisample_state *ms,
                const struct vk_color_blend_state *cb,
                const struct vk_render_pass_state *rp)
{
   const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);

   anv_pipeline_emit(pipeline, partial.wm, GENX(3DSTATE_WM), wm) {
      wm.StatisticsEnable                    = true;
      wm.LineEndCapAntialiasingRegionWidth   = _05pixels;
      wm.LineAntialiasingRegionWidth         = _10pixels;
      wm.PointRasterizationRule              = RASTRULE_UPPER_LEFT;

      if (anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT)) {
         if (wm_prog_data->early_fragment_tests) {
            wm.EarlyDepthStencilControl         = EDSC_PREPS;
         } else if (wm_prog_data->has_side_effects) {
            wm.EarlyDepthStencilControl         = EDSC_PSEXEC;
         } else {
            wm.EarlyDepthStencilControl         = EDSC_NORMAL;
         }
      }
   }
}

static void
emit_3dstate_ps(struct anv_graphics_pipeline *pipeline,
                const struct vk_multisample_state *ms,
                const struct vk_color_blend_state *cb)
{
   UNUSED const struct intel_device_info *devinfo =
      pipeline->base.base.device->info;
   const struct anv_shader_bin *fs_bin =
      pipeline->base.shaders[MESA_SHADER_FRAGMENT];

   if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT)) {
      anv_pipeline_emit(pipeline, partial.ps, GENX(3DSTATE_PS), ps);
      anv_pipeline_emit(pipeline, partial.ps_protected, GENX(3DSTATE_PS), ps);
      return;
   }

   const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);

   uint32_t ps_dwords[GENX(3DSTATE_PS_length)];
   anv_pipeline_emit_tmp(pipeline, ps_dwords, GENX(3DSTATE_PS), ps) {
#if GFX_VER == 12
      assert(wm_prog_data->dispatch_multi == 0 ||
             (wm_prog_data->dispatch_multi == 16 && wm_prog_data->max_polygons == 2));
      ps.DualSIMD8DispatchEnable = wm_prog_data->dispatch_multi;
      /* XXX - No major improvement observed from enabling
       *       overlapping subspans, but it could be helpful
       *       in theory when the requirements listed on the
       *       BSpec page for 3DSTATE_PS_BODY are met.
       */
      ps.OverlappingSubspansEnable = false;
#endif

      ps.SingleProgramFlow          = false;
      ps.VectorMaskEnable           = wm_prog_data->uses_vmask;
      /* Wa_1606682166 */
      ps.SamplerCount               = GFX_VER == 11 ? 0 : get_sampler_count(fs_bin);
      ps.BindingTableEntryCount     = fs_bin->bind_map.surface_count;
#if GFX_VER < 20
      ps.PushConstantEnable         =
         devinfo->needs_null_push_constant_tbimr_workaround ||
         wm_prog_data->base.nr_params > 0 ||
         wm_prog_data->base.ubo_ranges[0].length;
#endif

      ps.MaximumNumberofThreadsPerPSD = devinfo->max_threads_per_psd - 1;

#if GFX_VERx10 < 125
      ps.PerThreadScratchSpace   = get_scratch_space(fs_bin);
      ps.ScratchSpaceBasePointer =
         get_scratch_address(&pipeline->base.base, MESA_SHADER_FRAGMENT, fs_bin);
#endif
   }
   anv_pipeline_emit_merge(pipeline, partial.ps, ps_dwords, GENX(3DSTATE_PS), ps) {
#if GFX_VERx10 >= 125
      ps.ScratchSpaceBuffer =
         get_scratch_surf(&pipeline->base.base, MESA_SHADER_FRAGMENT, fs_bin, false);
#endif
   }
   if (pipeline_needs_protected(&pipeline->base.base)) {
      anv_pipeline_emit_merge(pipeline, partial.ps_protected,
                              ps_dwords, GENX(3DSTATE_PS), ps) {
#if GFX_VERx10 >= 125
         ps.ScratchSpaceBuffer =
            get_scratch_surf(&pipeline->base.base, MESA_SHADER_FRAGMENT, fs_bin, true);
#endif
      }
   }
}

static void
emit_3dstate_ps_extra(struct anv_graphics_pipeline *pipeline,
                      const struct vk_rasterization_state *rs,
                      const struct vk_graphics_pipeline_state *state)
{
   const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);

   if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT)) {
      anv_pipeline_emit(pipeline, partial.ps_extra, GENX(3DSTATE_PS_EXTRA), ps);
      return;
   }

   anv_pipeline_emit(pipeline, partial.ps_extra, GENX(3DSTATE_PS_EXTRA), ps) {
      ps.PixelShaderValid              = true;
#if GFX_VER < 20
      ps.AttributeEnable               = wm_prog_data->num_varying_inputs > 0;
#endif
      ps.oMaskPresenttoRenderTarget    = wm_prog_data->uses_omask;
      ps.PixelShaderComputedDepthMode  = wm_prog_data->computed_depth_mode;
      ps.PixelShaderUsesSourceDepth    = wm_prog_data->uses_src_depth;
      ps.PixelShaderUsesSourceW        = wm_prog_data->uses_src_w;

      ps.PixelShaderComputesStencil = wm_prog_data->computed_stencil;
#if GFX_VER >= 20
      assert(!wm_prog_data->pulls_bary);
#else
      ps.PixelShaderPullsBary    = wm_prog_data->pulls_bary;
#endif

      ps.InputCoverageMaskState = ICMS_NONE;
      assert(!wm_prog_data->inner_coverage); /* Not available in SPIR-V */
      if (!wm_prog_data->uses_sample_mask)
         ps.InputCoverageMaskState = ICMS_NONE;
      else if (brw_wm_prog_data_is_coarse(wm_prog_data, 0))
         ps.InputCoverageMaskState  = ICMS_NORMAL;
      else if (wm_prog_data->post_depth_coverage)
         ps.InputCoverageMaskState = ICMS_DEPTH_COVERAGE;
      else
         ps.InputCoverageMaskState = ICMS_NORMAL;

#if GFX_VER >= 11
      ps.PixelShaderRequiresSubpixelSampleOffsets =
         wm_prog_data->uses_sample_offsets;
      ps.PixelShaderRequiresNonPerspectiveBaryPlaneCoefficients =
         wm_prog_data->uses_npc_bary_coefficients;
      ps.PixelShaderRequiresPerspectiveBaryPlaneCoefficients =
         wm_prog_data->uses_pc_bary_coefficients;
      ps.PixelShaderRequiresSourceDepthandorWPlaneCoefficients =
         wm_prog_data->uses_depth_w_coefficients;
#endif
   }
}

static void
emit_3dstate_vf_statistics(struct anv_graphics_pipeline *pipeline)
{
   anv_pipeline_emit(pipeline, final.vf_statistics,
                     GENX(3DSTATE_VF_STATISTICS), vfs) {
      vfs.StatisticsEnable = true;
   }
}

static void
compute_kill_pixel(struct anv_graphics_pipeline *pipeline,
                   const struct vk_multisample_state *ms,
                   const struct vk_graphics_pipeline_state *state)
{
   if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_FRAGMENT)) {
      pipeline->kill_pixel = false;
      return;
   }

   const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline);

   /* This computes the KillPixel portion of the computation for whether or
    * not we want to enable the PMA fix on gfx8 or gfx9.  It's given by this
    * chunk of the giant formula:
    *
    *    (3DSTATE_PS_EXTRA::PixelShaderKillsPixels ||
    *     3DSTATE_PS_EXTRA::oMask Present to RenderTarget ||
    *     3DSTATE_PS_BLEND::AlphaToCoverageEnable ||
    *     3DSTATE_PS_BLEND::AlphaTestEnable ||
    *     3DSTATE_WM_CHROMAKEY::ChromaKeyKillEnable)
    *
    * 3DSTATE_WM_CHROMAKEY::ChromaKeyKillEnable is always false and so is
    * 3DSTATE_PS_BLEND::AlphaTestEnable since Vulkan doesn't have a concept
    * of an alpha test.
    */
   pipeline->rp_has_ds_self_dep =
      (state->pipeline_flags &
       VK_PIPELINE_CREATE_2_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT) != 0;
   pipeline->kill_pixel =
      pipeline->rp_has_ds_self_dep ||
      wm_prog_data->uses_kill ||
      wm_prog_data->uses_omask ||
      (ms && ms->alpha_to_coverage_enable);
}

#if GFX_VER >= 12
static void
emit_3dstate_primitive_replication(struct anv_graphics_pipeline *pipeline,
                                   const struct vk_render_pass_state *rp)
{
   if (anv_pipeline_is_mesh(pipeline)) {
      anv_pipeline_emit(pipeline, final.primitive_replication,
                        GENX(3DSTATE_PRIMITIVE_REPLICATION), pr);
      return;
   }

   const int replication_count =
      anv_pipeline_get_last_vue_prog_data(pipeline)->vue_map.num_pos_slots;

   assert(replication_count >= 1);
   if (replication_count == 1) {
      anv_pipeline_emit(pipeline, final.primitive_replication,
                        GENX(3DSTATE_PRIMITIVE_REPLICATION), pr);
      return;
   }

   assert(replication_count == util_bitcount(rp->view_mask));
   assert(replication_count <= MAX_VIEWS_FOR_PRIMITIVE_REPLICATION);

   anv_pipeline_emit(pipeline, final.primitive_replication,
                     GENX(3DSTATE_PRIMITIVE_REPLICATION), pr) {
      pr.ReplicaMask = (1 << replication_count) - 1;
      pr.ReplicationCount = replication_count - 1;

      int i = 0;
      u_foreach_bit(view_index, rp->view_mask) {
         pr.RTAIOffset[i] = view_index;
         i++;
      }
   }
}
#endif

#if GFX_VERx10 >= 125
static void
emit_task_state(struct anv_graphics_pipeline *pipeline)
{
   assert(anv_pipeline_is_mesh(pipeline));

   if (!anv_pipeline_has_stage(pipeline, MESA_SHADER_TASK)) {
      anv_pipeline_emit(pipeline, final.task_control,
                        GENX(3DSTATE_TASK_CONTROL), zero);
      anv_pipeline_emit(pipeline, final.task_control_protected,
                        GENX(3DSTATE_TASK_CONTROL), zero);
      anv_pipeline_emit(pipeline, final.task_shader,
                        GENX(3DSTATE_TASK_SHADER), zero);
      anv_pipeline_emit(pipeline, final.task_redistrib,
                        GENX(3DSTATE_TASK_REDISTRIB), zero);
      return;
   }

   const struct anv_shader_bin *task_bin =
      pipeline->base.shaders[MESA_SHADER_TASK];

   uint32_t task_control_dwords[GENX(3DSTATE_TASK_CONTROL_length)];
   anv_pipeline_emit_tmp(pipeline, task_control_dwords, GENX(3DSTATE_TASK_CONTROL), tc) {
      tc.TaskShaderEnable = true;
      tc.StatisticsEnable = true;
      tc.MaximumNumberofThreadGroups = 511;
   }

   anv_pipeline_emit_merge(pipeline, final.task_control,
                           task_control_dwords, GENX(3DSTATE_TASK_CONTROL), tc) {
      tc.ScratchSpaceBuffer =
         get_scratch_surf(&pipeline->base.base, MESA_SHADER_TASK, task_bin, false);
   }
   if (pipeline_needs_protected(&pipeline->base.base)) {
      anv_pipeline_emit_merge(pipeline, final.task_control_protected,
                              task_control_dwords, GENX(3DSTATE_TASK_CONTROL), tc) {
         tc.ScratchSpaceBuffer =
            get_scratch_surf(&pipeline->base.base, MESA_SHADER_TASK, task_bin, true);
      }
   }

   const struct intel_device_info *devinfo = pipeline->base.base.device->info;
   const struct brw_task_prog_data *task_prog_data = get_task_prog_data(pipeline);
   const struct intel_cs_dispatch_info task_dispatch =
      brw_cs_get_dispatch_info(devinfo, &task_prog_data->base, NULL);

   anv_pipeline_emit(pipeline, final.task_shader,
                     GENX(3DSTATE_TASK_SHADER), task) {
      task.KernelStartPointer                = task_bin->kernel.offset;
      task.SIMDSize                          = task_dispatch.simd_size / 16;
      task.MessageSIMD                       = task.SIMDSize;
      task.NumberofThreadsinGPGPUThreadGroup = task_dispatch.threads;
      task.ExecutionMask                     = task_dispatch.right_mask;
      task.LocalXMaximum                     = task_dispatch.group_size - 1;
      task.EmitLocalIDX                      = true;

      task.NumberofBarriers                  = task_prog_data->base.uses_barrier;
      task.SharedLocalMemorySize             =
         intel_compute_slm_encode_size(GFX_VER, task_prog_data->base.base.total_shared);
      task.PreferredSLMAllocationSize        =
         intel_compute_preferred_slm_calc_encode_size(devinfo,
                                                      task_prog_data->base.base.total_shared,
                                                      task_dispatch.group_size,
                                                      task_dispatch.simd_size);

      /*
       * 3DSTATE_TASK_SHADER_DATA.InlineData[0:1] will be used for an address
       * of a buffer with push constants and descriptor set table and
       * InlineData[2:7] will be used for first few push constants.
       */
      task.EmitInlineParameter = true;

      task.XP0Required = task_prog_data->uses_drawid;
   }

   /* Recommended values from "Task and Mesh Distribution Programming". */
   anv_pipeline_emit(pipeline, final.task_redistrib,
                     GENX(3DSTATE_TASK_REDISTRIB), redistrib) {
      redistrib.LocalBOTAccumulatorThreshold = MULTIPLIER_1;
      redistrib.SmallTaskThreshold = 1; /* 2^N */
      redistrib.TargetMeshBatchSize = devinfo->num_slices > 2 ? 3 : 5; /* 2^N */
      redistrib.TaskRedistributionLevel = TASKREDISTRIB_BOM;
      redistrib.TaskRedistributionMode = TASKREDISTRIB_RR_STRICT;
   }
}

static void
emit_mesh_state(struct anv_graphics_pipeline *pipeline)
{
   assert(anv_pipeline_is_mesh(pipeline));

   const struct anv_shader_bin *mesh_bin = pipeline->base.shaders[MESA_SHADER_MESH];
   const struct brw_mesh_prog_data *mesh_prog_data = get_mesh_prog_data(pipeline);

   uint32_t mesh_control_dwords[GENX(3DSTATE_MESH_CONTROL_length)];
   anv_pipeline_emit_tmp(pipeline, mesh_control_dwords, GENX(3DSTATE_MESH_CONTROL), mc) {
      mc.MeshShaderEnable = true;
      mc.StatisticsEnable = true;
      mc.MaximumNumberofThreadGroups = 511;
#if GFX_VER >= 20
      mc.VPandRTAIndexAutostripEnable = mesh_prog_data->autostrip_enable;
#endif
   }

   anv_pipeline_emit_merge(pipeline, final.mesh_control,
                           mesh_control_dwords, GENX(3DSTATE_MESH_CONTROL), mc) {
      mc.ScratchSpaceBuffer =
         get_scratch_surf(&pipeline->base.base, MESA_SHADER_MESH, mesh_bin, false);
   }
   if (pipeline_needs_protected(&pipeline->base.base)) {
      anv_pipeline_emit_merge(pipeline, final.mesh_control_protected,
                           mesh_control_dwords, GENX(3DSTATE_MESH_CONTROL), mc) {
         mc.ScratchSpaceBuffer =
            get_scratch_surf(&pipeline->base.base, MESA_SHADER_MESH, mesh_bin, true);
      }
   }

   const struct intel_device_info *devinfo = pipeline->base.base.device->info;
   const struct intel_cs_dispatch_info mesh_dispatch =
      brw_cs_get_dispatch_info(devinfo, &mesh_prog_data->base, NULL);

   const unsigned output_topology =
      mesh_prog_data->primitive_type == MESA_PRIM_POINTS ? OUTPUT_POINT :
      mesh_prog_data->primitive_type == MESA_PRIM_LINES  ? OUTPUT_LINE :
                                                             OUTPUT_TRI;

   uint32_t index_format;
   switch (mesh_prog_data->index_format) {
   case BRW_INDEX_FORMAT_U32:
      index_format = INDEX_U32;
      break;
   case BRW_INDEX_FORMAT_U888X:
      index_format = INDEX_U888X;
      break;
   default:
      unreachable("invalid index format");
   }

   anv_pipeline_emit(pipeline, final.mesh_shader,
                     GENX(3DSTATE_MESH_SHADER), mesh) {
      mesh.KernelStartPointer                = mesh_bin->kernel.offset;
      mesh.SIMDSize                          = mesh_dispatch.simd_size / 16;
      mesh.MessageSIMD                       = mesh.SIMDSize;
      mesh.NumberofThreadsinGPGPUThreadGroup = mesh_dispatch.threads;
      mesh.ExecutionMask                     = mesh_dispatch.right_mask;
      mesh.LocalXMaximum                     = mesh_dispatch.group_size - 1;
      mesh.EmitLocalIDX                      = true;

      mesh.MaximumPrimitiveCount             = MAX2(mesh_prog_data->map.max_primitives, 1) - 1;
      mesh.OutputTopology                    = output_topology;
      mesh.PerVertexDataPitch                = mesh_prog_data->map.per_vertex_pitch_dw / 8;
      mesh.PerPrimitiveDataPresent           = mesh_prog_data->map.per_primitive_pitch_dw > 0;
      mesh.PerPrimitiveDataPitch             = mesh_prog_data->map.per_primitive_pitch_dw / 8;
      mesh.IndexFormat                       = index_format;

      mesh.NumberofBarriers                  = mesh_prog_data->base.uses_barrier;
      mesh.SharedLocalMemorySize             =
         intel_compute_slm_encode_size(GFX_VER, mesh_prog_data->base.base.total_shared);
      mesh.PreferredSLMAllocationSize        =
         intel_compute_preferred_slm_calc_encode_size(devinfo,
                                                      mesh_prog_data->base.base.total_shared,
                                                      mesh_dispatch.group_size,
                                                      mesh_dispatch.simd_size);

      /*
       * 3DSTATE_MESH_SHADER_DATA.InlineData[0:1] will be used for an address
       * of a buffer with push constants and descriptor set table and
       * InlineData[2:7] will be used for first few push constants.
       */
      mesh.EmitInlineParameter = true;

      mesh.XP0Required = mesh_prog_data->uses_drawid;
   }

   /* Recommended values from "Task and Mesh Distribution Programming". */
   anv_pipeline_emit(pipeline, final.mesh_distrib,
                     GENX(3DSTATE_MESH_DISTRIB), distrib) {
      distrib.DistributionMode = MESH_RR_FREE;
      distrib.TaskDistributionBatchSize = devinfo->num_slices > 2 ? 4 : 9; /* 2^N thread groups */
      distrib.MeshDistributionBatchSize = devinfo->num_slices > 2 ? 3 : 3; /* 2^N thread groups */
   }
}
#endif

void
genX(graphics_pipeline_emit)(struct anv_graphics_pipeline *pipeline,
                             const struct vk_graphics_pipeline_state *state)
{
   enum intel_urb_deref_block_size urb_deref_block_size;
   emit_urb_setup(pipeline, &urb_deref_block_size);

   emit_rs_state(pipeline, state->ia, state->rs, state->ms, state->rp,
                 urb_deref_block_size);
   emit_ms_state(pipeline, state->ms);
   compute_kill_pixel(pipeline, state->ms, state);

   emit_3dstate_clip(pipeline, state->ia, state->vp, state->rs);

#if GFX_VER >= 12
   emit_3dstate_primitive_replication(pipeline, state->rp);
#endif

#if GFX_VERx10 >= 125
   bool needs_instance_granularity =
      intel_needs_workaround(pipeline->base.base.device->info, 14019166699) &&
      (sbe_primitive_id_override(pipeline) ||
       geom_or_tess_prim_id_used(pipeline));

   anv_pipeline_emit(pipeline, partial.vfg, GENX(3DSTATE_VFG), vfg) {
      /* If 3DSTATE_TE: TE Enable == 1 then RR_STRICT else RR_FREE*/
      vfg.DistributionMode =
         anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL) ? RR_STRICT :
         RR_FREE;
      vfg.DistributionGranularity = needs_instance_granularity ?
         InstanceLevelGranularity : BatchLevelGranularity;
#if INTEL_WA_14014851047_GFX_VER
      vfg.GranularityThresholdDisable =
         intel_needs_workaround(pipeline->base.base.device->info, 14014851047);
#endif
      /* 192 vertices for TRILIST_ADJ */
      vfg.ListNBatchSizeScale = 0;
      /* Batch size of 384 vertices */
      vfg.List3BatchSizeScale = 2;
      /* Batch size of 128 vertices */
      vfg.List2BatchSizeScale = 1;
      /* Batch size of 128 vertices */
      vfg.List1BatchSizeScale = 2;
      /* Batch size of 256 vertices for STRIP topologies */
      vfg.StripBatchSizeScale = 3;
      /* 192 control points for PATCHLIST_3 */
      vfg.PatchBatchSizeScale = 1;
      /* 192 control points for PATCHLIST_3 */
      vfg.PatchBatchSizeMultiplier = 31;
   }
#endif

   emit_3dstate_vf_statistics(pipeline);

   if (anv_pipeline_is_primitive(pipeline)) {
      emit_vertex_input(pipeline, state, state->vi);

      emit_3dstate_vs(pipeline);
      emit_3dstate_hs_ds(pipeline, state->ts);
      emit_3dstate_te(pipeline);
      emit_3dstate_gs(pipeline);

      emit_3dstate_streamout(pipeline, state->rs);

#if GFX_VERx10 >= 125
      const struct anv_device *device = pipeline->base.base.device;
      /* Disable Mesh. */
      if (device->vk.enabled_extensions.EXT_mesh_shader) {
         anv_pipeline_emit(pipeline, final.mesh_control,
                           GENX(3DSTATE_MESH_CONTROL), zero);
         anv_pipeline_emit(pipeline, final.mesh_control_protected,
                           GENX(3DSTATE_MESH_CONTROL), zero);
         anv_pipeline_emit(pipeline, final.mesh_shader,
                           GENX(3DSTATE_MESH_SHADER), zero);
         anv_pipeline_emit(pipeline, final.mesh_distrib,
                           GENX(3DSTATE_MESH_DISTRIB), zero);
         anv_pipeline_emit(pipeline, final.clip_mesh,
                           GENX(3DSTATE_CLIP_MESH), zero);
         anv_pipeline_emit(pipeline, final.sbe_mesh,
                           GENX(3DSTATE_SBE_MESH), zero);
         anv_pipeline_emit(pipeline, final.task_control,
                           GENX(3DSTATE_TASK_CONTROL), zero);
         anv_pipeline_emit(pipeline, final.task_control_protected,
                           GENX(3DSTATE_TASK_CONTROL), zero);
         anv_pipeline_emit(pipeline, final.task_shader,
                           GENX(3DSTATE_TASK_SHADER), zero);
         anv_pipeline_emit(pipeline, final.task_redistrib,
                           GENX(3DSTATE_TASK_REDISTRIB), zero);
      }
#endif
   } else {
      assert(anv_pipeline_is_mesh(pipeline));

      anv_pipeline_emit(pipeline, final.vf_sgvs, GENX(3DSTATE_VF_SGVS), sgvs);
#if GFX_VER >= 11
      anv_pipeline_emit(pipeline, final.vf_sgvs_2, GENX(3DSTATE_VF_SGVS_2), sgvs);
#endif
      anv_pipeline_emit(pipeline, final.vs, GENX(3DSTATE_VS), vs);
      anv_pipeline_emit(pipeline, final.hs, GENX(3DSTATE_HS), hs);
      anv_pipeline_emit(pipeline, final.ds, GENX(3DSTATE_DS), ds);
      anv_pipeline_emit(pipeline, partial.te, GENX(3DSTATE_TE), te);
      anv_pipeline_emit(pipeline, partial.gs, GENX(3DSTATE_GS), gs);

      anv_pipeline_emit(pipeline, final.vs_protected, GENX(3DSTATE_VS), vs);
      anv_pipeline_emit(pipeline, final.hs_protected, GENX(3DSTATE_HS), hs);
      anv_pipeline_emit(pipeline, final.ds_protected, GENX(3DSTATE_DS), ds);
      anv_pipeline_emit(pipeline, partial.gs_protected, GENX(3DSTATE_GS), gs);

      /* BSpec 46303 forbids both 3DSTATE_MESH_CONTROL.MeshShaderEnable
       * and 3DSTATE_STREAMOUT.SOFunctionEnable to be 1.
       */
      anv_pipeline_emit(pipeline, partial.so, GENX(3DSTATE_STREAMOUT), so);

#if GFX_VERx10 >= 125
      emit_task_state(pipeline);
      emit_mesh_state(pipeline);
#endif
   }

   emit_3dstate_sbe(pipeline);
   emit_3dstate_wm(pipeline, state->ia, state->rs,
                   state->ms, state->cb, state->rp);
   emit_3dstate_ps(pipeline, state->ms, state->cb);
   emit_3dstate_ps_extra(pipeline, state->rs, state);
}

#if GFX_VERx10 >= 125

void
genX(compute_pipeline_emit)(struct anv_compute_pipeline *pipeline)
{
   const struct brw_cs_prog_data *cs_prog_data = get_cs_prog_data(pipeline);
   anv_pipeline_setup_l3_config(&pipeline->base, cs_prog_data->base.total_shared > 0);
}

#else /* #if GFX_VERx10 >= 125 */

void
genX(compute_pipeline_emit)(struct anv_compute_pipeline *pipeline)
{
   struct anv_device *device = pipeline->base.device;
   const struct intel_device_info *devinfo = device->info;
   const struct brw_cs_prog_data *cs_prog_data = get_cs_prog_data(pipeline);

   anv_pipeline_setup_l3_config(&pipeline->base, cs_prog_data->base.total_shared > 0);

   const struct intel_cs_dispatch_info dispatch =
      brw_cs_get_dispatch_info(devinfo, cs_prog_data, NULL);
   const uint32_t vfe_curbe_allocation =
      ALIGN(cs_prog_data->push.per_thread.regs * dispatch.threads +
            cs_prog_data->push.cross_thread.regs, 2);

   const struct anv_shader_bin *cs_bin = pipeline->cs;

   anv_batch_emit(&pipeline->base.batch, GENX(MEDIA_VFE_STATE), vfe) {
      vfe.StackSize              = 0;
      vfe.MaximumNumberofThreads =
         devinfo->max_cs_threads * devinfo->subslice_total - 1;
      vfe.NumberofURBEntries     = 2;
#if GFX_VER < 11
      vfe.ResetGatewayTimer      = true;
#endif
      vfe.URBEntryAllocationSize = 2;
      vfe.CURBEAllocationSize    = vfe_curbe_allocation;

      if (cs_prog_data->base.total_scratch) {
         /* Broadwell's Per Thread Scratch Space is in the range [0, 11]
          * where 0 = 1k, 1 = 2k, 2 = 4k, ..., 11 = 2M.
          */
         vfe.PerThreadScratchSpace = ffs(cs_prog_data->base.total_scratch) - 11;
         vfe.ScratchSpaceBasePointer =
            get_scratch_address(&pipeline->base, MESA_SHADER_COMPUTE, cs_bin);
      }
   }

   struct GENX(INTERFACE_DESCRIPTOR_DATA) desc = {
      .KernelStartPointer     =
         cs_bin->kernel.offset +
         brw_cs_prog_data_prog_offset(cs_prog_data, dispatch.simd_size),

      /* Wa_1606682166 */
      .SamplerCount           = GFX_VER == 11 ? 0 : get_sampler_count(cs_bin),
      /* We add 1 because the CS indirect parameters buffer isn't accounted
       * for in bind_map.surface_count.
       *
       * Typically set to 0 to avoid prefetching on every thread dispatch.
       */
      .BindingTableEntryCount = devinfo->verx10 == 125 ?
         0 : 1 + MIN2(pipeline->cs->bind_map.surface_count, 30),
      .BarrierEnable          = cs_prog_data->uses_barrier,
      .SharedLocalMemorySize  =
         intel_compute_slm_encode_size(GFX_VER, cs_prog_data->base.total_shared),

      .ConstantURBEntryReadOffset = 0,
      .ConstantURBEntryReadLength = cs_prog_data->push.per_thread.regs,
      .CrossThreadConstantDataReadLength =
         cs_prog_data->push.cross_thread.regs,
#if GFX_VER >= 12
      /* TODO: Check if we are missing workarounds and enable mid-thread
       * preemption.
       *
       * We still have issues with mid-thread preemption (it was already
       * disabled by the kernel on gfx11, due to missing workarounds). It's
       * possible that we are just missing some workarounds, and could enable
       * it later, but for now let's disable it to fix a GPU in compute in Car
       * Chase (and possibly more).
       */
      .ThreadPreemptionDisable = true,
#endif

      .NumberofThreadsinGPGPUThreadGroup = dispatch.threads,
   };
   GENX(INTERFACE_DESCRIPTOR_DATA_pack)(NULL,
                                        pipeline->interface_descriptor_data,
                                        &desc);
}

#endif /* #if GFX_VERx10 >= 125 */

#if GFX_VERx10 >= 125

void
genX(ray_tracing_pipeline_emit)(struct anv_ray_tracing_pipeline *pipeline)
{
   for (uint32_t i = 0; i < pipeline->group_count; i++) {
      struct anv_rt_shader_group *group = &pipeline->groups[i];

      switch (group->type) {
      case VK_RAY_TRACING_SHADER_GROUP_TYPE_GENERAL_KHR: {
         struct GENX(RT_GENERAL_SBT_HANDLE) sh = {};
         sh.General = anv_shader_bin_get_bsr(group->general, 32);
         GENX(RT_GENERAL_SBT_HANDLE_pack)(NULL, group->handle, &sh);
         break;
      }

      case VK_RAY_TRACING_SHADER_GROUP_TYPE_TRIANGLES_HIT_GROUP_KHR: {
         struct GENX(RT_TRIANGLES_SBT_HANDLE) sh = {};
         if (group->closest_hit)
            sh.ClosestHit = anv_shader_bin_get_bsr(group->closest_hit, 32);
         if (group->any_hit)
            sh.AnyHit = anv_shader_bin_get_bsr(group->any_hit, 24);
         GENX(RT_TRIANGLES_SBT_HANDLE_pack)(NULL, group->handle, &sh);
         break;
      }

      case VK_RAY_TRACING_SHADER_GROUP_TYPE_PROCEDURAL_HIT_GROUP_KHR: {
         struct GENX(RT_PROCEDURAL_SBT_HANDLE) sh = {};
         if (group->closest_hit)
            sh.ClosestHit = anv_shader_bin_get_bsr(group->closest_hit, 32);
         sh.Intersection = anv_shader_bin_get_bsr(group->intersection, 24);
         GENX(RT_PROCEDURAL_SBT_HANDLE_pack)(NULL, group->handle, &sh);
         break;
      }

      default:
         unreachable("Invalid shader group type");
      }
   }
}

#else

void
genX(ray_tracing_pipeline_emit)(struct anv_ray_tracing_pipeline *pipeline)
{
   unreachable("Ray tracing not supported");
}

#endif /* GFX_VERx10 >= 125 */