1 // Copyright © 2023 Collabora, Ltd.
2 // SPDX-License-Identifier: MIT
3
4 extern crate bitview;
5 extern crate nvidia_headers;
6
7 use crate::ir::{ShaderInfo, ShaderIoInfo, ShaderModel, ShaderStageInfo};
8 use bitview::{
9 BitMutView, BitMutViewable, BitView, BitViewable, SetBit, SetField,
10 SetFieldU64,
11 };
12 use nak_bindings::*;
13 use nvidia_headers::classes::cla097::sph::*;
14 use std::ops::Range;
15
16 pub const _SPHV3_SHADER_HEADER_SIZE: usize = 20;
17 pub const SPHV4_SHADER_HEADER_SIZE: usize = 32;
18 pub const CURRENT_MAX_SHADER_HEADER_SIZE: usize = SPHV4_SHADER_HEADER_SIZE;
19
20 type SubSPHView<'a> = BitMutView<'a, [u32; CURRENT_MAX_SHADER_HEADER_SIZE]>;
21
22 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
23 pub enum ShaderType {
24 Vertex,
25 TessellationInit,
26 Tessellation,
27 Geometry,
28 Fragment,
29 }
30
31 impl From<&ShaderStageInfo> for ShaderType {
from(value: &ShaderStageInfo) -> Self32 fn from(value: &ShaderStageInfo) -> Self {
33 match value {
34 ShaderStageInfo::Vertex => ShaderType::Vertex,
35 ShaderStageInfo::Fragment(_) => ShaderType::Fragment,
36 ShaderStageInfo::Geometry(_) => ShaderType::Geometry,
37 ShaderStageInfo::TessellationInit(_) => {
38 ShaderType::TessellationInit
39 }
40 ShaderStageInfo::Tessellation(_) => ShaderType::Tessellation,
41 _ => panic!("Invalid ShaderStageInfo {:?}", value),
42 }
43 }
44 }
45
46 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
47 pub enum OutputTopology {
48 PointList,
49 LineStrip,
50 TriangleStrip,
51 }
52
53 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
54 pub enum PixelImap {
55 Unused,
56 Constant,
57 Perspective,
58 ScreenLinear,
59 }
60
61 impl From<PixelImap> for u8 {
from(value: PixelImap) -> u862 fn from(value: PixelImap) -> u8 {
63 match value {
64 PixelImap::Unused => 0,
65 PixelImap::Constant => 1,
66 PixelImap::Perspective => 2,
67 PixelImap::ScreenLinear => 3,
68 }
69 }
70 }
71
72 #[derive(Debug)]
73 pub struct ShaderProgramHeader {
74 pub data: [u32; CURRENT_MAX_SHADER_HEADER_SIZE],
75 shader_type: ShaderType,
76 }
77
78 impl BitViewable for ShaderProgramHeader {
bits(&self) -> usize79 fn bits(&self) -> usize {
80 BitView::new(&self.data).bits()
81 }
82
get_bit_range_u64(&self, range: Range<usize>) -> u6483 fn get_bit_range_u64(&self, range: Range<usize>) -> u64 {
84 BitView::new(&self.data).get_bit_range_u64(range)
85 }
86 }
87
88 impl BitMutViewable for ShaderProgramHeader {
set_bit_range_u64(&mut self, range: Range<usize>, val: u64)89 fn set_bit_range_u64(&mut self, range: Range<usize>, val: u64) {
90 BitMutView::new(&mut self.data).set_bit_range_u64(range, val);
91 }
92 }
93
94 impl SetFieldU64 for ShaderProgramHeader {
set_field_u64(&mut self, range: Range<usize>, val: u64)95 fn set_field_u64(&mut self, range: Range<usize>, val: u64) {
96 BitMutView::new(&mut self.data).set_field_u64(range, val);
97 }
98 }
99
100 impl ShaderProgramHeader {
new(shader_type: ShaderType, sm: u8) -> Self101 pub fn new(shader_type: ShaderType, sm: u8) -> Self {
102 let mut res = Self {
103 data: [0; CURRENT_MAX_SHADER_HEADER_SIZE],
104 shader_type,
105 };
106
107 let sph_type = if shader_type == ShaderType::Fragment {
108 SPHV3_T1_SPH_TYPE_TYPE_02_PS
109 } else {
110 SPHV3_T1_SPH_TYPE_TYPE_01_VTG
111 };
112
113 let sph_version = if sm >= 75 { 4 } else { 3 };
114 res.set_sph_type(sph_type, sph_version);
115 res.set_shader_type(shader_type);
116
117 res
118 }
119
120 #[inline]
imap_system_values_ab(&mut self) -> SubSPHView<'_>121 fn imap_system_values_ab(&mut self) -> SubSPHView<'_> {
122 BitMutView::new_subset(&mut self.data, 160..192)
123 }
124
125 #[inline]
imap_g_vtg(&mut self) -> SubSPHView<'_>126 fn imap_g_vtg(&mut self) -> SubSPHView<'_> {
127 assert!(self.shader_type != ShaderType::Fragment);
128
129 BitMutView::new_subset(&mut self.data, 192..320)
130 }
131
132 #[inline]
imap_g_ps(&mut self) -> SubSPHView<'_>133 fn imap_g_ps(&mut self) -> SubSPHView<'_> {
134 assert!(self.shader_type == ShaderType::Fragment);
135
136 BitMutView::new_subset(&mut self.data, 192..448)
137 }
138
139 #[inline]
imap_system_values_c(&mut self) -> SubSPHView<'_>140 fn imap_system_values_c(&mut self) -> SubSPHView<'_> {
141 if self.shader_type == ShaderType::Fragment {
142 BitMutView::new_subset(&mut self.data, 464..480)
143 } else {
144 BitMutView::new_subset(&mut self.data, 336..352)
145 }
146 }
147
148 #[inline]
imap_system_values_d_vtg(&mut self) -> SubSPHView<'_>149 fn imap_system_values_d_vtg(&mut self) -> SubSPHView<'_> {
150 assert!(self.shader_type != ShaderType::Fragment);
151 BitMutView::new_subset(&mut self.data, 392..400)
152 }
153
154 #[inline]
omap_system_values_ab(&mut self) -> SubSPHView<'_>155 fn omap_system_values_ab(&mut self) -> SubSPHView<'_> {
156 assert!(self.shader_type != ShaderType::Fragment);
157 BitMutView::new_subset(&mut self.data, 400..432)
158 }
159
160 #[inline]
omap_g(&mut self) -> SubSPHView<'_>161 fn omap_g(&mut self) -> SubSPHView<'_> {
162 assert!(self.shader_type != ShaderType::Fragment);
163
164 BitMutView::new_subset(&mut self.data, 432..560)
165 }
166
167 #[inline]
omap_system_values_c(&mut self) -> SubSPHView<'_>168 fn omap_system_values_c(&mut self) -> SubSPHView<'_> {
169 assert!(self.shader_type != ShaderType::Fragment);
170 BitMutView::new_subset(&mut self.data, 576..592)
171 }
172
173 #[inline]
imap_system_values_d_ps(&mut self) -> SubSPHView<'_>174 fn imap_system_values_d_ps(&mut self) -> SubSPHView<'_> {
175 assert!(self.shader_type == ShaderType::Fragment);
176 BitMutView::new_subset(&mut self.data, 560..576)
177 }
178
179 #[inline]
omap_target(&mut self) -> SubSPHView<'_>180 fn omap_target(&mut self) -> SubSPHView<'_> {
181 assert!(self.shader_type == ShaderType::Fragment);
182
183 BitMutView::new_subset(&mut self.data, 576..608)
184 }
185
186 #[inline]
omap_system_values_d_vtg(&mut self) -> SubSPHView<'_>187 fn omap_system_values_d_vtg(&mut self) -> SubSPHView<'_> {
188 assert!(self.shader_type != ShaderType::Fragment);
189 BitMutView::new_subset(&mut self.data, 632..640)
190 }
191
192 #[inline]
set_sph_type(&mut self, sph_type: u32, sph_version: u8)193 fn set_sph_type(&mut self, sph_type: u32, sph_version: u8) {
194 self.set_field(SPHV3_T1_SPH_TYPE, sph_type);
195 self.set_field(SPHV3_T1_VERSION, sph_version);
196 }
197
198 #[inline]
set_shader_type(&mut self, shader_type: ShaderType)199 fn set_shader_type(&mut self, shader_type: ShaderType) {
200 self.set_field(
201 SPHV3_T1_SHADER_TYPE,
202 match shader_type {
203 ShaderType::Vertex => SPHV3_T1_SHADER_TYPE_VERTEX,
204 ShaderType::TessellationInit => {
205 SPHV3_T1_SHADER_TYPE_TESSELLATION_INIT
206 }
207 ShaderType::Tessellation => SPHV3_T1_SHADER_TYPE_TESSELLATION,
208 ShaderType::Geometry => SPHV3_T1_SHADER_TYPE_GEOMETRY,
209 ShaderType::Fragment => SPHV3_T1_SHADER_TYPE_PIXEL,
210 },
211 );
212 }
213
214 #[inline]
set_multiple_render_target_enable(&mut self, mrt_enable: bool)215 pub fn set_multiple_render_target_enable(&mut self, mrt_enable: bool) {
216 self.set_field(SPHV3_T1_MRT_ENABLE, mrt_enable);
217 }
218
219 #[inline]
set_kills_pixels(&mut self, kills_pixels: bool)220 pub fn set_kills_pixels(&mut self, kills_pixels: bool) {
221 self.set_field(SPHV3_T1_KILLS_PIXELS, kills_pixels);
222 }
223
224 #[inline]
set_does_global_store(&mut self, does_global_store: bool)225 pub fn set_does_global_store(&mut self, does_global_store: bool) {
226 self.set_field(SPHV3_T1_DOES_GLOBAL_STORE, does_global_store);
227 }
228
229 #[inline]
set_sass_version(&mut self, sass_version: u8)230 pub fn set_sass_version(&mut self, sass_version: u8) {
231 self.set_field(SPHV3_T1_SASS_VERSION, sass_version);
232 }
233
234 #[inline]
set_gs_passthrough_enable(&mut self, gs_passthrough_enable: bool)235 pub fn set_gs_passthrough_enable(&mut self, gs_passthrough_enable: bool) {
236 assert!(self.shader_type == ShaderType::Geometry);
237 self.set_bit(24, gs_passthrough_enable);
238 }
239
240 #[inline]
set_does_load_or_store(&mut self, does_load_or_store: bool)241 pub fn set_does_load_or_store(&mut self, does_load_or_store: bool) {
242 self.set_field(SPHV3_T1_DOES_LOAD_OR_STORE, does_load_or_store);
243 }
244
245 #[inline]
set_does_fp64(&mut self, does_fp64: bool)246 pub fn set_does_fp64(&mut self, does_fp64: bool) {
247 self.set_field(SPHV3_T1_DOES_FP64, does_fp64);
248 }
249
250 #[inline]
set_stream_out_mask(&mut self, stream_out_mask: u8)251 pub fn set_stream_out_mask(&mut self, stream_out_mask: u8) {
252 self.set_field(SPHV3_T1_STREAM_OUT_MASK, stream_out_mask);
253 }
254
255 #[inline]
set_shader_local_memory_size( &mut self, shader_local_memory_size: u64, )256 pub fn set_shader_local_memory_size(
257 &mut self,
258 shader_local_memory_size: u64,
259 ) {
260 assert!(shader_local_memory_size <= 0xffffffffffff);
261 assert!(shader_local_memory_size % 0x10 == 0);
262
263 let low = (shader_local_memory_size & 0xffffff) as u32;
264 let high = ((shader_local_memory_size >> 32) & 0xffffff) as u32;
265
266 self.set_field(SPHV3_T1_SHADER_LOCAL_MEMORY_LOW_SIZE, low);
267 self.set_field(SPHV3_T1_SHADER_LOCAL_MEMORY_HIGH_SIZE, high);
268 }
269
270 #[inline]
set_per_patch_attribute_count( &mut self, per_patch_attribute_count: u8, )271 pub fn set_per_patch_attribute_count(
272 &mut self,
273 per_patch_attribute_count: u8,
274 ) {
275 assert!(self.shader_type == ShaderType::TessellationInit);
276
277 self.set_field(
278 SPHV3_T1_PER_PATCH_ATTRIBUTE_COUNT,
279 per_patch_attribute_count,
280 );
281
282 // This is Kepler+
283 self.set_field(
284 SPHV3_T1_RESERVED_COMMON_B,
285 per_patch_attribute_count & 0xf,
286 );
287 self.set_field(148..152, per_patch_attribute_count >> 4);
288 }
289
290 #[inline]
set_threads_per_input_primitive( &mut self, threads_per_input_primitive: u8, )291 pub fn set_threads_per_input_primitive(
292 &mut self,
293 threads_per_input_primitive: u8,
294 ) {
295 self.set_field(
296 SPHV3_T1_THREADS_PER_INPUT_PRIMITIVE,
297 threads_per_input_primitive,
298 );
299 }
300
301 #[inline]
302 #[allow(dead_code)]
set_shader_local_memory_crs_size( &mut self, shader_local_memory_crs_size: u32, )303 pub fn set_shader_local_memory_crs_size(
304 &mut self,
305 shader_local_memory_crs_size: u32,
306 ) {
307 assert!(shader_local_memory_crs_size <= 0xffffff);
308 self.set_field(
309 SPHV3_T1_SHADER_LOCAL_MEMORY_CRS_SIZE,
310 shader_local_memory_crs_size,
311 );
312 }
313
314 #[inline]
set_output_topology(&mut self, output_topology: OutputTopology)315 pub fn set_output_topology(&mut self, output_topology: OutputTopology) {
316 self.set_field(
317 SPHV3_T1_OUTPUT_TOPOLOGY,
318 match output_topology {
319 OutputTopology::PointList => SPHV3_T1_OUTPUT_TOPOLOGY_POINTLIST,
320 OutputTopology::LineStrip => SPHV3_T1_OUTPUT_TOPOLOGY_LINESTRIP,
321 OutputTopology::TriangleStrip => {
322 SPHV3_T1_OUTPUT_TOPOLOGY_TRIANGLESTRIP
323 }
324 },
325 );
326 }
327
328 #[inline]
set_max_output_vertex_count( &mut self, max_output_vertex_count: u16, )329 pub fn set_max_output_vertex_count(
330 &mut self,
331 max_output_vertex_count: u16,
332 ) {
333 assert!(max_output_vertex_count <= 0xfff);
334 self.set_field(
335 SPHV3_T1_MAX_OUTPUT_VERTEX_COUNT,
336 max_output_vertex_count,
337 );
338 }
339
340 #[inline]
set_store_req_start(&mut self, store_req_start: u8)341 pub fn set_store_req_start(&mut self, store_req_start: u8) {
342 self.set_field(SPHV3_T1_STORE_REQ_START, store_req_start);
343 }
344
345 #[inline]
set_store_req_end(&mut self, store_req_end: u8)346 pub fn set_store_req_end(&mut self, store_req_end: u8) {
347 self.set_field(SPHV3_T1_STORE_REQ_END, store_req_end);
348 }
349
set_imap_system_values_ab(&mut self, val: u32)350 pub fn set_imap_system_values_ab(&mut self, val: u32) {
351 self.imap_system_values_ab().set_field(0..32, val);
352 }
353
set_imap_system_values_c(&mut self, val: u16)354 pub fn set_imap_system_values_c(&mut self, val: u16) {
355 self.imap_system_values_c().set_field(0..16, val);
356 }
357
set_imap_system_values_d_vtg(&mut self, val: u8)358 pub fn set_imap_system_values_d_vtg(&mut self, val: u8) {
359 assert!(self.shader_type != ShaderType::Fragment);
360 self.imap_system_values_d_vtg().set_field(0..8, val);
361 }
362
363 #[inline]
set_imap_vector_ps(&mut self, index: usize, value: PixelImap)364 pub fn set_imap_vector_ps(&mut self, index: usize, value: PixelImap) {
365 assert!(index < 128);
366 assert!(self.shader_type == ShaderType::Fragment);
367
368 self.imap_g_ps()
369 .set_field(index * 2..(index + 1) * 2, u8::from(value));
370 }
371
372 #[inline]
set_imap_system_values_d_ps( &mut self, index: usize, value: PixelImap, )373 pub fn set_imap_system_values_d_ps(
374 &mut self,
375 index: usize,
376 value: PixelImap,
377 ) {
378 assert!(index < 8);
379 assert!(self.shader_type == ShaderType::Fragment);
380
381 self.imap_system_values_d_ps()
382 .set_field(index * 2..(index + 1) * 2, u8::from(value));
383 }
384
385 #[inline]
set_imap_vector_vtg(&mut self, index: usize, value: u32)386 pub fn set_imap_vector_vtg(&mut self, index: usize, value: u32) {
387 assert!(index < 4);
388 assert!(self.shader_type != ShaderType::Fragment);
389
390 self.imap_g_vtg()
391 .set_field(index * 32..(index + 1) * 32, value);
392 }
393
394 #[inline]
set_omap_system_values_ab(&mut self, val: u32)395 pub fn set_omap_system_values_ab(&mut self, val: u32) {
396 self.omap_system_values_ab().set_field(0..32, val);
397 }
398
399 #[inline]
set_omap_system_values_c(&mut self, val: u16)400 pub fn set_omap_system_values_c(&mut self, val: u16) {
401 self.omap_system_values_c().set_field(0..16, val);
402 }
403
set_omap_system_values_d_vtg(&mut self, val: u8)404 pub fn set_omap_system_values_d_vtg(&mut self, val: u8) {
405 assert!(self.shader_type != ShaderType::Fragment);
406 self.omap_system_values_d_vtg().set_field(0..8, val);
407 }
408
409 #[inline]
set_omap_vector(&mut self, index: usize, value: u32)410 pub fn set_omap_vector(&mut self, index: usize, value: u32) {
411 assert!(index < 4);
412 assert!(self.shader_type != ShaderType::Fragment);
413
414 self.omap_g().set_field(index * 32..(index + 1) * 32, value);
415 }
416
417 #[inline]
set_omap_targets(&mut self, value: u32)418 pub fn set_omap_targets(&mut self, value: u32) {
419 self.omap_target().set_field(0..32, value)
420 }
421
422 #[inline]
set_omap_sample_mask(&mut self, sample_mask: bool)423 pub fn set_omap_sample_mask(&mut self, sample_mask: bool) {
424 assert!(self.shader_type == ShaderType::Fragment);
425 self.set_field(SPHV3_T2_OMAP_SAMPLE_MASK, sample_mask);
426 }
427
428 #[inline]
set_omap_depth(&mut self, depth: bool)429 pub fn set_omap_depth(&mut self, depth: bool) {
430 assert!(self.shader_type == ShaderType::Fragment);
431 self.set_field(SPHV3_T2_OMAP_DEPTH, depth);
432 }
433
434 #[inline]
set_does_interlock(&mut self, does_interlock: bool)435 pub fn set_does_interlock(&mut self, does_interlock: bool) {
436 assert!(self.shader_type == ShaderType::Fragment);
437 self.set_bit(610, does_interlock);
438 }
439
440 #[inline]
441 #[allow(dead_code)]
set_uses_underestimate(&mut self, uses_underestimate: bool)442 pub fn set_uses_underestimate(&mut self, uses_underestimate: bool) {
443 assert!(self.shader_type == ShaderType::Fragment);
444 self.set_bit(611, uses_underestimate);
445 }
446
447 #[inline]
pervertex_imap_vector_ps(&mut self) -> SubSPHView<'_>448 fn pervertex_imap_vector_ps(&mut self) -> SubSPHView<'_> {
449 assert!(self.shader_type == ShaderType::Fragment);
450
451 BitMutView::new_subset(&mut self.data, 672..800)
452 }
453
454 #[inline]
set_pervertex_imap_vector(&mut self, index: usize, value: u32)455 pub fn set_pervertex_imap_vector(&mut self, index: usize, value: u32) {
456 assert!(index < 4);
457 assert!(self.shader_type == ShaderType::Fragment);
458
459 self.pervertex_imap_vector_ps()
460 .set_field(index * 32..(index + 1) * 32, value);
461 }
462 }
463
encode_header( sm: &dyn ShaderModel, shader_info: &ShaderInfo, fs_key: Option<&nak_fs_key>, ) -> [u32; CURRENT_MAX_SHADER_HEADER_SIZE]464 pub fn encode_header(
465 sm: &dyn ShaderModel,
466 shader_info: &ShaderInfo,
467 fs_key: Option<&nak_fs_key>,
468 ) -> [u32; CURRENT_MAX_SHADER_HEADER_SIZE] {
469 if let ShaderStageInfo::Compute(_) = shader_info.stage {
470 return [0_u32; CURRENT_MAX_SHADER_HEADER_SIZE];
471 }
472
473 let mut sph =
474 ShaderProgramHeader::new(ShaderType::from(&shader_info.stage), sm.sm());
475
476 sph.set_sass_version(1);
477 sph.set_does_load_or_store(shader_info.uses_global_mem);
478 sph.set_does_global_store(shader_info.writes_global_mem);
479 sph.set_does_fp64(shader_info.uses_fp64);
480
481 let slm_size = shader_info.slm_size.next_multiple_of(16);
482 sph.set_shader_local_memory_size(slm_size.into());
483 let crs_size = sm.crs_size(shader_info.max_crs_depth);
484 sph.set_shader_local_memory_crs_size(crs_size);
485
486 match &shader_info.io {
487 ShaderIoInfo::Vtg(io) => {
488 sph.set_imap_system_values_ab(io.sysvals_in.ab);
489 sph.set_imap_system_values_c(io.sysvals_in.c);
490 sph.set_imap_system_values_d_vtg(io.sysvals_in_d);
491
492 for (index, value) in io.attr_in.iter().enumerate() {
493 sph.set_imap_vector_vtg(index, *value);
494 }
495
496 for (index, value) in io.attr_out.iter().enumerate() {
497 sph.set_omap_vector(index, *value);
498 }
499
500 sph.set_store_req_start(io.store_req_start);
501 sph.set_store_req_end(io.store_req_end);
502
503 sph.set_omap_system_values_ab(io.sysvals_out.ab);
504 sph.set_omap_system_values_c(io.sysvals_out.c);
505 sph.set_omap_system_values_d_vtg(io.sysvals_out_d);
506 }
507 ShaderIoInfo::Fragment(io) => {
508 sph.set_imap_system_values_ab(io.sysvals_in.ab);
509 sph.set_imap_system_values_c(io.sysvals_in.c);
510
511 for (index, imap) in io.sysvals_in_d.iter().enumerate() {
512 sph.set_imap_system_values_d_ps(index, *imap);
513 }
514
515 for (index, imap) in io.attr_in.iter().enumerate() {
516 sph.set_imap_vector_ps(index, *imap);
517 }
518
519 let uses_underestimate =
520 fs_key.map_or(false, |key| key.uses_underestimate);
521
522 // This isn't so much a "Do we write multiple render targets?" bit
523 // as a "Should color0 be broadcast to all render targets?" bit. In
524 // other words, it's the gl_FragCoord behavior, not gl_FragData.
525 //
526 // For now, we always set it to true because Vulkan requires
527 // explicit fragment output locations.
528 sph.set_multiple_render_target_enable(true);
529
530 sph.set_omap_sample_mask(io.writes_sample_mask);
531 sph.set_omap_depth(io.writes_depth);
532 sph.set_omap_targets(io.writes_color);
533 sph.set_uses_underestimate(uses_underestimate);
534
535 for (index, value) in io.barycentric_attr_in.iter().enumerate() {
536 sph.set_pervertex_imap_vector(index, *value);
537 }
538 }
539 _ => {}
540 }
541
542 match &shader_info.stage {
543 ShaderStageInfo::Fragment(stage) => {
544 let zs_self_dep = fs_key.map_or(false, |key| key.zs_self_dep);
545 sph.set_kills_pixels(stage.uses_kill || zs_self_dep);
546 sph.set_does_interlock(stage.does_interlock);
547 }
548 ShaderStageInfo::Geometry(stage) => {
549 sph.set_gs_passthrough_enable(stage.passthrough_enable);
550 sph.set_stream_out_mask(stage.stream_out_mask);
551 sph.set_threads_per_input_primitive(
552 stage.threads_per_input_primitive,
553 );
554 sph.set_output_topology(stage.output_topology);
555 sph.set_max_output_vertex_count(stage.max_output_vertex_count);
556 }
557 ShaderStageInfo::TessellationInit(stage) => {
558 sph.set_per_patch_attribute_count(stage.per_patch_attribute_count);
559 sph.set_threads_per_input_primitive(stage.threads_per_patch);
560 }
561 ShaderStageInfo::Compute(_) => {
562 panic!("Compute shaders don't have a SPH!")
563 }
564 _ => {}
565 };
566
567 sph.data
568 }
569