1 // Copyright (c) 2022 The Vulkano developers
2 // Licensed under the Apache License, Version 2.0
3 // <LICENSE-APACHE or
4 // https://www.apache.org/licenses/LICENSE-2.0> or the MIT
5 // license <LICENSE-MIT or https://opensource.org/licenses/MIT>,
6 // at your option. All files in the project carrying such
7 // notice may not be copied, modified, or distributed except
8 // according to those terms.
9
10 pub use self::{
11 bind_push::*, clear::*, copy::*, debug::*, dynamic_state::*, pipeline::*, query::*,
12 render_pass::*, secondary::*, sync::*,
13 };
14 use super::{PrimaryCommandBuffer, SecondaryCommandBuffer, SubmitState};
15 pub use crate::command_buffer::{
16 BlitImageInfo, BufferCopy, BufferImageCopy, ClearAttachment, ClearColorImageInfo,
17 ClearDepthStencilImageInfo, ClearError, ClearRect, CopyBufferInfo, CopyBufferInfoTyped,
18 CopyBufferToImageInfo, CopyError, CopyErrorResource, CopyImageInfo, CopyImageToBufferInfo,
19 DebugUtilsError, ExecuteCommandsError, ImageBlit, ImageCopy, ImageResolve,
20 PipelineExecutionError, QueryError, RenderPassBeginInfo, RenderPassError,
21 RenderingAttachmentInfo, RenderingAttachmentResolveInfo, RenderingInfo, ResolveImageInfo,
22 };
23 use crate::{
24 buffer::{Buffer, Subbuffer},
25 command_buffer::{
26 allocator::{
27 CommandBufferAllocator, CommandBufferBuilderAlloc, StandardCommandBufferAllocator,
28 },
29 sys::CommandBufferBeginInfo,
30 BuildError, CommandBufferBeginError, CommandBufferInheritanceInfo,
31 CommandBufferInheritanceRenderPassInfo, CommandBufferInheritanceRenderPassType,
32 CommandBufferInheritanceRenderingInfo, CommandBufferLevel, CommandBufferUsage,
33 ResourceInCommand, ResourceUseRef, SubpassContents,
34 },
35 descriptor_set::{DescriptorSetResources, DescriptorSetWithOffsets},
36 device::{Device, DeviceOwned, QueueFamilyProperties, QueueFlags},
37 format::FormatFeatures,
38 image::{sys::Image, ImageAspects, ImageLayout, ImageSubresourceRange, ImageViewAbstract},
39 pipeline::{
40 graphics::{
41 color_blend::LogicOp,
42 depth_stencil::{CompareOp, StencilOps},
43 input_assembly::{IndexType, PrimitiveTopology},
44 rasterization::{CullMode, DepthBias, FrontFace, LineStipple},
45 render_pass::PipelineRenderingCreateInfo,
46 viewport::{Scissor, Viewport},
47 },
48 ComputePipeline, DynamicState, GraphicsPipeline, PipelineBindPoint, PipelineLayout,
49 },
50 query::{QueryControlFlags, QueryType},
51 range_map::RangeMap,
52 range_set::RangeSet,
53 render_pass::{Framebuffer, LoadOp, StoreOp, Subpass},
54 sync::{
55 BufferMemoryBarrier, DependencyInfo, ImageMemoryBarrier, PipelineStage,
56 PipelineStageAccess, PipelineStageAccessSet, PipelineStages,
57 },
58 DeviceSize, OomError, RequiresOneOf, VulkanError, VulkanObject,
59 };
60 use ahash::HashMap;
61 use parking_lot::Mutex;
62 use smallvec::SmallVec;
63 use std::{
64 any::Any,
65 collections::hash_map::Entry,
66 marker::PhantomData,
67 ops::{Range, RangeInclusive},
68 ptr,
69 sync::{atomic::AtomicBool, Arc},
70 };
71
72 mod bind_push;
73 mod clear;
74 mod copy;
75 mod debug;
76 mod dynamic_state;
77 mod pipeline;
78 mod query;
79 mod render_pass;
80 mod secondary;
81 mod sync;
82
83 /// Records commands to a command buffer.
84 pub struct CommandBufferBuilder<L, A = StandardCommandBufferAllocator>
85 where
86 A: CommandBufferAllocator,
87 {
88 builder_alloc: A::Builder,
89 inheritance_info: Option<CommandBufferInheritanceInfo>, // Must be `None` in a primary command buffer and `Some` in a secondary command buffer.
90 queue_family_index: u32,
91 usage: CommandBufferUsage,
92
93 next_command_index: usize,
94 resources: Vec<Box<dyn Any + Send + Sync>>,
95 builder_state: CommandBufferBuilderState,
96 resources_usage_state: ResourcesState,
97
98 _data: PhantomData<L>,
99 }
100
101 unsafe impl<L, A> DeviceOwned for CommandBufferBuilder<L, A>
102 where
103 A: CommandBufferAllocator,
104 {
105 #[inline]
device(&self) -> &Arc<Device>106 fn device(&self) -> &Arc<Device> {
107 self.builder_alloc.device()
108 }
109 }
110
111 unsafe impl<L, A> VulkanObject for CommandBufferBuilder<L, A>
112 where
113 A: CommandBufferAllocator,
114 {
115 type Handle = ash::vk::CommandBuffer;
116
117 #[inline]
handle(&self) -> Self::Handle118 fn handle(&self) -> Self::Handle {
119 self.builder_alloc.inner().handle()
120 }
121 }
122
123 impl<A> CommandBufferBuilder<PrimaryCommandBuffer, A>
124 where
125 A: CommandBufferAllocator,
126 {
127 /// Starts recording a primary command buffer.
128 #[inline]
primary( allocator: &A, queue_family_index: u32, usage: CommandBufferUsage, ) -> Result<Self, CommandBufferBeginError>129 pub fn primary(
130 allocator: &A,
131 queue_family_index: u32,
132 usage: CommandBufferUsage,
133 ) -> Result<Self, CommandBufferBeginError> {
134 unsafe {
135 CommandBufferBuilder::begin(
136 allocator,
137 queue_family_index,
138 CommandBufferLevel::Primary,
139 CommandBufferBeginInfo {
140 usage,
141 inheritance_info: None,
142 _ne: crate::NonExhaustive(()),
143 },
144 )
145 }
146 }
147 }
148
149 impl<A> CommandBufferBuilder<SecondaryCommandBuffer, A>
150 where
151 A: CommandBufferAllocator,
152 {
153 /// Starts recording a secondary command buffer.
154 #[inline]
secondary( allocator: &A, queue_family_index: u32, usage: CommandBufferUsage, inheritance_info: CommandBufferInheritanceInfo, ) -> Result<Self, CommandBufferBeginError>155 pub fn secondary(
156 allocator: &A,
157 queue_family_index: u32,
158 usage: CommandBufferUsage,
159 inheritance_info: CommandBufferInheritanceInfo,
160 ) -> Result<Self, CommandBufferBeginError> {
161 unsafe {
162 CommandBufferBuilder::begin(
163 allocator,
164 queue_family_index,
165 CommandBufferLevel::Secondary,
166 CommandBufferBeginInfo {
167 usage,
168 inheritance_info: Some(inheritance_info),
169 _ne: crate::NonExhaustive(()),
170 },
171 )
172 }
173 }
174 }
175
176 impl<L, A> CommandBufferBuilder<L, A>
177 where
178 A: CommandBufferAllocator,
179 {
180 // Actual constructor. Private.
181 //
182 // `begin_info.inheritance_info` must match `level`.
begin( allocator: &A, queue_family_index: u32, level: CommandBufferLevel, begin_info: CommandBufferBeginInfo, ) -> Result<Self, CommandBufferBeginError>183 unsafe fn begin(
184 allocator: &A,
185 queue_family_index: u32,
186 level: CommandBufferLevel,
187 begin_info: CommandBufferBeginInfo,
188 ) -> Result<Self, CommandBufferBeginError> {
189 Self::validate_begin(allocator.device(), queue_family_index, level, &begin_info)?;
190 Ok(Self::begin_unchecked(
191 allocator,
192 queue_family_index,
193 level,
194 begin_info,
195 )?)
196 }
197
validate_begin( device: &Device, _queue_family_index: u32, level: CommandBufferLevel, begin_info: &CommandBufferBeginInfo, ) -> Result<(), CommandBufferBeginError>198 fn validate_begin(
199 device: &Device,
200 _queue_family_index: u32,
201 level: CommandBufferLevel,
202 begin_info: &CommandBufferBeginInfo,
203 ) -> Result<(), CommandBufferBeginError> {
204 let physical_device = device.physical_device();
205 let properties = physical_device.properties();
206
207 let &CommandBufferBeginInfo {
208 usage: _,
209 ref inheritance_info,
210 _ne: _,
211 } = &begin_info;
212
213 if let Some(inheritance_info) = &inheritance_info {
214 debug_assert!(level == CommandBufferLevel::Secondary);
215
216 let &CommandBufferInheritanceInfo {
217 ref render_pass,
218 occlusion_query,
219 query_statistics_flags,
220 _ne: _,
221 } = inheritance_info;
222
223 if let Some(render_pass) = render_pass {
224 // VUID-VkCommandBufferBeginInfo-flags-06000
225 // VUID-VkCommandBufferBeginInfo-flags-06002
226 // Ensured by the definition of the `CommandBufferInheritanceRenderPassType` enum.
227
228 match render_pass {
229 CommandBufferInheritanceRenderPassType::BeginRenderPass(render_pass_info) => {
230 let &CommandBufferInheritanceRenderPassInfo {
231 ref subpass,
232 ref framebuffer,
233 } = render_pass_info;
234
235 // VUID-VkCommandBufferInheritanceInfo-commonparent
236 assert_eq!(device, subpass.render_pass().device().as_ref());
237
238 // VUID-VkCommandBufferBeginInfo-flags-06001
239 // Ensured by how the `Subpass` type is constructed.
240
241 if let Some(framebuffer) = framebuffer {
242 // VUID-VkCommandBufferInheritanceInfo-commonparent
243 assert_eq!(device, framebuffer.device().as_ref());
244
245 // VUID-VkCommandBufferBeginInfo-flags-00055
246 if !framebuffer
247 .render_pass()
248 .is_compatible_with(subpass.render_pass())
249 {
250 return Err(CommandBufferBeginError::FramebufferNotCompatible);
251 }
252 }
253 }
254 CommandBufferInheritanceRenderPassType::BeginRendering(rendering_info) => {
255 let &CommandBufferInheritanceRenderingInfo {
256 view_mask,
257 ref color_attachment_formats,
258 depth_attachment_format,
259 stencil_attachment_format,
260 rasterization_samples,
261 } = rendering_info;
262
263 // VUID-VkCommandBufferInheritanceRenderingInfo-multiview-06008
264 if view_mask != 0 && !device.enabled_features().multiview {
265 return Err(CommandBufferBeginError::RequirementNotMet {
266 required_for: "`inheritance_info.render_pass` is \
267 `CommandBufferInheritanceRenderPassType::BeginRendering`, \
268 where `view_mask` is not `0`",
269 requires_one_of: RequiresOneOf {
270 features: &["multiview"],
271 ..Default::default()
272 },
273 });
274 }
275
276 let view_count = u32::BITS - view_mask.leading_zeros();
277
278 // VUID-VkCommandBufferInheritanceRenderingInfo-viewMask-06009
279 if view_count > properties.max_multiview_view_count.unwrap_or(0) {
280 return Err(CommandBufferBeginError::MaxMultiviewViewCountExceeded {
281 view_count,
282 max: properties.max_multiview_view_count.unwrap_or(0),
283 });
284 }
285
286 for (attachment_index, format) in color_attachment_formats
287 .iter()
288 .enumerate()
289 .flat_map(|(i, f)| f.map(|f| (i, f)))
290 {
291 let attachment_index = attachment_index as u32;
292
293 // VUID-VkCommandBufferInheritanceRenderingInfo-pColorAttachmentFormats-parameter
294 format.validate_device(device)?;
295
296 // VUID-VkCommandBufferInheritanceRenderingInfo-pColorAttachmentFormats-06006
297 // Use unchecked, because all validation has been done above.
298 if !unsafe { physical_device.format_properties_unchecked(format) }
299 .potential_format_features()
300 .intersects(FormatFeatures::COLOR_ATTACHMENT)
301 {
302 return Err(
303 CommandBufferBeginError::ColorAttachmentFormatUsageNotSupported {
304 attachment_index,
305 },
306 );
307 }
308 }
309
310 if let Some(format) = depth_attachment_format {
311 // VUID-VkCommandBufferInheritanceRenderingInfo-depthAttachmentFormat-parameter
312 format.validate_device(device)?;
313
314 // VUID-VkCommandBufferInheritanceRenderingInfo-depthAttachmentFormat-06540
315 if !format.aspects().intersects(ImageAspects::DEPTH) {
316 return Err(
317 CommandBufferBeginError::DepthAttachmentFormatUsageNotSupported,
318 );
319 }
320
321 // VUID-VkCommandBufferInheritanceRenderingInfo-depthAttachmentFormat-06007
322 // Use unchecked, because all validation has been done above.
323 if !unsafe { physical_device.format_properties_unchecked(format) }
324 .potential_format_features()
325 .intersects(FormatFeatures::DEPTH_STENCIL_ATTACHMENT)
326 {
327 return Err(
328 CommandBufferBeginError::DepthAttachmentFormatUsageNotSupported,
329 );
330 }
331 }
332
333 if let Some(format) = stencil_attachment_format {
334 // VUID-VkCommandBufferInheritanceRenderingInfo-stencilAttachmentFormat-parameter
335 format.validate_device(device)?;
336
337 // VUID-VkCommandBufferInheritanceRenderingInfo-stencilAttachmentFormat-06541
338 if !format.aspects().intersects(ImageAspects::STENCIL) {
339 return Err(
340 CommandBufferBeginError::StencilAttachmentFormatUsageNotSupported,
341 );
342 }
343
344 // VUID-VkCommandBufferInheritanceRenderingInfo-stencilAttachmentFormat-06199
345 // Use unchecked, because all validation has been done above.
346 if !unsafe { physical_device.format_properties_unchecked(format) }
347 .potential_format_features()
348 .intersects(FormatFeatures::DEPTH_STENCIL_ATTACHMENT)
349 {
350 return Err(
351 CommandBufferBeginError::StencilAttachmentFormatUsageNotSupported,
352 );
353 }
354 }
355
356 if let (Some(depth_format), Some(stencil_format)) =
357 (depth_attachment_format, stencil_attachment_format)
358 {
359 // VUID-VkCommandBufferInheritanceRenderingInfo-depthAttachmentFormat-06200
360 if depth_format != stencil_format {
361 return Err(
362 CommandBufferBeginError::DepthStencilAttachmentFormatMismatch,
363 );
364 }
365 }
366
367 // VUID-VkCommandBufferInheritanceRenderingInfo-rasterizationSamples-parameter
368 rasterization_samples.validate_device(device)?;
369 }
370 }
371 }
372
373 if let Some(control_flags) = occlusion_query {
374 // VUID-VkCommandBufferInheritanceInfo-queryFlags-00057
375 control_flags.validate_device(device)?;
376
377 // VUID-VkCommandBufferInheritanceInfo-occlusionQueryEnable-00056
378 // VUID-VkCommandBufferInheritanceInfo-queryFlags-02788
379 if !device.enabled_features().inherited_queries {
380 return Err(CommandBufferBeginError::RequirementNotMet {
381 required_for: "`inheritance_info.occlusion_query` is `Some`",
382 requires_one_of: RequiresOneOf {
383 features: &["inherited_queries"],
384 ..Default::default()
385 },
386 });
387 }
388
389 // VUID-vkBeginCommandBuffer-commandBuffer-00052
390 if control_flags.intersects(QueryControlFlags::PRECISE)
391 && !device.enabled_features().occlusion_query_precise
392 {
393 return Err(CommandBufferBeginError::RequirementNotMet {
394 required_for: "`inheritance_info.occlusion_query` is \
395 `Some(control_flags)`, where `control_flags` contains \
396 `QueryControlFlags::PRECISE`",
397 requires_one_of: RequiresOneOf {
398 features: &["occlusion_query_precise"],
399 ..Default::default()
400 },
401 });
402 }
403 }
404
405 // VUID-VkCommandBufferInheritanceInfo-pipelineStatistics-02789
406 query_statistics_flags.validate_device(device)?;
407
408 // VUID-VkCommandBufferInheritanceInfo-pipelineStatistics-00058
409 if query_statistics_flags.count() > 0
410 && !device.enabled_features().pipeline_statistics_query
411 {
412 return Err(CommandBufferBeginError::RequirementNotMet {
413 required_for: "`inheritance_info.query_statistics_flags` is not empty",
414 requires_one_of: RequiresOneOf {
415 features: &["pipeline_statistics_query"],
416 ..Default::default()
417 },
418 });
419 }
420 } else {
421 debug_assert!(level == CommandBufferLevel::Primary);
422
423 // VUID-vkBeginCommandBuffer-commandBuffer-02840
424 // Ensured by the definition of the `CommandBufferUsage` enum.
425 }
426
427 Ok(())
428 }
429
begin_unchecked( allocator: &A, queue_family_index: u32, level: CommandBufferLevel, begin_info: CommandBufferBeginInfo, ) -> Result<Self, OomError>430 unsafe fn begin_unchecked(
431 allocator: &A,
432 queue_family_index: u32,
433 level: CommandBufferLevel,
434 begin_info: CommandBufferBeginInfo,
435 ) -> Result<Self, OomError> {
436 let CommandBufferBeginInfo {
437 usage,
438 inheritance_info,
439 _ne: _,
440 } = begin_info;
441
442 let builder_alloc = allocator
443 .allocate(queue_family_index, level, 1)?
444 .next()
445 .expect("requested one command buffer from the command pool, but got zero");
446
447 {
448 let device = builder_alloc.device();
449
450 let mut flags = ash::vk::CommandBufferUsageFlags::from(usage);
451 let mut inheritance_info_vk = None;
452 let mut inheritance_rendering_info_vk = None;
453 let mut color_attachment_formats_vk: SmallVec<[_; 4]> = SmallVec::new();
454
455 if let Some(inheritance_info) = &inheritance_info {
456 let &CommandBufferInheritanceInfo {
457 ref render_pass,
458 occlusion_query,
459 query_statistics_flags,
460 _ne: _,
461 } = inheritance_info;
462
463 let inheritance_info_vk =
464 inheritance_info_vk.insert(ash::vk::CommandBufferInheritanceInfo {
465 render_pass: ash::vk::RenderPass::null(),
466 subpass: 0,
467 framebuffer: ash::vk::Framebuffer::null(),
468 occlusion_query_enable: ash::vk::FALSE,
469 query_flags: ash::vk::QueryControlFlags::empty(),
470 pipeline_statistics: query_statistics_flags.into(),
471 ..Default::default()
472 });
473
474 if let Some(flags) = occlusion_query {
475 inheritance_info_vk.occlusion_query_enable = ash::vk::TRUE;
476
477 if flags.intersects(QueryControlFlags::PRECISE) {
478 inheritance_info_vk.query_flags = ash::vk::QueryControlFlags::PRECISE;
479 }
480 }
481
482 if let Some(render_pass) = render_pass {
483 flags |= ash::vk::CommandBufferUsageFlags::RENDER_PASS_CONTINUE;
484
485 match render_pass {
486 CommandBufferInheritanceRenderPassType::BeginRenderPass(
487 render_pass_info,
488 ) => {
489 let &CommandBufferInheritanceRenderPassInfo {
490 ref subpass,
491 ref framebuffer,
492 } = render_pass_info;
493
494 inheritance_info_vk.render_pass = subpass.render_pass().handle();
495 inheritance_info_vk.subpass = subpass.index();
496 inheritance_info_vk.framebuffer = framebuffer
497 .as_ref()
498 .map(|fb| fb.handle())
499 .unwrap_or_default();
500 }
501 CommandBufferInheritanceRenderPassType::BeginRendering(rendering_info) => {
502 let &CommandBufferInheritanceRenderingInfo {
503 view_mask,
504 ref color_attachment_formats,
505 depth_attachment_format,
506 stencil_attachment_format,
507 rasterization_samples,
508 } = rendering_info;
509
510 color_attachment_formats_vk.extend(
511 color_attachment_formats.iter().map(|format| {
512 format.map_or(ash::vk::Format::UNDEFINED, Into::into)
513 }),
514 );
515
516 let inheritance_rendering_info_vk = inheritance_rendering_info_vk
517 .insert(ash::vk::CommandBufferInheritanceRenderingInfo {
518 flags: ash::vk::RenderingFlags::empty(),
519 view_mask,
520 color_attachment_count: color_attachment_formats_vk.len()
521 as u32,
522 p_color_attachment_formats: color_attachment_formats_vk
523 .as_ptr(),
524 depth_attachment_format: depth_attachment_format
525 .map_or(ash::vk::Format::UNDEFINED, Into::into),
526 stencil_attachment_format: stencil_attachment_format
527 .map_or(ash::vk::Format::UNDEFINED, Into::into),
528 rasterization_samples: rasterization_samples.into(),
529 ..Default::default()
530 });
531
532 inheritance_info_vk.p_next =
533 inheritance_rendering_info_vk as *const _ as *const _;
534 }
535 }
536 }
537 }
538
539 let begin_info_vk = ash::vk::CommandBufferBeginInfo {
540 flags,
541 p_inheritance_info: inheritance_info_vk
542 .as_ref()
543 .map_or(ptr::null(), |info| info),
544 ..Default::default()
545 };
546
547 let fns = device.fns();
548
549 (fns.v1_0.begin_command_buffer)(builder_alloc.inner().handle(), &begin_info_vk)
550 .result()
551 .map_err(VulkanError::from)?;
552 }
553
554 let mut builder_state: CommandBufferBuilderState = Default::default();
555
556 if let Some(inheritance_info) = &inheritance_info {
557 let &CommandBufferInheritanceInfo {
558 ref render_pass,
559 occlusion_query: _,
560 query_statistics_flags: _,
561 _ne: _,
562 } = inheritance_info;
563
564 if let Some(render_pass) = render_pass {
565 builder_state.render_pass = Some(RenderPassState::from_inheritance(render_pass));
566 }
567 }
568
569 Ok(CommandBufferBuilder {
570 builder_alloc,
571 inheritance_info,
572 queue_family_index,
573 usage,
574
575 next_command_index: 0,
576 resources: Vec::new(),
577 builder_state,
578 resources_usage_state: Default::default(),
579
580 _data: PhantomData,
581 })
582 }
583
queue_family_properties(&self) -> &QueueFamilyProperties584 fn queue_family_properties(&self) -> &QueueFamilyProperties {
585 &self.device().physical_device().queue_family_properties()[self.queue_family_index as usize]
586 }
587 }
588
589 impl<A> CommandBufferBuilder<PrimaryCommandBuffer<A::Alloc>, A>
590 where
591 A: CommandBufferAllocator,
592 {
593 /// Builds the command buffer.
build(self) -> Result<PrimaryCommandBuffer<A::Alloc>, BuildError>594 pub fn build(self) -> Result<PrimaryCommandBuffer<A::Alloc>, BuildError> {
595 if self.builder_state.render_pass.is_some() {
596 return Err(BuildError::RenderPassActive);
597 }
598
599 if !self.builder_state.queries.is_empty() {
600 return Err(BuildError::QueryActive);
601 }
602
603 Ok(unsafe { self.build_unchecked()? })
604 }
605
606 #[cfg_attr(not(feature = "document_unchecked"), doc(hidden))]
build_unchecked(self) -> Result<PrimaryCommandBuffer<A::Alloc>, OomError>607 pub unsafe fn build_unchecked(self) -> Result<PrimaryCommandBuffer<A::Alloc>, OomError> {
608 let fns = self.device().fns();
609 (fns.v1_0.end_command_buffer)(self.builder_alloc.inner().handle())
610 .result()
611 .map_err(VulkanError::from)?;
612
613 Ok(PrimaryCommandBuffer {
614 alloc: self.builder_alloc.into_alloc(),
615 _usage: self.usage,
616 _resources: self.resources,
617
618 _state: Mutex::new(Default::default()),
619 })
620 }
621 }
622
623 impl<A> CommandBufferBuilder<SecondaryCommandBuffer<A::Alloc>, A>
624 where
625 A: CommandBufferAllocator,
626 {
627 /// Builds the command buffer.
build(self) -> Result<SecondaryCommandBuffer<A::Alloc>, BuildError>628 pub fn build(self) -> Result<SecondaryCommandBuffer<A::Alloc>, BuildError> {
629 if !self.builder_state.queries.is_empty() {
630 return Err(BuildError::QueryActive);
631 }
632
633 Ok(unsafe { self.build_unchecked()? })
634 }
635
636 #[cfg_attr(not(feature = "document_unchecked"), doc(hidden))]
build_unchecked(self) -> Result<SecondaryCommandBuffer<A::Alloc>, OomError>637 pub unsafe fn build_unchecked(self) -> Result<SecondaryCommandBuffer<A::Alloc>, OomError> {
638 let fns = self.device().fns();
639 (fns.v1_0.end_command_buffer)(self.builder_alloc.inner().handle())
640 .result()
641 .map_err(VulkanError::from)?;
642
643 let submit_state = match self.usage {
644 CommandBufferUsage::MultipleSubmit => SubmitState::ExclusiveUse {
645 in_use: AtomicBool::new(false),
646 },
647 CommandBufferUsage::SimultaneousUse => SubmitState::Concurrent,
648 CommandBufferUsage::OneTimeSubmit => SubmitState::OneTime {
649 already_submitted: AtomicBool::new(false),
650 },
651 };
652
653 Ok(SecondaryCommandBuffer {
654 alloc: self.builder_alloc.into_alloc(),
655 inheritance_info: self.inheritance_info.unwrap(),
656 usage: self.usage,
657
658 _resources: self.resources,
659
660 submit_state,
661 })
662 }
663 }
664
665 /// Holds the current binding and setting state.
666 #[derive(Default)]
667 struct CommandBufferBuilderState {
668 // Render pass
669 render_pass: Option<RenderPassState>,
670
671 // Bind/push
672 descriptor_sets: HashMap<PipelineBindPoint, DescriptorSetState>,
673 index_buffer: Option<(Subbuffer<[u8]>, IndexType)>,
674 pipeline_compute: Option<Arc<ComputePipeline>>,
675 pipeline_graphics: Option<Arc<GraphicsPipeline>>,
676 vertex_buffers: HashMap<u32, Subbuffer<[u8]>>,
677 push_constants: RangeSet<u32>,
678 push_constants_pipeline_layout: Option<Arc<PipelineLayout>>,
679
680 // Dynamic state
681 blend_constants: Option<[f32; 4]>,
682 color_write_enable: Option<SmallVec<[bool; 4]>>,
683 cull_mode: Option<CullMode>,
684 depth_bias: Option<DepthBias>,
685 depth_bias_enable: Option<bool>,
686 depth_bounds: Option<RangeInclusive<f32>>,
687 depth_bounds_test_enable: Option<bool>,
688 depth_compare_op: Option<CompareOp>,
689 depth_test_enable: Option<bool>,
690 depth_write_enable: Option<bool>,
691 discard_rectangle: HashMap<u32, Scissor>,
692 front_face: Option<FrontFace>,
693 line_stipple: Option<LineStipple>,
694 line_width: Option<f32>,
695 logic_op: Option<LogicOp>,
696 patch_control_points: Option<u32>,
697 primitive_restart_enable: Option<bool>,
698 primitive_topology: Option<PrimitiveTopology>,
699 rasterizer_discard_enable: Option<bool>,
700 scissor: HashMap<u32, Scissor>,
701 scissor_with_count: Option<SmallVec<[Scissor; 2]>>,
702 stencil_compare_mask: StencilStateDynamic,
703 stencil_op: StencilOpStateDynamic,
704 stencil_reference: StencilStateDynamic,
705 stencil_test_enable: Option<bool>,
706 stencil_write_mask: StencilStateDynamic,
707 viewport: HashMap<u32, Viewport>,
708 viewport_with_count: Option<SmallVec<[Viewport; 2]>>,
709
710 // Active queries
711 queries: HashMap<ash::vk::QueryType, QueryState>,
712 }
713
714 impl CommandBufferBuilderState {
reset_dynamic_states(&mut self, states: impl IntoIterator<Item = DynamicState>)715 fn reset_dynamic_states(&mut self, states: impl IntoIterator<Item = DynamicState>) {
716 for state in states {
717 match state {
718 DynamicState::BlendConstants => self.blend_constants = None,
719 DynamicState::ColorWriteEnable => self.color_write_enable = None,
720 DynamicState::CullMode => self.cull_mode = None,
721 DynamicState::DepthBias => self.depth_bias = None,
722 DynamicState::DepthBiasEnable => self.depth_bias_enable = None,
723 DynamicState::DepthBounds => self.depth_bounds = None,
724 DynamicState::DepthBoundsTestEnable => self.depth_bounds_test_enable = None,
725 DynamicState::DepthCompareOp => self.depth_compare_op = None,
726 DynamicState::DepthTestEnable => self.depth_test_enable = None,
727 DynamicState::DepthWriteEnable => self.depth_write_enable = None,
728 DynamicState::DiscardRectangle => self.discard_rectangle.clear(),
729 DynamicState::ExclusiveScissor => (), // TODO;
730 DynamicState::FragmentShadingRate => (), // TODO:
731 DynamicState::FrontFace => self.front_face = None,
732 DynamicState::LineStipple => self.line_stipple = None,
733 DynamicState::LineWidth => self.line_width = None,
734 DynamicState::LogicOp => self.logic_op = None,
735 DynamicState::PatchControlPoints => self.patch_control_points = None,
736 DynamicState::PrimitiveRestartEnable => self.primitive_restart_enable = None,
737 DynamicState::PrimitiveTopology => self.primitive_topology = None,
738 DynamicState::RasterizerDiscardEnable => self.rasterizer_discard_enable = None,
739 DynamicState::RayTracingPipelineStackSize => (), // TODO:
740 DynamicState::SampleLocations => (), // TODO:
741 DynamicState::Scissor => self.scissor.clear(),
742 DynamicState::ScissorWithCount => self.scissor_with_count = None,
743 DynamicState::StencilCompareMask => self.stencil_compare_mask = Default::default(),
744 DynamicState::StencilOp => self.stencil_op = Default::default(),
745 DynamicState::StencilReference => self.stencil_reference = Default::default(),
746 DynamicState::StencilTestEnable => self.stencil_test_enable = None,
747 DynamicState::StencilWriteMask => self.stencil_write_mask = Default::default(),
748 DynamicState::VertexInput => (), // TODO:
749 DynamicState::VertexInputBindingStride => (), // TODO:
750 DynamicState::Viewport => self.viewport.clear(),
751 DynamicState::ViewportCoarseSampleOrder => (), // TODO:
752 DynamicState::ViewportShadingRatePalette => (), // TODO:
753 DynamicState::ViewportWScaling => (), // TODO:
754 DynamicState::ViewportWithCount => self.viewport_with_count = None,
755 DynamicState::TessellationDomainOrigin => (), // TODO:
756 DynamicState::DepthClampEnable => (), // TODO:
757 DynamicState::PolygonMode => (), // TODO:
758 DynamicState::RasterizationSamples => (), // TODO:
759 DynamicState::SampleMask => (), // TODO:
760 DynamicState::AlphaToCoverageEnable => (), // TODO:
761 DynamicState::AlphaToOneEnable => (), // TODO:
762 DynamicState::LogicOpEnable => (), // TODO:
763 DynamicState::ColorBlendEnable => (), // TODO:
764 DynamicState::ColorBlendEquation => (), // TODO:
765 DynamicState::ColorWriteMask => (), // TODO:
766 DynamicState::RasterizationStream => (), // TODO:
767 DynamicState::ConservativeRasterizationMode => (), // TODO:
768 DynamicState::ExtraPrimitiveOverestimationSize => (), // TODO:
769 DynamicState::DepthClipEnable => (), // TODO:
770 DynamicState::SampleLocationsEnable => (), // TODO:
771 DynamicState::ColorBlendAdvanced => (), // TODO:
772 DynamicState::ProvokingVertexMode => (), // TODO:
773 DynamicState::LineRasterizationMode => (), // TODO:
774 DynamicState::LineStippleEnable => (), // TODO:
775 DynamicState::DepthClipNegativeOneToOne => (), // TODO:
776 DynamicState::ViewportWScalingEnable => (), // TODO:
777 DynamicState::ViewportSwizzle => (), // TODO:
778 DynamicState::CoverageToColorEnable => (), // TODO:
779 DynamicState::CoverageToColorLocation => (), // TODO:
780 DynamicState::CoverageModulationMode => (), // TODO:
781 DynamicState::CoverageModulationTableEnable => (), // TODO:
782 DynamicState::CoverageModulationTable => (), // TODO:
783 DynamicState::ShadingRateImageEnable => (), // TODO:
784 DynamicState::RepresentativeFragmentTestEnable => (), // TODO:
785 DynamicState::CoverageReductionMode => (), // TODO:
786 }
787 }
788 }
789
invalidate_descriptor_sets( &mut self, pipeline_bind_point: PipelineBindPoint, pipeline_layout: Arc<PipelineLayout>, first_set: u32, num_descriptor_sets: u32, ) -> &mut DescriptorSetState790 fn invalidate_descriptor_sets(
791 &mut self,
792 pipeline_bind_point: PipelineBindPoint,
793 pipeline_layout: Arc<PipelineLayout>,
794 first_set: u32,
795 num_descriptor_sets: u32,
796 ) -> &mut DescriptorSetState {
797 match self.descriptor_sets.entry(pipeline_bind_point) {
798 Entry::Vacant(entry) => entry.insert(DescriptorSetState {
799 descriptor_sets: Default::default(),
800 pipeline_layout,
801 }),
802 Entry::Occupied(entry) => {
803 let state = entry.into_mut();
804
805 let invalidate_from = if state.pipeline_layout == pipeline_layout {
806 // If we're still using the exact same layout, then of course it's compatible.
807 None
808 } else if state.pipeline_layout.push_constant_ranges()
809 != pipeline_layout.push_constant_ranges()
810 {
811 // If the push constant ranges don't match,
812 // all bound descriptor sets are disturbed.
813 Some(0)
814 } else {
815 // Find the first descriptor set layout in the current pipeline layout that
816 // isn't compatible with the corresponding set in the new pipeline layout.
817 // If an incompatible set was found, all bound sets from that slot onwards will
818 // be disturbed.
819 let current_layouts = state.pipeline_layout.set_layouts();
820 let new_layouts = pipeline_layout.set_layouts();
821 let max = (current_layouts.len() as u32).min(first_set + num_descriptor_sets);
822 (0..max).find(|&num| {
823 let num = num as usize;
824 !current_layouts[num].is_compatible_with(&new_layouts[num])
825 })
826 };
827
828 if let Some(invalidate_from) = invalidate_from {
829 // Remove disturbed sets and set new pipeline layout.
830 state
831 .descriptor_sets
832 .retain(|&num, _| num < invalidate_from);
833 state.pipeline_layout = pipeline_layout;
834 } else if (first_set + num_descriptor_sets) as usize
835 >= state.pipeline_layout.set_layouts().len()
836 {
837 // New layout is a superset of the old one.
838 state.pipeline_layout = pipeline_layout;
839 }
840
841 state
842 }
843 }
844 }
845 }
846
847 struct RenderPassState {
848 contents: SubpassContents,
849 render_area_offset: [u32; 2],
850 render_area_extent: [u32; 2],
851
852 rendering_info: PipelineRenderingCreateInfo,
853 attachments: Option<RenderPassStateAttachments>,
854
855 render_pass: RenderPassStateType,
856 }
857
858 impl RenderPassState {
from_inheritance(render_pass: &CommandBufferInheritanceRenderPassType) -> Self859 fn from_inheritance(render_pass: &CommandBufferInheritanceRenderPassType) -> Self {
860 match render_pass {
861 CommandBufferInheritanceRenderPassType::BeginRenderPass(info) => {
862 RenderPassState {
863 contents: SubpassContents::Inline,
864 render_area_offset: [0, 0],
865 render_area_extent: (info.framebuffer.as_ref())
866 // Still not exact, but it's a better upper bound.
867 .map_or([u32::MAX, u32::MAX], |framebuffer| framebuffer.extent()),
868
869 rendering_info: PipelineRenderingCreateInfo::from_subpass(&info.subpass),
870 attachments: info.framebuffer.as_ref().map(|framebuffer| {
871 RenderPassStateAttachments::from_subpass(&info.subpass, framebuffer)
872 }),
873
874 render_pass: BeginRenderPassState {
875 subpass: info.subpass.clone(),
876 framebuffer: info.framebuffer.clone(),
877 }
878 .into(),
879 }
880 }
881 CommandBufferInheritanceRenderPassType::BeginRendering(info) => RenderPassState {
882 contents: SubpassContents::Inline,
883 render_area_offset: [0, 0],
884 render_area_extent: [u32::MAX, u32::MAX],
885
886 rendering_info: PipelineRenderingCreateInfo::from_inheritance_rendering_info(info),
887 attachments: None,
888
889 render_pass: BeginRenderingState {
890 pipeline_used: false,
891 }
892 .into(),
893 },
894 }
895 }
896 }
897
898 enum RenderPassStateType {
899 BeginRenderPass(BeginRenderPassState),
900 BeginRendering(BeginRenderingState),
901 }
902
903 impl From<BeginRenderPassState> for RenderPassStateType {
904 #[inline]
from(val: BeginRenderPassState) -> Self905 fn from(val: BeginRenderPassState) -> Self {
906 Self::BeginRenderPass(val)
907 }
908 }
909
910 impl From<BeginRenderingState> for RenderPassStateType {
911 #[inline]
from(val: BeginRenderingState) -> Self912 fn from(val: BeginRenderingState) -> Self {
913 Self::BeginRendering(val)
914 }
915 }
916
917 struct BeginRenderPassState {
918 subpass: Subpass,
919 framebuffer: Option<Arc<Framebuffer>>,
920 }
921
922 struct BeginRenderingState {
923 pipeline_used: bool,
924 }
925
926 struct RenderPassStateAttachments {
927 color_attachments: Vec<Option<RenderPassStateAttachmentInfo>>,
928 depth_attachment: Option<RenderPassStateAttachmentInfo>,
929 stencil_attachment: Option<RenderPassStateAttachmentInfo>,
930 }
931
932 impl RenderPassStateAttachments {
from_subpass(subpass: &Subpass, framebuffer: &Framebuffer) -> Self933 fn from_subpass(subpass: &Subpass, framebuffer: &Framebuffer) -> Self {
934 let subpass_desc = subpass.subpass_desc();
935 let rp_attachments = subpass.render_pass().attachments();
936 let fb_attachments = framebuffer.attachments();
937
938 Self {
939 color_attachments: (subpass_desc.color_attachments.iter().enumerate())
940 .map(|(index, atch_ref)| {
941 (atch_ref.as_ref()).map(|atch_ref| RenderPassStateAttachmentInfo {
942 image_view: fb_attachments[atch_ref.attachment as usize].clone(),
943 image_layout: atch_ref.layout,
944 load_access: subpass
945 .load_op(atch_ref.attachment)
946 .and_then(color_load_access),
947 store_access: subpass
948 .store_op(atch_ref.attachment)
949 .and_then(color_store_access),
950 resolve_info: (subpass_desc.resolve_attachments.get(index))
951 .and_then(|atch_ref| atch_ref.as_ref())
952 .map(|atch_ref| RenderPassStateAttachmentResolveInfo {
953 image_view: fb_attachments[atch_ref.attachment as usize].clone(),
954 image_layout: atch_ref.layout,
955 load_access: subpass
956 .load_op(atch_ref.attachment)
957 .and_then(color_load_access),
958 store_access: subpass
959 .store_op(atch_ref.attachment)
960 .and_then(color_store_access),
961 }),
962 })
963 })
964 .collect(),
965 depth_attachment: (subpass_desc.depth_stencil_attachment.as_ref())
966 .filter(|atch_ref| {
967 (rp_attachments[atch_ref.attachment as usize].format.unwrap())
968 .aspects()
969 .intersects(ImageAspects::DEPTH)
970 })
971 .map(|atch_ref| RenderPassStateAttachmentInfo {
972 image_view: fb_attachments[atch_ref.attachment as usize].clone(),
973 image_layout: atch_ref.layout,
974 load_access: subpass
975 .load_op(atch_ref.attachment)
976 .and_then(depth_stencil_load_access),
977 store_access: subpass
978 .store_op(atch_ref.attachment)
979 .and_then(depth_stencil_store_access),
980 resolve_info: None,
981 }),
982 stencil_attachment: (subpass_desc.depth_stencil_attachment.as_ref())
983 .filter(|atch_ref| {
984 (rp_attachments[atch_ref.attachment as usize].format.unwrap())
985 .aspects()
986 .intersects(ImageAspects::STENCIL)
987 })
988 .map(|atch_ref| RenderPassStateAttachmentInfo {
989 image_view: fb_attachments[atch_ref.attachment as usize].clone(),
990 image_layout: atch_ref.layout,
991 load_access: subpass
992 .stencil_load_op(atch_ref.attachment)
993 .and_then(depth_stencil_load_access),
994 store_access: subpass
995 .stencil_store_op(atch_ref.attachment)
996 .and_then(depth_stencil_store_access),
997 resolve_info: None,
998 }),
999 }
1000 }
1001
from_rendering_info(info: &RenderingInfo) -> Self1002 fn from_rendering_info(info: &RenderingInfo) -> Self {
1003 Self {
1004 color_attachments: (info.color_attachments.iter())
1005 .map(|atch_info| {
1006 (atch_info.as_ref()).map(|atch_info| RenderPassStateAttachmentInfo {
1007 image_view: atch_info.image_view.clone(),
1008 image_layout: atch_info.image_layout,
1009 load_access: color_load_access(atch_info.load_op),
1010 store_access: color_store_access(atch_info.store_op),
1011 resolve_info: atch_info.resolve_info.as_ref().map(|resolve_atch_info| {
1012 RenderPassStateAttachmentResolveInfo {
1013 image_view: resolve_atch_info.image_view.clone(),
1014 image_layout: resolve_atch_info.image_layout,
1015 load_access: None,
1016 store_access: None,
1017 }
1018 }),
1019 })
1020 })
1021 .collect(),
1022 depth_attachment: (info.depth_attachment.as_ref()).map(|atch_info| {
1023 RenderPassStateAttachmentInfo {
1024 image_view: atch_info.image_view.clone(),
1025 image_layout: atch_info.image_layout,
1026 load_access: depth_stencil_load_access(atch_info.load_op),
1027 store_access: depth_stencil_store_access(atch_info.store_op),
1028 resolve_info: atch_info.resolve_info.as_ref().map(|resolve_atch_info| {
1029 RenderPassStateAttachmentResolveInfo {
1030 image_view: resolve_atch_info.image_view.clone(),
1031 image_layout: resolve_atch_info.image_layout,
1032 load_access: None,
1033 store_access: None,
1034 }
1035 }),
1036 }
1037 }),
1038 stencil_attachment: (info.stencil_attachment.as_ref()).map(|atch_info| {
1039 RenderPassStateAttachmentInfo {
1040 image_view: atch_info.image_view.clone(),
1041 image_layout: atch_info.image_layout,
1042 load_access: depth_stencil_load_access(atch_info.load_op),
1043 store_access: depth_stencil_store_access(atch_info.store_op),
1044 resolve_info: atch_info.resolve_info.as_ref().map(|resolve_atch_info| {
1045 RenderPassStateAttachmentResolveInfo {
1046 image_view: resolve_atch_info.image_view.clone(),
1047 image_layout: resolve_atch_info.image_layout,
1048 load_access: None,
1049 store_access: None,
1050 }
1051 }),
1052 }
1053 }),
1054 }
1055 }
1056 }
1057
color_load_access(load_op: LoadOp) -> Option<PipelineStageAccess>1058 fn color_load_access(load_op: LoadOp) -> Option<PipelineStageAccess> {
1059 match load_op {
1060 LoadOp::Load => Some(PipelineStageAccess::ColorAttachmentOutput_ColorAttachmentRead),
1061 LoadOp::Clear => Some(PipelineStageAccess::ColorAttachmentOutput_ColorAttachmentWrite),
1062 LoadOp::DontCare => Some(PipelineStageAccess::ColorAttachmentOutput_ColorAttachmentWrite),
1063 //LoadOp::None => None,
1064 }
1065 }
1066
depth_stencil_load_access(load_op: LoadOp) -> Option<PipelineStageAccess>1067 fn depth_stencil_load_access(load_op: LoadOp) -> Option<PipelineStageAccess> {
1068 match load_op {
1069 LoadOp::Load => Some(PipelineStageAccess::EarlyFragmentTests_DepthStencilAttachmentRead),
1070 LoadOp::Clear => Some(PipelineStageAccess::EarlyFragmentTests_DepthStencilAttachmentWrite),
1071 LoadOp::DontCare => {
1072 Some(PipelineStageAccess::EarlyFragmentTests_DepthStencilAttachmentWrite)
1073 } //LoadOp::None => None,
1074 }
1075 }
1076
color_store_access(store_op: StoreOp) -> Option<PipelineStageAccess>1077 fn color_store_access(store_op: StoreOp) -> Option<PipelineStageAccess> {
1078 match store_op {
1079 StoreOp::Store => Some(PipelineStageAccess::ColorAttachmentOutput_ColorAttachmentWrite),
1080 StoreOp::DontCare => Some(PipelineStageAccess::ColorAttachmentOutput_ColorAttachmentWrite),
1081 // StoreOp::None => None,
1082 }
1083 }
1084
depth_stencil_store_access(store_op: StoreOp) -> Option<PipelineStageAccess>1085 fn depth_stencil_store_access(store_op: StoreOp) -> Option<PipelineStageAccess> {
1086 match store_op {
1087 StoreOp::Store => Some(PipelineStageAccess::LateFragmentTests_DepthStencilAttachmentWrite),
1088 StoreOp::DontCare => {
1089 Some(PipelineStageAccess::LateFragmentTests_DepthStencilAttachmentWrite)
1090 } // StoreOp::None => None,
1091 }
1092 }
1093
1094 struct RenderPassStateAttachmentInfo {
1095 image_view: Arc<dyn ImageViewAbstract>,
1096 image_layout: ImageLayout,
1097 load_access: Option<PipelineStageAccess>,
1098 store_access: Option<PipelineStageAccess>,
1099 resolve_info: Option<RenderPassStateAttachmentResolveInfo>,
1100 }
1101
1102 struct RenderPassStateAttachmentResolveInfo {
1103 image_view: Arc<dyn ImageViewAbstract>,
1104 image_layout: ImageLayout,
1105 load_access: Option<PipelineStageAccess>,
1106 store_access: Option<PipelineStageAccess>,
1107 }
1108
1109 struct DescriptorSetState {
1110 descriptor_sets: HashMap<u32, SetOrPush>,
1111 pipeline_layout: Arc<PipelineLayout>,
1112 }
1113
1114 #[derive(Clone)]
1115 enum SetOrPush {
1116 Set(DescriptorSetWithOffsets),
1117 Push(DescriptorSetResources),
1118 }
1119
1120 impl SetOrPush {
resources(&self) -> &DescriptorSetResources1121 pub fn resources(&self) -> &DescriptorSetResources {
1122 match self {
1123 Self::Set(set) => set.as_ref().0.resources(),
1124 Self::Push(resources) => resources,
1125 }
1126 }
1127
1128 #[inline]
dynamic_offsets(&self) -> &[u32]1129 pub fn dynamic_offsets(&self) -> &[u32] {
1130 match self {
1131 Self::Set(set) => set.as_ref().1,
1132 Self::Push(_) => &[],
1133 }
1134 }
1135 }
1136
1137 #[derive(Clone, Copy, Debug, Default)]
1138 struct StencilStateDynamic {
1139 front: Option<u32>,
1140 back: Option<u32>,
1141 }
1142
1143 #[derive(Clone, Copy, Debug, Default)]
1144 struct StencilOpStateDynamic {
1145 front: Option<StencilOps>,
1146 back: Option<StencilOps>,
1147 }
1148
1149 struct QueryState {
1150 query_pool: ash::vk::QueryPool,
1151 query: u32,
1152 ty: QueryType,
1153 flags: QueryControlFlags,
1154 in_subpass: bool,
1155 }
1156
1157 #[derive(Debug, Default)]
1158 struct ResourcesState {
1159 buffers: HashMap<Arc<Buffer>, RangeMap<DeviceSize, BufferRangeState>>,
1160 images: HashMap<Arc<Image>, RangeMap<DeviceSize, ImageRangeState>>,
1161 }
1162
1163 #[derive(Clone, Debug, Default, PartialEq, Eq)]
1164 struct BufferRangeState {
1165 resource_uses: Vec<ResourceUseRef>,
1166 memory_access: MemoryAccessState,
1167 }
1168
1169 #[derive(Clone, Debug, Default, PartialEq, Eq)]
1170 struct ImageRangeState {
1171 resource_uses: Vec<ResourceUseRef>,
1172 memory_access: MemoryAccessState,
1173 expected_layout: ImageLayout,
1174 current_layout: ImageLayout,
1175 }
1176
1177 impl ResourcesState {
record_buffer_access( &mut self, use_ref: &ResourceUseRef, buffer: &Arc<Buffer>, range: Range<DeviceSize>, stage_access: PipelineStageAccess, )1178 fn record_buffer_access(
1179 &mut self,
1180 use_ref: &ResourceUseRef,
1181 buffer: &Arc<Buffer>,
1182 range: Range<DeviceSize>,
1183 stage_access: PipelineStageAccess,
1184 ) {
1185 let range_map = self.buffers.entry(buffer.clone()).or_insert_with(|| {
1186 [(0..buffer.size(), Default::default())]
1187 .into_iter()
1188 .collect()
1189 });
1190 range_map.split_at(&range.start);
1191 range_map.split_at(&range.end);
1192
1193 for (_range, state) in range_map.range_mut(&range) {
1194 state.resource_uses.push(*use_ref);
1195 state.memory_access.record_access(use_ref, stage_access);
1196 }
1197 }
1198
record_image_access( &mut self, use_ref: &ResourceUseRef, image: &Arc<Image>, subresource_range: ImageSubresourceRange, stage_access: PipelineStageAccess, image_layout: ImageLayout, )1199 fn record_image_access(
1200 &mut self,
1201 use_ref: &ResourceUseRef,
1202 image: &Arc<Image>,
1203 subresource_range: ImageSubresourceRange,
1204 stage_access: PipelineStageAccess,
1205 image_layout: ImageLayout,
1206 ) {
1207 let range_map = self.images.entry(image.clone()).or_insert_with(|| {
1208 [(0..image.range_size(), Default::default())]
1209 .into_iter()
1210 .collect()
1211 });
1212
1213 for range in image.iter_ranges(subresource_range) {
1214 range_map.split_at(&range.start);
1215 range_map.split_at(&range.end);
1216
1217 for (_range, state) in range_map.range_mut(&range) {
1218 if state.resource_uses.is_empty() {
1219 state.expected_layout = image_layout;
1220 }
1221
1222 state.resource_uses.push(*use_ref);
1223 state.memory_access.record_access(use_ref, stage_access);
1224 }
1225 }
1226 }
1227
record_pipeline_barrier( &mut self, command_index: usize, command_name: &'static str, dependency_info: &DependencyInfo, queue_flags: QueueFlags, )1228 fn record_pipeline_barrier(
1229 &mut self,
1230 command_index: usize,
1231 command_name: &'static str,
1232 dependency_info: &DependencyInfo,
1233 queue_flags: QueueFlags,
1234 ) {
1235 for barrier in &dependency_info.buffer_memory_barriers {
1236 let barrier_scopes = BarrierScopes::from_buffer_memory_barrier(barrier, queue_flags);
1237 let &BufferMemoryBarrier {
1238 src_stages: _,
1239 src_access: _,
1240 dst_stages: _,
1241 dst_access: _,
1242 queue_family_ownership_transfer: _,
1243 ref buffer,
1244 ref range,
1245 _ne: _,
1246 } = barrier;
1247
1248 let range_map = self.buffers.entry(buffer.clone()).or_insert_with(|| {
1249 [(0..buffer.size(), Default::default())]
1250 .into_iter()
1251 .collect()
1252 });
1253 range_map.split_at(&range.start);
1254 range_map.split_at(&range.end);
1255
1256 for (_range, state) in range_map.range_mut(range) {
1257 state.memory_access.record_barrier(&barrier_scopes, None);
1258 }
1259 }
1260
1261 for (index, barrier) in dependency_info.image_memory_barriers.iter().enumerate() {
1262 let index = index as u32;
1263 let barrier_scopes = BarrierScopes::from_image_memory_barrier(barrier, queue_flags);
1264 let &ImageMemoryBarrier {
1265 src_stages: _,
1266 src_access: _,
1267 dst_stages: _,
1268 dst_access: _,
1269 old_layout,
1270 new_layout,
1271 queue_family_ownership_transfer: _,
1272 ref image,
1273 ref subresource_range,
1274 _ne,
1275 } = barrier;
1276
1277 // This is only used if there is a layout transition.
1278 let use_ref = ResourceUseRef {
1279 command_index,
1280 command_name,
1281 resource_in_command: ResourceInCommand::ImageMemoryBarrier { index },
1282 secondary_use_ref: None,
1283 };
1284 let layout_transition = (old_layout != new_layout).then_some(&use_ref);
1285
1286 let range_map = self.images.entry(image.clone()).or_insert_with(|| {
1287 [(0..image.range_size(), Default::default())]
1288 .into_iter()
1289 .collect()
1290 });
1291
1292 for range in image.iter_ranges(subresource_range.clone()) {
1293 range_map.split_at(&range.start);
1294 range_map.split_at(&range.end);
1295
1296 for (_range, state) in range_map.range_mut(&range) {
1297 if old_layout != new_layout {
1298 if state.resource_uses.is_empty() {
1299 state.expected_layout = old_layout;
1300 }
1301
1302 state.resource_uses.push(ResourceUseRef {
1303 command_index,
1304 command_name,
1305 resource_in_command: ResourceInCommand::ImageMemoryBarrier { index },
1306 secondary_use_ref: None,
1307 });
1308 state.current_layout = new_layout;
1309 }
1310
1311 state
1312 .memory_access
1313 .record_barrier(&barrier_scopes, layout_transition);
1314 }
1315 }
1316 }
1317
1318 for barrier in &dependency_info.buffer_memory_barriers {
1319 let &BufferMemoryBarrier {
1320 ref buffer,
1321 ref range,
1322 ..
1323 } = barrier;
1324
1325 let range_map = self.buffers.get_mut(buffer).unwrap();
1326 for (_range, state) in range_map.range_mut(range) {
1327 state.memory_access.apply_pending();
1328 }
1329 }
1330
1331 for barrier in &dependency_info.image_memory_barriers {
1332 let &ImageMemoryBarrier {
1333 ref image,
1334 ref subresource_range,
1335 ..
1336 } = barrier;
1337
1338 let range_map = self.images.get_mut(image).unwrap();
1339 for range in image.iter_ranges(subresource_range.clone()) {
1340 for (_range, state) in range_map.range_mut(&range) {
1341 state.memory_access.apply_pending();
1342 }
1343 }
1344 }
1345 }
1346 }
1347
1348 #[derive(Clone, Debug, Default, PartialEq, Eq)]
1349 struct MemoryAccessState {
1350 mutable: bool,
1351 last_write: Option<WriteState>,
1352 reads_since_last_write: HashMap<PipelineStage, ReadState>,
1353
1354 /// Pending changes that have not yet been applied. This is used during barrier recording.
1355 pending: Option<PendingWriteState>,
1356 }
1357
1358 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
1359 struct WriteState {
1360 use_ref: ResourceUseRef,
1361 access: PipelineStageAccess,
1362
1363 /// The `dst_stages` and `dst_access` of all barriers that protect against this write.
1364 barriers_since: PipelineStageAccessSet,
1365
1366 /// The `dst_stages` of all barriers that form a dependency chain with this write.
1367 dependency_chain: PipelineStages,
1368
1369 /// The union of all `barriers_since` of all `reads_since_last_write`.
1370 read_barriers_since: PipelineStages,
1371 }
1372
1373 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1374 struct PendingWriteState {
1375 /// If this is `Some`, then the barrier is treated as a new write,
1376 /// and the previous `last_write` is discarded.
1377 /// Otherwise, the values below are added to the existing `last_write`.
1378 layout_transition: Option<ResourceUseRef>,
1379
1380 barriers_since: PipelineStageAccessSet,
1381 dependency_chain: PipelineStages,
1382 }
1383
1384 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
1385 struct ReadState {
1386 use_ref: ResourceUseRef,
1387 access: PipelineStageAccess,
1388
1389 /// The `dst_stages` of all barriers that protect against this read.
1390 /// This always includes the stage of `self`.
1391 barriers_since: PipelineStages,
1392
1393 /// Stages of reads recorded after this read,
1394 /// that were in scope of `barriers_since` at the time of recording.
1395 /// This always includes the stage of `self`.
1396 barriered_reads_since: PipelineStages,
1397
1398 /// Pending changes that have not yet been applied. This is used during barrier recording.
1399 pending: Option<PendingReadState>,
1400 }
1401
1402 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1403 struct PendingReadState {
1404 barriers_since: PipelineStages,
1405 }
1406
1407 impl MemoryAccessState {
record_access(&mut self, use_ref: &ResourceUseRef, access: PipelineStageAccess)1408 fn record_access(&mut self, use_ref: &ResourceUseRef, access: PipelineStageAccess) {
1409 if access.is_write() {
1410 self.mutable = true;
1411 self.last_write = Some(WriteState {
1412 use_ref: *use_ref,
1413 access,
1414 barriers_since: Default::default(),
1415 dependency_chain: Default::default(),
1416 read_barriers_since: Default::default(),
1417 });
1418 self.reads_since_last_write.clear();
1419 } else {
1420 let pipeline_stage = PipelineStage::try_from(access).unwrap();
1421 let pipeline_stages = PipelineStages::from(pipeline_stage);
1422
1423 for read_state in self.reads_since_last_write.values_mut() {
1424 if read_state.barriers_since.intersects(pipeline_stages) {
1425 read_state.barriered_reads_since |= pipeline_stages;
1426 } else {
1427 read_state.barriered_reads_since -= pipeline_stages;
1428 }
1429 }
1430
1431 self.reads_since_last_write.insert(
1432 pipeline_stage,
1433 ReadState {
1434 use_ref: *use_ref,
1435 access,
1436 barriers_since: pipeline_stages,
1437 barriered_reads_since: pipeline_stages,
1438 pending: None,
1439 },
1440 );
1441 }
1442 }
1443
record_barrier( &mut self, barrier_scopes: &BarrierScopes, layout_transition: Option<&ResourceUseRef>, )1444 fn record_barrier(
1445 &mut self,
1446 barrier_scopes: &BarrierScopes,
1447 layout_transition: Option<&ResourceUseRef>,
1448 ) {
1449 let skip_reads = if let Some(use_ref) = layout_transition {
1450 let pending = self.pending.get_or_insert_with(Default::default);
1451 pending.layout_transition = Some(*use_ref);
1452 true
1453 } else {
1454 self.pending
1455 .map_or(false, |pending| pending.layout_transition.is_some())
1456 };
1457
1458 // If the last write is in the src scope of the barrier, then add the dst scopes.
1459 // If the barrier includes a layout transition, then that layout transition is
1460 // considered the last write, and it is always in the src scope of the barrier.
1461 if layout_transition.is_some()
1462 || self.last_write.as_ref().map_or(false, |write_state| {
1463 barrier_scopes
1464 .src_access_scope
1465 .contains_enum(write_state.access)
1466 || barrier_scopes
1467 .src_exec_scope
1468 .intersects(write_state.dependency_chain)
1469 })
1470 {
1471 let pending = self.pending.get_or_insert_with(Default::default);
1472 pending.barriers_since |= barrier_scopes.dst_access_scope;
1473 pending.dependency_chain |= barrier_scopes.dst_exec_scope;
1474 }
1475
1476 // A layout transition counts as a write, which means that `reads_since_last_write` will
1477 // be cleared when applying pending operations.
1478 // Therefore, there is no need to update the reads.
1479 if !skip_reads {
1480 // Gather all reads for which `barriers_since` is in the barrier's `src_exec_scope`.
1481 let reads_in_src_exec_scope = self.reads_since_last_write.iter().fold(
1482 PipelineStages::empty(),
1483 |total, (&stage, read_state)| {
1484 if barrier_scopes
1485 .src_exec_scope
1486 .intersects(read_state.barriers_since)
1487 {
1488 total.union(stage.into())
1489 } else {
1490 total
1491 }
1492 },
1493 );
1494
1495 for read_state in self.reads_since_last_write.values_mut() {
1496 if reads_in_src_exec_scope.intersects(read_state.barriered_reads_since) {
1497 let pending = read_state.pending.get_or_insert_with(Default::default);
1498 pending.barriers_since |= barrier_scopes.dst_exec_scope;
1499 }
1500 }
1501 }
1502 }
1503
apply_pending(&mut self)1504 fn apply_pending(&mut self) {
1505 if let Some(PendingWriteState {
1506 layout_transition,
1507 barriers_since,
1508 dependency_chain,
1509 }) = self.pending.take()
1510 {
1511 // If there is a pending layout transition, it is treated as the new `last_write`.
1512 if let Some(use_ref) = layout_transition {
1513 self.mutable = true;
1514 self.last_write = Some(WriteState {
1515 use_ref,
1516 access: PipelineStageAccess::ImageLayoutTransition,
1517 barriers_since,
1518 dependency_chain,
1519 read_barriers_since: Default::default(),
1520 });
1521 self.reads_since_last_write.clear();
1522 } else if let Some(write_state) = &mut self.last_write {
1523 write_state.barriers_since |= barriers_since;
1524 write_state.dependency_chain |= dependency_chain;
1525 }
1526 }
1527
1528 for read_state in self.reads_since_last_write.values_mut() {
1529 if let Some(PendingReadState { barriers_since }) = read_state.pending.take() {
1530 read_state.barriers_since |= barriers_since;
1531
1532 if let Some(write_state) = &mut self.last_write {
1533 write_state.read_barriers_since |= read_state.barriers_since;
1534 }
1535 }
1536 }
1537 }
1538 }
1539
1540 struct BarrierScopes {
1541 src_exec_scope: PipelineStages,
1542 src_access_scope: PipelineStageAccessSet,
1543 dst_exec_scope: PipelineStages,
1544 dst_access_scope: PipelineStageAccessSet,
1545 }
1546
1547 impl BarrierScopes {
from_buffer_memory_barrier(barrier: &BufferMemoryBarrier, queue_flags: QueueFlags) -> Self1548 fn from_buffer_memory_barrier(barrier: &BufferMemoryBarrier, queue_flags: QueueFlags) -> Self {
1549 let src_stages_expanded = barrier.src_stages.expand(queue_flags);
1550 let src_exec_scope = src_stages_expanded.with_earlier();
1551 let src_access_scope = PipelineStageAccessSet::from(barrier.src_access)
1552 & PipelineStageAccessSet::from(src_stages_expanded);
1553
1554 let dst_stages_expanded = barrier.dst_stages.expand(queue_flags);
1555 let dst_exec_scope = dst_stages_expanded.with_later();
1556 let dst_access_scope = PipelineStageAccessSet::from(barrier.dst_access)
1557 & PipelineStageAccessSet::from(dst_stages_expanded);
1558
1559 Self {
1560 src_exec_scope,
1561 src_access_scope,
1562 dst_exec_scope,
1563 dst_access_scope,
1564 }
1565 }
1566
from_image_memory_barrier(barrier: &ImageMemoryBarrier, queue_flags: QueueFlags) -> Self1567 fn from_image_memory_barrier(barrier: &ImageMemoryBarrier, queue_flags: QueueFlags) -> Self {
1568 let src_stages_expanded = barrier.src_stages.expand(queue_flags);
1569 let src_exec_scope = src_stages_expanded.with_earlier();
1570 let src_access_scope = PipelineStageAccessSet::from(barrier.src_access)
1571 & PipelineStageAccessSet::from(src_stages_expanded);
1572
1573 let dst_stages_expanded = barrier.dst_stages.expand(queue_flags);
1574 let dst_exec_scope = dst_stages_expanded.with_later();
1575 let dst_access_scope = PipelineStageAccessSet::from(barrier.dst_access)
1576 & PipelineStageAccessSet::from(dst_stages_expanded);
1577
1578 Self {
1579 src_exec_scope,
1580 src_access_scope,
1581 dst_exec_scope,
1582 dst_access_scope,
1583 }
1584 }
1585 }
1586