1 // Copyright 2024 Google LLC 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 pub mod gainmap; 16 pub mod item; 17 pub mod tile; 18 pub mod track; 19 20 use crate::decoder::gainmap::*; 21 use crate::decoder::item::*; 22 use crate::decoder::tile::*; 23 use crate::decoder::track::*; 24 25 #[cfg(feature = "dav1d")] 26 use crate::codecs::dav1d::Dav1d; 27 28 #[cfg(feature = "libgav1")] 29 use crate::codecs::libgav1::Libgav1; 30 31 #[cfg(feature = "android_mediacodec")] 32 use crate::codecs::android_mediacodec::MediaCodec; 33 34 use crate::codecs::DecoderConfig; 35 use crate::image::*; 36 use crate::internal_utils::io::*; 37 use crate::internal_utils::*; 38 use crate::parser::exif; 39 use crate::parser::mp4box; 40 use crate::parser::mp4box::*; 41 use crate::parser::obu::Av1SequenceHeader; 42 use crate::*; 43 44 use std::cmp::max; 45 use std::cmp::min; 46 47 pub trait IO { read(&mut self, offset: u64, max_read_size: usize) -> AvifResult<&[u8]>48 fn read(&mut self, offset: u64, max_read_size: usize) -> AvifResult<&[u8]>; size_hint(&self) -> u6449 fn size_hint(&self) -> u64; persistent(&self) -> bool50 fn persistent(&self) -> bool; 51 } 52 53 impl dyn IO { read_exact(&mut self, offset: u64, read_size: usize) -> AvifResult<&[u8]>54 pub fn read_exact(&mut self, offset: u64, read_size: usize) -> AvifResult<&[u8]> { 55 let result = self.read(offset, read_size)?; 56 if result.len() < read_size { 57 Err(AvifError::TruncatedData) 58 } else { 59 assert!(result.len() == read_size); 60 Ok(result) 61 } 62 } 63 } 64 65 pub type GenericIO = Box<dyn IO>; 66 pub type Codec = Box<dyn crate::codecs::Decoder>; 67 68 #[derive(Debug, Default)] 69 pub enum CodecChoice { 70 #[default] 71 Auto, 72 Dav1d, 73 Libgav1, 74 MediaCodec, 75 } 76 77 impl CodecChoice { get_codec(&self, is_avif: bool) -> AvifResult<Codec>78 fn get_codec(&self, is_avif: bool) -> AvifResult<Codec> { 79 match self { 80 CodecChoice::Auto => { 81 // Preferred order of codecs in Auto mode: Android MediaCodec, Dav1d, Libgav1. 82 CodecChoice::MediaCodec 83 .get_codec(is_avif) 84 .or_else(|_| CodecChoice::Dav1d.get_codec(is_avif)) 85 .or_else(|_| CodecChoice::Libgav1.get_codec(is_avif)) 86 } 87 CodecChoice::Dav1d => { 88 if !is_avif { 89 return Err(AvifError::NoCodecAvailable); 90 } 91 #[cfg(feature = "dav1d")] 92 return Ok(Box::<Dav1d>::default()); 93 #[cfg(not(feature = "dav1d"))] 94 return Err(AvifError::NoCodecAvailable); 95 } 96 CodecChoice::Libgav1 => { 97 if !is_avif { 98 return Err(AvifError::NoCodecAvailable); 99 } 100 #[cfg(feature = "libgav1")] 101 return Ok(Box::<Libgav1>::default()); 102 #[cfg(not(feature = "libgav1"))] 103 return Err(AvifError::NoCodecAvailable); 104 } 105 CodecChoice::MediaCodec => { 106 #[cfg(feature = "android_mediacodec")] 107 return Ok(Box::<MediaCodec>::default()); 108 #[cfg(not(feature = "android_mediacodec"))] 109 return Err(AvifError::NoCodecAvailable); 110 } 111 } 112 } 113 } 114 115 #[repr(C)] 116 #[derive(Clone, Copy, Debug, Default, PartialEq)] 117 pub enum Source { 118 #[default] 119 Auto = 0, 120 PrimaryItem = 1, 121 Tracks = 2, 122 // TODO: Thumbnail, 123 } 124 125 pub const DEFAULT_IMAGE_SIZE_LIMIT: u32 = 16384 * 16384; 126 pub const DEFAULT_IMAGE_DIMENSION_LIMIT: u32 = 32768; 127 pub const DEFAULT_IMAGE_COUNT_LIMIT: u32 = 12 * 3600 * 60; 128 129 #[derive(Debug, PartialEq)] 130 pub enum ImageContentType { 131 None, 132 ColorAndAlpha, 133 GainMap, 134 All, 135 } 136 137 impl ImageContentType { categories(&self) -> Vec<Category>138 pub fn categories(&self) -> Vec<Category> { 139 match self { 140 Self::None => vec![], 141 Self::ColorAndAlpha => vec![Category::Color, Category::Alpha], 142 Self::GainMap => vec![Category::Gainmap], 143 Self::All => Category::ALL.to_vec(), 144 } 145 } 146 gainmap(&self) -> bool147 pub fn gainmap(&self) -> bool { 148 matches!(self, Self::GainMap | Self::All) 149 } 150 } 151 152 #[derive(Debug)] 153 pub struct Settings { 154 pub source: Source, 155 pub ignore_exif: bool, 156 pub ignore_xmp: bool, 157 pub strictness: Strictness, 158 pub allow_progressive: bool, 159 pub allow_incremental: bool, 160 pub image_content_to_decode: ImageContentType, 161 pub codec_choice: CodecChoice, 162 pub image_size_limit: u32, 163 pub image_dimension_limit: u32, 164 pub image_count_limit: u32, 165 pub max_threads: u32, 166 pub android_mediacodec_output_color_format: AndroidMediaCodecOutputColorFormat, 167 } 168 169 impl Default for Settings { default() -> Self170 fn default() -> Self { 171 Self { 172 source: Default::default(), 173 ignore_exif: false, 174 ignore_xmp: false, 175 strictness: Default::default(), 176 allow_progressive: false, 177 allow_incremental: false, 178 image_content_to_decode: ImageContentType::ColorAndAlpha, 179 codec_choice: Default::default(), 180 image_size_limit: DEFAULT_IMAGE_SIZE_LIMIT, 181 image_dimension_limit: DEFAULT_IMAGE_DIMENSION_LIMIT, 182 image_count_limit: DEFAULT_IMAGE_COUNT_LIMIT, 183 max_threads: 1, 184 android_mediacodec_output_color_format: AndroidMediaCodecOutputColorFormat::default(), 185 } 186 } 187 } 188 189 #[derive(Clone, Copy, Debug, Default)] 190 #[repr(C)] 191 pub struct Extent { 192 pub offset: u64, 193 pub size: usize, 194 } 195 196 impl Extent { merge(&mut self, extent: &Extent) -> AvifResult<()>197 fn merge(&mut self, extent: &Extent) -> AvifResult<()> { 198 if self.size == 0 { 199 *self = *extent; 200 return Ok(()); 201 } 202 if extent.size == 0 { 203 return Ok(()); 204 } 205 let max_extent_1 = checked_add!(self.offset, u64_from_usize(self.size)?)?; 206 let max_extent_2 = checked_add!(extent.offset, u64_from_usize(extent.size)?)?; 207 self.offset = min(self.offset, extent.offset); 208 // The extents may not be contiguous. It does not matter for nth_image_max_extent(). 209 self.size = usize_from_u64(checked_sub!(max(max_extent_1, max_extent_2), self.offset)?)?; 210 Ok(()) 211 } 212 } 213 214 #[derive(Debug)] 215 pub enum StrictnessFlag { 216 PixiRequired, 217 ClapValid, 218 AlphaIspeRequired, 219 } 220 221 #[derive(Debug, Default)] 222 pub enum Strictness { 223 None, 224 #[default] 225 All, 226 SpecificInclude(Vec<StrictnessFlag>), 227 SpecificExclude(Vec<StrictnessFlag>), 228 } 229 230 impl Strictness { pixi_required(&self) -> bool231 pub fn pixi_required(&self) -> bool { 232 match self { 233 Strictness::All => true, 234 Strictness::SpecificInclude(flags) => flags 235 .iter() 236 .any(|x| matches!(x, StrictnessFlag::PixiRequired)), 237 Strictness::SpecificExclude(flags) => !flags 238 .iter() 239 .any(|x| matches!(x, StrictnessFlag::PixiRequired)), 240 _ => false, 241 } 242 } 243 alpha_ispe_required(&self) -> bool244 pub fn alpha_ispe_required(&self) -> bool { 245 match self { 246 Strictness::All => true, 247 Strictness::SpecificInclude(flags) => flags 248 .iter() 249 .any(|x| matches!(x, StrictnessFlag::AlphaIspeRequired)), 250 Strictness::SpecificExclude(flags) => !flags 251 .iter() 252 .any(|x| matches!(x, StrictnessFlag::AlphaIspeRequired)), 253 _ => false, 254 } 255 } 256 } 257 258 #[repr(C)] 259 #[derive(Clone, Copy, Debug, Default)] 260 pub enum ProgressiveState { 261 #[default] 262 Unavailable = 0, 263 Available = 1, 264 Active = 2, 265 } 266 267 #[derive(Default, PartialEq)] 268 enum ParseState { 269 #[default] 270 None, 271 AwaitingSequenceHeader, 272 Complete, 273 } 274 275 /// cbindgen:field-names=[colorOBUSize,alphaOBUSize] 276 #[repr(C)] 277 #[derive(Clone, Copy, Debug, Default)] 278 pub struct IOStats { 279 pub color_obu_size: usize, 280 pub alpha_obu_size: usize, 281 } 282 283 #[derive(Default)] 284 pub struct Decoder { 285 pub settings: Settings, 286 image_count: u32, 287 image_index: i32, 288 image_timing: ImageTiming, 289 timescale: u64, 290 duration_in_timescales: u64, 291 duration: f64, 292 repetition_count: RepetitionCount, 293 gainmap: GainMap, 294 gainmap_present: bool, 295 image: Image, 296 source: Source, 297 tile_info: [TileInfo; Category::COUNT], 298 tiles: [Vec<Tile>; Category::COUNT], 299 items: Items, 300 tracks: Vec<Track>, 301 // To replicate the C-API, we need to keep this optional. Otherwise this 302 // could be part of the initialization. 303 io: Option<GenericIO>, 304 codecs: Vec<Codec>, 305 color_track_id: Option<u32>, 306 parse_state: ParseState, 307 io_stats: IOStats, 308 compression_format: CompressionFormat, 309 } 310 311 #[repr(C)] 312 #[derive(Clone, Copy, Debug, Default, PartialEq)] 313 pub enum CompressionFormat { 314 #[default] 315 Avif = 0, 316 Heic = 1, 317 } 318 319 #[derive(Clone, Copy, Debug, Default, PartialEq)] 320 pub enum Category { 321 #[default] 322 Color, 323 Alpha, 324 Gainmap, 325 } 326 327 impl Category { 328 const COUNT: usize = 3; 329 const ALL: [Category; Category::COUNT] = [Self::Color, Self::Alpha, Self::Gainmap]; 330 const ALL_USIZE: [usize; Category::COUNT] = [0, 1, 2]; 331 usize(self) -> usize332 pub fn usize(self) -> usize { 333 match self { 334 Category::Color => 0, 335 Category::Alpha => 1, 336 Category::Gainmap => 2, 337 } 338 } 339 planes(&self) -> &[Plane]340 pub fn planes(&self) -> &[Plane] { 341 match self { 342 Category::Alpha => &A_PLANE, 343 _ => &YUV_PLANES, 344 } 345 } 346 } 347 348 macro_rules! find_property { 349 ($properties:expr, $property_name:ident) => { 350 $properties.iter().find_map(|p| match p { 351 ItemProperty::$property_name(value) => Some(value.clone()), 352 _ => None, 353 }) 354 }; 355 } 356 357 impl Decoder { image_count(&self) -> u32358 pub fn image_count(&self) -> u32 { 359 self.image_count 360 } image_index(&self) -> i32361 pub fn image_index(&self) -> i32 { 362 self.image_index 363 } image_timing(&self) -> ImageTiming364 pub fn image_timing(&self) -> ImageTiming { 365 self.image_timing 366 } timescale(&self) -> u64367 pub fn timescale(&self) -> u64 { 368 self.timescale 369 } duration_in_timescales(&self) -> u64370 pub fn duration_in_timescales(&self) -> u64 { 371 self.duration_in_timescales 372 } duration(&self) -> f64373 pub fn duration(&self) -> f64 { 374 self.duration 375 } repetition_count(&self) -> RepetitionCount376 pub fn repetition_count(&self) -> RepetitionCount { 377 self.repetition_count 378 } gainmap(&self) -> &GainMap379 pub fn gainmap(&self) -> &GainMap { 380 &self.gainmap 381 } gainmap_present(&self) -> bool382 pub fn gainmap_present(&self) -> bool { 383 self.gainmap_present 384 } io_stats(&self) -> IOStats385 pub fn io_stats(&self) -> IOStats { 386 self.io_stats 387 } compression_format(&self) -> CompressionFormat388 pub fn compression_format(&self) -> CompressionFormat { 389 self.compression_format 390 } 391 parsing_complete(&self) -> bool392 fn parsing_complete(&self) -> bool { 393 self.parse_state == ParseState::Complete 394 } 395 set_io_file(&mut self, filename: &String) -> AvifResult<()>396 pub fn set_io_file(&mut self, filename: &String) -> AvifResult<()> { 397 self.io = Some(Box::new(DecoderFileIO::create(filename)?)); 398 self.parse_state = ParseState::None; 399 Ok(()) 400 } 401 set_io_vec(&mut self, data: Vec<u8>)402 pub fn set_io_vec(&mut self, data: Vec<u8>) { 403 self.io = Some(Box::new(DecoderMemoryIO { data })); 404 self.parse_state = ParseState::None; 405 } 406 407 /// # Safety 408 /// 409 /// This function is intended for use only from the C API. The assumption is that the caller 410 /// will always pass in a valid pointer and size. set_io_raw(&mut self, data: *const u8, size: usize) -> AvifResult<()>411 pub unsafe fn set_io_raw(&mut self, data: *const u8, size: usize) -> AvifResult<()> { 412 self.io = Some(Box::new(unsafe { DecoderRawIO::create(data, size) })); 413 self.parse_state = ParseState::None; 414 Ok(()) 415 } 416 set_io(&mut self, io: GenericIO)417 pub fn set_io(&mut self, io: GenericIO) { 418 self.io = Some(io); 419 self.parse_state = ParseState::None; 420 } 421 find_alpha_item(&mut self, color_item_index: u32) -> AvifResult<Option<u32>>422 fn find_alpha_item(&mut self, color_item_index: u32) -> AvifResult<Option<u32>> { 423 let color_item = self.items.get(&color_item_index).unwrap(); 424 if let Some(item) = self.items.iter().find(|x| { 425 !x.1.should_skip() && x.1.aux_for_id == color_item.id && x.1.is_auxiliary_alpha() 426 }) { 427 return Ok(Some(*item.0)); 428 } 429 if color_item.item_type != "grid" || color_item.grid_item_ids.is_empty() { 430 return Ok(None); 431 } 432 // If color item is a grid, check if there is an alpha channel which is represented as an 433 // auxl item to each color tile item. 434 let mut alpha_item_indices: Vec<u32> = create_vec_exact(color_item.grid_item_ids.len())?; 435 for color_grid_item_id in &color_item.grid_item_ids { 436 match self 437 .items 438 .iter() 439 .find(|x| x.1.aux_for_id == *color_grid_item_id && x.1.is_auxiliary_alpha()) 440 { 441 Some(item) => alpha_item_indices.push(*item.0), 442 None => { 443 if alpha_item_indices.is_empty() { 444 return Ok(None); 445 } else { 446 return Err(AvifError::BmffParseFailed( 447 "Some tiles but not all have an alpha auxiliary image item".into(), 448 )); 449 } 450 } 451 } 452 } 453 454 // Make up an alpha item for convenience. For the item_id, choose the first id that is not 455 // found in the actual image. In the very unlikely case that all the item ids are used, 456 // treat this as an image without alpha channel. 457 let alpha_item_id = match (1..u32::MAX).find(|&id| !self.items.contains_key(&id)) { 458 Some(id) => id, 459 None => return Ok(None), 460 }; 461 let first_item = self.items.get(&alpha_item_indices[0]).unwrap(); 462 let properties = match first_item.codec_config() { 463 Some(config) => vec![ItemProperty::CodecConfiguration(config.clone())], 464 None => return Ok(None), 465 }; 466 let alpha_item = Item { 467 id: alpha_item_id, 468 item_type: String::from("grid"), 469 width: color_item.width, 470 height: color_item.height, 471 grid_item_ids: alpha_item_indices, 472 properties, 473 is_made_up: true, 474 ..Item::default() 475 }; 476 self.tile_info[Category::Alpha.usize()].grid = self.tile_info[Category::Color.usize()].grid; 477 self.items.insert(alpha_item_id, alpha_item); 478 Ok(Some(alpha_item_id)) 479 } 480 481 // returns (tone_mapped_image_item_id, gain_map_item_id) if found find_tone_mapped_image_item(&self, color_item_id: u32) -> AvifResult<Option<(u32, u32)>>482 fn find_tone_mapped_image_item(&self, color_item_id: u32) -> AvifResult<Option<(u32, u32)>> { 483 let tmap_items: Vec<_> = self.items.values().filter(|x| x.is_tmap()).collect(); 484 for item in tmap_items { 485 let dimg_items: Vec<_> = self 486 .items 487 .values() 488 .filter(|x| x.dimg_for_id == item.id) 489 .collect(); 490 if dimg_items.len() != 2 { 491 return Err(AvifError::InvalidToneMappedImage( 492 "Expected tmap to have 2 dimg items".into(), 493 )); 494 } 495 let item0 = if dimg_items[0].dimg_index == 0 { dimg_items[0] } else { dimg_items[1] }; 496 if item0.id != color_item_id { 497 continue; 498 } 499 let item1 = if dimg_items[0].dimg_index == 0 { dimg_items[1] } else { dimg_items[0] }; 500 return Ok(Some((item.id, item1.id))); 501 } 502 Ok(None) 503 } 504 505 // returns (tone_mapped_image_item_id, gain_map_item_id) if found find_gainmap_item(&self, color_item_id: u32) -> AvifResult<Option<(u32, u32)>>506 fn find_gainmap_item(&self, color_item_id: u32) -> AvifResult<Option<(u32, u32)>> { 507 if let Some((tonemap_id, gainmap_id)) = self.find_tone_mapped_image_item(color_item_id)? { 508 let gainmap_item = self 509 .items 510 .get(&gainmap_id) 511 .ok_or(AvifError::InvalidToneMappedImage("".into()))?; 512 if gainmap_item.should_skip() { 513 return Err(AvifError::InvalidToneMappedImage("".into())); 514 } 515 Ok(Some((tonemap_id, gainmap_id))) 516 } else { 517 Ok(None) 518 } 519 } 520 validate_gainmap_item(&mut self, gainmap_id: u32, tonemap_id: u32) -> AvifResult<()>521 fn validate_gainmap_item(&mut self, gainmap_id: u32, tonemap_id: u32) -> AvifResult<()> { 522 let gainmap_item = self 523 .items 524 .get(&gainmap_id) 525 .ok_or(AvifError::InvalidToneMappedImage("".into()))?; 526 // Find and adopt all colr boxes "at most one for a given value of colour type" 527 // (HEIF 6.5.5.1, from Amendment 3). Accept one of each type, and bail out if more than one 528 // of a given type is provided. 529 if let Some(nclx) = find_nclx(&gainmap_item.properties)? { 530 self.gainmap.image.color_primaries = nclx.color_primaries; 531 self.gainmap.image.transfer_characteristics = nclx.transfer_characteristics; 532 self.gainmap.image.matrix_coefficients = nclx.matrix_coefficients; 533 self.gainmap.image.yuv_range = nclx.yuv_range; 534 } 535 if tonemap_id == 0 { 536 return Ok(()); 537 } 538 // Find and adopt all colr boxes "at most one for a given value of colour type" 539 // (HEIF 6.5.5.1, from Amendment 3). Accept one of each type, and bail out if more than one 540 // of a given type is provided. 541 let tonemap_item = self 542 .items 543 .get(&tonemap_id) 544 .ok_or(AvifError::InvalidToneMappedImage("".into()))?; 545 if let Some(nclx) = find_nclx(&tonemap_item.properties)? { 546 self.gainmap.alt_color_primaries = nclx.color_primaries; 547 self.gainmap.alt_transfer_characteristics = nclx.transfer_characteristics; 548 self.gainmap.alt_matrix_coefficients = nclx.matrix_coefficients; 549 self.gainmap.alt_yuv_range = nclx.yuv_range; 550 } 551 if let Some(icc) = find_icc(&tonemap_item.properties)? { 552 self.gainmap.alt_icc.clone_from(icc); 553 } 554 if let Some(clli) = tonemap_item.clli() { 555 self.gainmap.alt_clli = *clli; 556 } 557 if let Some(pixi) = tonemap_item.pixi() { 558 self.gainmap.alt_plane_count = pixi.plane_depths.len() as u8; 559 self.gainmap.alt_plane_depth = pixi.plane_depths[0]; 560 } 561 // HEIC files created by Apple have some of these properties set in the Tonemap item. So do 562 // not perform this validation when HEIC is enabled. 563 #[cfg(not(feature = "heic"))] 564 if find_property!(tonemap_item.properties, PixelAspectRatio).is_some() 565 || find_property!(tonemap_item.properties, CleanAperture).is_some() 566 || find_property!(tonemap_item.properties, ImageRotation).is_some() 567 || find_property!(tonemap_item.properties, ImageMirror).is_some() 568 { 569 return Err(AvifError::InvalidToneMappedImage("".into())); 570 } 571 Ok(()) 572 } 573 search_exif_or_xmp_metadata( items: &mut Items, color_item_index: Option<u32>, settings: &Settings, io: &mut GenericIO, image: &mut Image, ) -> AvifResult<()>574 fn search_exif_or_xmp_metadata( 575 items: &mut Items, 576 color_item_index: Option<u32>, 577 settings: &Settings, 578 io: &mut GenericIO, 579 image: &mut Image, 580 ) -> AvifResult<()> { 581 if !settings.ignore_exif { 582 if let Some(exif) = items.iter_mut().rfind(|x| x.1.is_exif(color_item_index)) { 583 let mut stream = exif.1.stream(io)?; 584 exif::parse(&mut stream)?; 585 image 586 .exif 587 .extend_from_slice(stream.get_slice(stream.bytes_left()?)?); 588 } 589 } 590 if !settings.ignore_xmp { 591 if let Some(xmp) = items.iter_mut().rfind(|x| x.1.is_xmp(color_item_index)) { 592 let mut stream = xmp.1.stream(io)?; 593 image 594 .xmp 595 .extend_from_slice(stream.get_slice(stream.bytes_left()?)?); 596 } 597 } 598 Ok(()) 599 } 600 generate_tiles(&mut self, item_id: u32, category: Category) -> AvifResult<Vec<Tile>>601 fn generate_tiles(&mut self, item_id: u32, category: Category) -> AvifResult<Vec<Tile>> { 602 let mut tiles: Vec<Tile> = Vec::new(); 603 let item = self 604 .items 605 .get(&item_id) 606 .ok_or(AvifError::MissingImageItem)?; 607 if item.grid_item_ids.is_empty() { 608 if item.size == 0 { 609 return Err(AvifError::MissingImageItem); 610 } 611 let mut tile = Tile::create_from_item( 612 self.items.get_mut(&item_id).unwrap(), 613 self.settings.allow_progressive, 614 self.settings.image_count_limit, 615 self.io.unwrap_ref().size_hint(), 616 )?; 617 tile.input.category = category; 618 tiles.push(tile); 619 } else { 620 if !self.tile_info[category.usize()].is_grid() { 621 return Err(AvifError::InvalidImageGrid( 622 "dimg items were found but image is not grid.".into(), 623 )); 624 } 625 let mut progressive = true; 626 for grid_item_id in item.grid_item_ids.clone() { 627 let grid_item = self 628 .items 629 .get_mut(&grid_item_id) 630 .ok_or(AvifError::InvalidImageGrid("missing grid item".into()))?; 631 let mut tile = Tile::create_from_item( 632 grid_item, 633 self.settings.allow_progressive, 634 self.settings.image_count_limit, 635 self.io.unwrap_ref().size_hint(), 636 )?; 637 tile.input.category = category; 638 tiles.push(tile); 639 progressive = progressive && grid_item.progressive; 640 } 641 642 if category == Category::Color && progressive { 643 // Propagate the progressive status to the top-level grid item. 644 self.items.get_mut(&item_id).unwrap().progressive = true; 645 } 646 } 647 self.tile_info[category.usize()].tile_count = u32_from_usize(tiles.len())?; 648 Ok(tiles) 649 } 650 harvest_cicp_from_sequence_header(&mut self) -> AvifResult<()>651 fn harvest_cicp_from_sequence_header(&mut self) -> AvifResult<()> { 652 let category = Category::Color; 653 if self.tiles[category.usize()].is_empty() { 654 return Ok(()); 655 } 656 let mut search_size = 64; 657 while search_size < 4096 { 658 let tile_index = 0; 659 self.prepare_sample( 660 /*image_index=*/ 0, 661 category, 662 tile_index, 663 Some(search_size), 664 )?; 665 let io = &mut self.io.unwrap_mut(); 666 let sample = &self.tiles[category.usize()][tile_index].input.samples[0]; 667 let item_data_buffer = if sample.item_id == 0 { 668 &None 669 } else { 670 &self.items.get(&sample.item_id).unwrap().data_buffer 671 }; 672 if let Ok(sequence_header) = Av1SequenceHeader::parse_from_obus(sample.partial_data( 673 io, 674 item_data_buffer, 675 min(search_size, sample.size), 676 )?) { 677 self.image.color_primaries = sequence_header.color_primaries; 678 self.image.transfer_characteristics = sequence_header.transfer_characteristics; 679 self.image.matrix_coefficients = sequence_header.matrix_coefficients; 680 self.image.yuv_range = sequence_header.yuv_range; 681 break; 682 } 683 search_size += 64; 684 } 685 Ok(()) 686 } 687 populate_grid_item_ids(&mut self, item_id: u32, category: Category) -> AvifResult<()>688 fn populate_grid_item_ids(&mut self, item_id: u32, category: Category) -> AvifResult<()> { 689 if self.items.get(&item_id).unwrap().item_type != "grid" { 690 return Ok(()); 691 } 692 let tile_count = self.tile_info[category.usize()].grid_tile_count()? as usize; 693 let mut grid_item_ids: Vec<u32> = create_vec_exact(tile_count)?; 694 let mut first_codec_config: Option<CodecConfiguration> = None; 695 // Collect all the dimg items. 696 for dimg_item_id in self.items.keys() { 697 if *dimg_item_id == item_id { 698 continue; 699 } 700 let dimg_item = self 701 .items 702 .get(dimg_item_id) 703 .ok_or(AvifError::InvalidImageGrid("".into()))?; 704 if dimg_item.dimg_for_id != item_id { 705 continue; 706 } 707 if !dimg_item.is_image_codec_item() || dimg_item.has_unsupported_essential_property { 708 return Err(AvifError::InvalidImageGrid( 709 "invalid input item in dimg grid".into(), 710 )); 711 } 712 if first_codec_config.is_none() { 713 // Adopt the configuration property of the first tile. 714 // validate_properties() makes sure they are all equal. 715 first_codec_config = Some( 716 dimg_item 717 .codec_config() 718 .ok_or(AvifError::BmffParseFailed( 719 "missing codec config property".into(), 720 ))? 721 .clone(), 722 ); 723 } 724 if grid_item_ids.len() >= tile_count { 725 return Err(AvifError::InvalidImageGrid( 726 "Expected number of tiles not found".into(), 727 )); 728 } 729 grid_item_ids.push(*dimg_item_id); 730 } 731 if grid_item_ids.len() != tile_count { 732 return Err(AvifError::InvalidImageGrid( 733 "Expected number of tiles not found".into(), 734 )); 735 } 736 // ISO/IEC 23008-12: The input images are inserted in row-major order, 737 // top-row first, left to right, in the order of SingleItemTypeReferenceBox of type 'dimg' 738 // for this derived image item within the ItemReferenceBox. 739 // Sort the grid items by dimg_index. dimg_index is the order in which the items appear in 740 // the 'iref' box. 741 grid_item_ids.sort_by_key(|k| self.items.get(k).unwrap().dimg_index); 742 let item = self.items.get_mut(&item_id).unwrap(); 743 item.properties.push(ItemProperty::CodecConfiguration( 744 first_codec_config.unwrap(), 745 )); 746 item.grid_item_ids = grid_item_ids; 747 Ok(()) 748 } 749 reset(&mut self)750 fn reset(&mut self) { 751 let decoder = Decoder::default(); 752 // Reset all fields to default except the following: settings, io, source. 753 self.image_count = decoder.image_count; 754 self.image_timing = decoder.image_timing; 755 self.timescale = decoder.timescale; 756 self.duration_in_timescales = decoder.duration_in_timescales; 757 self.duration = decoder.duration; 758 self.repetition_count = decoder.repetition_count; 759 self.gainmap = decoder.gainmap; 760 self.gainmap_present = decoder.gainmap_present; 761 self.image = decoder.image; 762 self.tile_info = decoder.tile_info; 763 self.tiles = decoder.tiles; 764 self.image_index = decoder.image_index; 765 self.items = decoder.items; 766 self.tracks = decoder.tracks; 767 self.codecs = decoder.codecs; 768 self.color_track_id = decoder.color_track_id; 769 self.parse_state = decoder.parse_state; 770 self.compression_format = decoder.compression_format; 771 } 772 parse(&mut self) -> AvifResult<()>773 pub fn parse(&mut self) -> AvifResult<()> { 774 if self.parsing_complete() { 775 // Parse was called again. Reset the data and start over. 776 self.parse_state = ParseState::None; 777 } 778 if self.io.is_none() { 779 return Err(AvifError::IoNotSet); 780 } 781 782 if self.parse_state == ParseState::None { 783 self.reset(); 784 let avif_boxes = mp4box::parse(self.io.unwrap_mut())?; 785 self.tracks = avif_boxes.tracks; 786 if !self.tracks.is_empty() { 787 self.image.image_sequence_track_present = true; 788 for track in &self.tracks { 789 if !track.check_limits( 790 self.settings.image_size_limit, 791 self.settings.image_dimension_limit, 792 ) { 793 return Err(AvifError::BmffParseFailed( 794 "track dimension too large".into(), 795 )); 796 } 797 } 798 } 799 self.items = construct_items(&avif_boxes.meta)?; 800 if avif_boxes.ftyp.has_tmap() && !self.items.values().any(|x| x.item_type == "tmap") { 801 return Err(AvifError::BmffParseFailed( 802 "tmap was required but not found".into(), 803 )); 804 } 805 for item in self.items.values_mut() { 806 item.harvest_ispe( 807 self.settings.strictness.alpha_ispe_required(), 808 self.settings.image_size_limit, 809 self.settings.image_dimension_limit, 810 )?; 811 } 812 813 self.source = match self.settings.source { 814 // Decide the source based on the major brand. 815 Source::Auto => match avif_boxes.ftyp.major_brand.as_str() { 816 "avis" => Source::Tracks, 817 "avif" => Source::PrimaryItem, 818 _ => { 819 if self.tracks.is_empty() { 820 Source::PrimaryItem 821 } else { 822 Source::Tracks 823 } 824 } 825 }, 826 Source::Tracks => Source::Tracks, 827 Source::PrimaryItem => Source::PrimaryItem, 828 }; 829 830 let color_properties: &Vec<ItemProperty>; 831 let gainmap_properties: Option<&Vec<ItemProperty>>; 832 if self.source == Source::Tracks { 833 let color_track = self 834 .tracks 835 .iter() 836 .find(|x| x.is_color()) 837 .ok_or(AvifError::NoContent)?; 838 if let Some(meta) = &color_track.meta { 839 let mut color_track_items = construct_items(meta)?; 840 Self::search_exif_or_xmp_metadata( 841 &mut color_track_items, 842 None, 843 &self.settings, 844 self.io.unwrap_mut(), 845 &mut self.image, 846 )?; 847 } 848 self.color_track_id = Some(color_track.id); 849 color_properties = color_track 850 .get_properties() 851 .ok_or(AvifError::BmffParseFailed("".into()))?; 852 gainmap_properties = None; 853 854 self.tiles[Category::Color.usize()].push(Tile::create_from_track( 855 color_track, 856 self.settings.image_count_limit, 857 self.io.unwrap_ref().size_hint(), 858 Category::Color, 859 )?); 860 self.tile_info[Category::Color.usize()].tile_count = 1; 861 862 if let Some(alpha_track) = self.tracks.iter().find(|x| x.is_aux(color_track.id)) { 863 self.tiles[Category::Alpha.usize()].push(Tile::create_from_track( 864 alpha_track, 865 self.settings.image_count_limit, 866 self.io.unwrap_ref().size_hint(), 867 Category::Alpha, 868 )?); 869 self.tile_info[Category::Alpha.usize()].tile_count = 1; 870 self.image.alpha_present = true; 871 self.image.alpha_premultiplied = color_track.prem_by_id == Some(alpha_track.id); 872 } 873 874 self.image_index = -1; 875 self.image_count = 876 self.tiles[Category::Color.usize()][0].input.samples.len() as u32; 877 self.timescale = color_track.media_timescale as u64; 878 self.duration_in_timescales = color_track.media_duration; 879 if self.timescale != 0 { 880 self.duration = (self.duration_in_timescales as f64) / (self.timescale as f64); 881 } else { 882 self.duration = 0.0; 883 } 884 self.repetition_count = color_track.repetition_count()?; 885 self.image_timing = Default::default(); 886 887 self.image.width = color_track.width; 888 self.image.height = color_track.height; 889 } else { 890 assert_eq!(self.source, Source::PrimaryItem); 891 let mut item_ids: [u32; Category::COUNT] = [0; Category::COUNT]; 892 893 // Mandatory color item (primary item). 894 let color_item_id = self 895 .items 896 .iter() 897 .find(|x| { 898 !x.1.should_skip() 899 && x.1.id != 0 900 && x.1.id == avif_boxes.meta.primary_item_id 901 }) 902 .map(|it| *it.0); 903 904 item_ids[Category::Color.usize()] = color_item_id.ok_or(AvifError::NoContent)?; 905 self.read_and_parse_item(item_ids[Category::Color.usize()], Category::Color)?; 906 self.populate_grid_item_ids(item_ids[Category::Color.usize()], Category::Color)?; 907 908 // Find exif/xmp from meta if any. 909 Self::search_exif_or_xmp_metadata( 910 &mut self.items, 911 Some(item_ids[Category::Color.usize()]), 912 &self.settings, 913 self.io.unwrap_mut(), 914 &mut self.image, 915 )?; 916 917 // Optional alpha auxiliary item 918 if let Some(alpha_item_id) = 919 self.find_alpha_item(item_ids[Category::Color.usize()])? 920 { 921 if !self.items.get(&alpha_item_id).unwrap().is_made_up { 922 self.read_and_parse_item(alpha_item_id, Category::Alpha)?; 923 self.populate_grid_item_ids(alpha_item_id, Category::Alpha)?; 924 } 925 item_ids[Category::Alpha.usize()] = alpha_item_id; 926 } 927 928 // Optional gainmap item 929 if avif_boxes.ftyp.has_tmap() { 930 if let Some((tonemap_id, gainmap_id)) = 931 self.find_gainmap_item(item_ids[Category::Color.usize()])? 932 { 933 let tonemap_item = self 934 .items 935 .get_mut(&tonemap_id) 936 .ok_or(AvifError::InvalidToneMappedImage("".into()))?; 937 let mut stream = tonemap_item.stream(self.io.unwrap_mut())?; 938 if let Some(metadata) = mp4box::parse_tmap(&mut stream)? { 939 self.gainmap.metadata = metadata; 940 self.read_and_parse_item(gainmap_id, Category::Gainmap)?; 941 self.populate_grid_item_ids(gainmap_id, Category::Gainmap)?; 942 self.validate_gainmap_item(gainmap_id, tonemap_id)?; 943 self.gainmap_present = true; 944 if self.settings.image_content_to_decode.gainmap() { 945 item_ids[Category::Gainmap.usize()] = gainmap_id; 946 } 947 } 948 } 949 } 950 951 self.image_index = -1; 952 self.image_count = 1; 953 self.timescale = 1; 954 self.duration = 1.0; 955 self.duration_in_timescales = 1; 956 self.image_timing.timescale = 1; 957 self.image_timing.duration = 1.0; 958 self.image_timing.duration_in_timescales = 1; 959 960 for category in Category::ALL { 961 let item_id = item_ids[category.usize()]; 962 if item_id == 0 { 963 continue; 964 } 965 966 let item = self.items.get(&item_id).unwrap(); 967 if category == Category::Alpha && item.width == 0 && item.height == 0 { 968 // NON-STANDARD: Alpha subimage does not have an ispe property; adopt 969 // width/height from color item. 970 assert!(!self.settings.strictness.alpha_ispe_required()); 971 let color_item = 972 self.items.get(&item_ids[Category::Color.usize()]).unwrap(); 973 let width = color_item.width; 974 let height = color_item.height; 975 let alpha_item = self.items.get_mut(&item_id).unwrap(); 976 // Note: We cannot directly use color_item.width here because borrow 977 // checker won't allow that. 978 alpha_item.width = width; 979 alpha_item.height = height; 980 } 981 982 self.tiles[category.usize()] = self.generate_tiles(item_id, category)?; 983 let item = self.items.get(&item_id).unwrap(); 984 // Made up alpha item does not contain the pixi property. So do not try to 985 // validate it. 986 let pixi_required = 987 self.settings.strictness.pixi_required() && !item.is_made_up; 988 item.validate_properties(&self.items, pixi_required)?; 989 } 990 991 let color_item = self.items.get(&item_ids[Category::Color.usize()]).unwrap(); 992 self.image.width = color_item.width; 993 self.image.height = color_item.height; 994 self.image.alpha_present = item_ids[Category::Alpha.usize()] != 0; 995 // alphapremultiplied. 996 997 if color_item.progressive { 998 self.image.progressive_state = ProgressiveState::Available; 999 let sample_count = self.tiles[Category::Color.usize()][0].input.samples.len(); 1000 if sample_count > 1 { 1001 self.image.progressive_state = ProgressiveState::Active; 1002 self.image_count = sample_count as u32; 1003 } 1004 } 1005 1006 if item_ids[Category::Gainmap.usize()] != 0 { 1007 let gainmap_item = self 1008 .items 1009 .get(&item_ids[Category::Gainmap.usize()]) 1010 .unwrap(); 1011 self.gainmap.image.width = gainmap_item.width; 1012 self.gainmap.image.height = gainmap_item.height; 1013 let codec_config = gainmap_item 1014 .codec_config() 1015 .ok_or(AvifError::BmffParseFailed("".into()))?; 1016 self.gainmap.image.depth = codec_config.depth(); 1017 self.gainmap.image.yuv_format = codec_config.pixel_format(); 1018 self.gainmap.image.chroma_sample_position = 1019 codec_config.chroma_sample_position(); 1020 } 1021 1022 // This borrow has to be in the end of this branch. 1023 color_properties = &self 1024 .items 1025 .get(&item_ids[Category::Color.usize()]) 1026 .unwrap() 1027 .properties; 1028 gainmap_properties = if item_ids[Category::Gainmap.usize()] != 0 { 1029 Some( 1030 &self 1031 .items 1032 .get(&item_ids[Category::Gainmap.usize()]) 1033 .unwrap() 1034 .properties, 1035 ) 1036 } else { 1037 None 1038 }; 1039 } 1040 1041 // Check validity of samples. 1042 for tiles in &self.tiles { 1043 for tile in tiles { 1044 for sample in &tile.input.samples { 1045 if sample.size == 0 { 1046 return Err(AvifError::BmffParseFailed( 1047 "sample has invalid size.".into(), 1048 )); 1049 } 1050 match tile.input.category { 1051 Category::Color => { 1052 checked_incr!(self.io_stats.color_obu_size, sample.size) 1053 } 1054 Category::Alpha => { 1055 checked_incr!(self.io_stats.alpha_obu_size, sample.size) 1056 } 1057 _ => {} 1058 } 1059 } 1060 } 1061 } 1062 1063 // Find and adopt all colr boxes "at most one for a given value of colour type" 1064 // (HEIF 6.5.5.1, from Amendment 3) Accept one of each type, and bail out if more than one 1065 // of a given type is provided. 1066 let mut cicp_set = false; 1067 1068 if let Some(nclx) = find_nclx(color_properties)? { 1069 self.image.color_primaries = nclx.color_primaries; 1070 self.image.transfer_characteristics = nclx.transfer_characteristics; 1071 self.image.matrix_coefficients = nclx.matrix_coefficients; 1072 self.image.yuv_range = nclx.yuv_range; 1073 cicp_set = true; 1074 } 1075 if let Some(icc) = find_icc(color_properties)? { 1076 self.image.icc.clone_from(icc); 1077 } 1078 1079 self.image.clli = find_property!(color_properties, ContentLightLevelInformation); 1080 self.image.pasp = find_property!(color_properties, PixelAspectRatio); 1081 self.image.clap = find_property!(color_properties, CleanAperture); 1082 self.image.irot_angle = find_property!(color_properties, ImageRotation); 1083 self.image.imir_axis = find_property!(color_properties, ImageMirror); 1084 1085 if let Some(gainmap_properties) = gainmap_properties { 1086 // Ensure that the bitstream contains the same 'pasp', 'clap', 'irot and 'imir' 1087 // properties for both the base and gain map image items. 1088 if self.image.pasp != find_property!(gainmap_properties, PixelAspectRatio) 1089 || self.image.clap != find_property!(gainmap_properties, CleanAperture) 1090 || self.image.irot_angle != find_property!(gainmap_properties, ImageRotation) 1091 || self.image.imir_axis != find_property!(gainmap_properties, ImageMirror) 1092 { 1093 return Err(AvifError::DecodeGainMapFailed); 1094 } 1095 } 1096 1097 let codec_config = find_property!(color_properties, CodecConfiguration) 1098 .ok_or(AvifError::BmffParseFailed("".into()))?; 1099 self.image.depth = codec_config.depth(); 1100 self.image.yuv_format = codec_config.pixel_format(); 1101 self.image.chroma_sample_position = codec_config.chroma_sample_position(); 1102 self.compression_format = if codec_config.is_avif() { 1103 CompressionFormat::Avif 1104 } else { 1105 CompressionFormat::Heic 1106 }; 1107 1108 if cicp_set { 1109 self.parse_state = ParseState::Complete; 1110 return Ok(()); 1111 } 1112 self.parse_state = ParseState::AwaitingSequenceHeader; 1113 } 1114 1115 // If cicp was not set, try to harvest it from the sequence header. 1116 self.harvest_cicp_from_sequence_header()?; 1117 self.parse_state = ParseState::Complete; 1118 1119 Ok(()) 1120 } 1121 read_and_parse_item(&mut self, item_id: u32, category: Category) -> AvifResult<()>1122 fn read_and_parse_item(&mut self, item_id: u32, category: Category) -> AvifResult<()> { 1123 if item_id == 0 { 1124 return Ok(()); 1125 } 1126 self.items.get_mut(&item_id).unwrap().read_and_parse( 1127 self.io.unwrap_mut(), 1128 &mut self.tile_info[category.usize()].grid, 1129 self.settings.image_size_limit, 1130 self.settings.image_dimension_limit, 1131 ) 1132 } 1133 can_use_single_codec(&self) -> AvifResult<bool>1134 fn can_use_single_codec(&self) -> AvifResult<bool> { 1135 let total_tile_count = checked_add!( 1136 checked_add!(self.tiles[0].len(), self.tiles[1].len())?, 1137 self.tiles[2].len() 1138 )?; 1139 if total_tile_count == 1 { 1140 return Ok(true); 1141 } 1142 if self.image_count != 1 { 1143 return Ok(false); 1144 } 1145 let mut image_buffers = 0; 1146 let mut stolen_image_buffers = 0; 1147 for category in Category::ALL_USIZE { 1148 if self.tile_info[category].tile_count > 0 { 1149 image_buffers += 1; 1150 } 1151 if self.tile_info[category].tile_count == 1 { 1152 stolen_image_buffers += 1; 1153 } 1154 } 1155 if stolen_image_buffers > 0 && image_buffers > 1 { 1156 // Stealing will cause problems. So we need separate codec instances. 1157 return Ok(false); 1158 } 1159 let operating_point = self.tiles[0][0].operating_point; 1160 let all_layers = self.tiles[0][0].input.all_layers; 1161 for tiles in &self.tiles { 1162 for tile in tiles { 1163 if tile.operating_point != operating_point || tile.input.all_layers != all_layers { 1164 return Ok(false); 1165 } 1166 } 1167 } 1168 Ok(true) 1169 } 1170 create_codec(&mut self, category: Category, tile_index: usize) -> AvifResult<()>1171 fn create_codec(&mut self, category: Category, tile_index: usize) -> AvifResult<()> { 1172 let tile = &self.tiles[category.usize()][tile_index]; 1173 let mut codec: Codec = self 1174 .settings 1175 .codec_choice 1176 .get_codec(tile.codec_config.is_avif())?; 1177 let config = DecoderConfig { 1178 operating_point: tile.operating_point, 1179 all_layers: tile.input.all_layers, 1180 width: tile.width, 1181 height: tile.height, 1182 depth: self.image.depth, 1183 max_threads: self.settings.max_threads, 1184 max_input_size: tile.max_sample_size(), 1185 codec_config: tile.codec_config.clone(), 1186 category, 1187 android_mediacodec_output_color_format: self 1188 .settings 1189 .android_mediacodec_output_color_format, 1190 }; 1191 codec.initialize(&config)?; 1192 self.codecs.push(codec); 1193 Ok(()) 1194 } 1195 create_codecs(&mut self) -> AvifResult<()>1196 fn create_codecs(&mut self) -> AvifResult<()> { 1197 if !self.codecs.is_empty() { 1198 return Ok(()); 1199 } 1200 if matches!(self.source, Source::Tracks) || cfg!(feature = "android_mediacodec") { 1201 // In this case, there are two possibilities in the following order: 1202 // 1) If source is Tracks, then we will use at most two codec instances (one each for 1203 // Color and Alpha). Gainmap will always be empty. 1204 // 2) If android_mediacodec is true, then we will use at most three codec instances 1205 // (one for each category). 1206 self.codecs = create_vec_exact(3)?; 1207 for category in self.settings.image_content_to_decode.categories() { 1208 if self.tiles[category.usize()].is_empty() { 1209 continue; 1210 } 1211 self.create_codec(category, 0)?; 1212 for tile in &mut self.tiles[category.usize()] { 1213 tile.codec_index = self.codecs.len() - 1; 1214 } 1215 } 1216 } else if self.can_use_single_codec()? { 1217 self.codecs = create_vec_exact(1)?; 1218 self.create_codec(Category::Color, 0)?; 1219 for tiles in &mut self.tiles { 1220 for tile in tiles { 1221 tile.codec_index = 0; 1222 } 1223 } 1224 } else { 1225 self.codecs = create_vec_exact(self.tiles.iter().map(|tiles| tiles.len()).sum())?; 1226 for category in self.settings.image_content_to_decode.categories() { 1227 for tile_index in 0..self.tiles[category.usize()].len() { 1228 self.create_codec(category, tile_index)?; 1229 self.tiles[category.usize()][tile_index].codec_index = self.codecs.len() - 1; 1230 } 1231 } 1232 } 1233 Ok(()) 1234 } 1235 prepare_sample( &mut self, image_index: usize, category: Category, tile_index: usize, max_num_bytes: Option<usize>, ) -> AvifResult<()>1236 fn prepare_sample( 1237 &mut self, 1238 image_index: usize, 1239 category: Category, 1240 tile_index: usize, 1241 max_num_bytes: Option<usize>, // Bytes read past that size will be ignored. 1242 ) -> AvifResult<()> { 1243 let tile = &mut self.tiles[category.usize()][tile_index]; 1244 if tile.input.samples.len() <= image_index { 1245 return Err(AvifError::NoImagesRemaining); 1246 } 1247 let sample = &tile.input.samples[image_index]; 1248 if sample.item_id == 0 { 1249 // Data comes from a track. Nothing to prepare. 1250 return Ok(()); 1251 } 1252 // Data comes from an item. 1253 let item = self 1254 .items 1255 .get_mut(&sample.item_id) 1256 .ok_or(AvifError::BmffParseFailed("".into()))?; 1257 if item.extents.len() == 1 { 1258 // Item has only one extent. Nothing to prepare. 1259 return Ok(()); 1260 } 1261 if let Some(data) = &item.data_buffer { 1262 if data.len() == item.size { 1263 return Ok(()); // All extents have already been merged. 1264 } 1265 if max_num_bytes.is_some_and(|max_num_bytes| data.len() >= max_num_bytes) { 1266 return Ok(()); // Some sufficient extents have already been merged. 1267 } 1268 } 1269 // Item has multiple extents, merge them into a contiguous buffer. 1270 if item.data_buffer.is_none() { 1271 item.data_buffer = Some(create_vec_exact(item.size)?); 1272 } 1273 let data = item.data_buffer.unwrap_mut(); 1274 let mut bytes_to_skip = data.len(); // These extents were already merged. 1275 for extent in &item.extents { 1276 if bytes_to_skip != 0 { 1277 checked_decr!(bytes_to_skip, extent.size); 1278 continue; 1279 } 1280 let io = self.io.unwrap_mut(); 1281 data.extend_from_slice(io.read_exact(extent.offset, extent.size)?); 1282 if max_num_bytes.is_some_and(|max_num_bytes| data.len() >= max_num_bytes) { 1283 return Ok(()); // There are enough merged extents to satisfy max_num_bytes. 1284 } 1285 } 1286 assert_eq!(bytes_to_skip, 0); 1287 assert_eq!(data.len(), item.size); 1288 Ok(()) 1289 } 1290 prepare_samples(&mut self, image_index: usize) -> AvifResult<()>1291 fn prepare_samples(&mut self, image_index: usize) -> AvifResult<()> { 1292 for category in self.settings.image_content_to_decode.categories() { 1293 for tile_index in 0..self.tiles[category.usize()].len() { 1294 self.prepare_sample(image_index, category, tile_index, None)?; 1295 } 1296 } 1297 Ok(()) 1298 } 1299 validate_grid_image_dimensions(image: &Image, grid: &Grid) -> AvifResult<()>1300 fn validate_grid_image_dimensions(image: &Image, grid: &Grid) -> AvifResult<()> { 1301 if checked_mul!(image.width, grid.columns)? < grid.width 1302 || checked_mul!(image.height, grid.rows)? < grid.height 1303 { 1304 return Err(AvifError::InvalidImageGrid( 1305 "Grid image tiles do not completely cover the image (HEIF (ISO/IEC 23008-12:2017), Section 6.6.2.3.1)".into(), 1306 )); 1307 } 1308 if checked_mul!(image.width, grid.columns)? < grid.width 1309 || checked_mul!(image.height, grid.rows)? < grid.height 1310 { 1311 return Err(AvifError::InvalidImageGrid( 1312 "Grid image tiles do not completely cover the image (HEIF (ISO/IEC 23008-12:2017), \ 1313 Section 6.6.2.3.1)" 1314 .into(), 1315 )); 1316 } 1317 if checked_mul!(image.width, grid.columns - 1)? >= grid.width 1318 || checked_mul!(image.height, grid.rows - 1)? >= grid.height 1319 { 1320 return Err(AvifError::InvalidImageGrid( 1321 "Grid image tiles in the rightmost column and bottommost row do not overlap the \ 1322 reconstructed image grid canvas. See MIAF (ISO/IEC 23000-22:2019), Section \ 1323 7.3.11.4.2, Figure 2" 1324 .into(), 1325 )); 1326 } 1327 // ISO/IEC 23000-22:2019, Section 7.3.11.4.2: 1328 // - the tile_width shall be greater than or equal to 64, and should be a multiple of 64 1329 // - the tile_height shall be greater than or equal to 64, and should be a multiple of 64 1330 // The "should" part is ignored here. 1331 if image.width < 64 || image.height < 64 { 1332 return Err(AvifError::InvalidImageGrid(format!( 1333 "Grid image tile width ({}) or height ({}) cannot be smaller than 64. See MIAF \ 1334 (ISO/IEC 23000-22:2019), Section 7.3.11.4.2", 1335 image.width, image.height 1336 ))); 1337 } 1338 // ISO/IEC 23000-22:2019, Section 7.3.11.4.2: 1339 // - when the images are in the 4:2:2 chroma sampling format the horizontal tile offsets 1340 // and widths, and the output width, shall be even numbers; 1341 // - when the images are in the 4:2:0 chroma sampling format both the horizontal and 1342 // vertical tile offsets and widths, and the output width and height, shall be even 1343 // numbers. 1344 if ((image.yuv_format == PixelFormat::Yuv420 || image.yuv_format == PixelFormat::Yuv422) 1345 && (grid.width % 2 != 0 || image.width % 2 != 0)) 1346 || (image.yuv_format == PixelFormat::Yuv420 1347 && (grid.height % 2 != 0 || image.height % 2 != 0)) 1348 { 1349 return Err(AvifError::InvalidImageGrid(format!( 1350 "Grid image width ({}) or height ({}) or tile width ({}) or height ({}) shall be \ 1351 even if chroma is subsampled in that dimension. See MIAF \ 1352 (ISO/IEC 23000-22:2019), Section 7.3.11.4.2", 1353 grid.width, grid.height, image.width, image.height 1354 ))); 1355 } 1356 Ok(()) 1357 } 1358 decode_tile( &mut self, image_index: usize, category: Category, tile_index: usize, ) -> AvifResult<()>1359 fn decode_tile( 1360 &mut self, 1361 image_index: usize, 1362 category: Category, 1363 tile_index: usize, 1364 ) -> AvifResult<()> { 1365 // Split the tiles array into two mutable arrays so that we can validate the 1366 // properties of tiles with index > 0 with that of the first tile. 1367 let (tiles_slice1, tiles_slice2) = self.tiles[category.usize()].split_at_mut(tile_index); 1368 let tile = &mut tiles_slice2[0]; 1369 let sample = &tile.input.samples[image_index]; 1370 let io = &mut self.io.unwrap_mut(); 1371 1372 let codec = &mut self.codecs[tile.codec_index]; 1373 let item_data_buffer = if sample.item_id == 0 { 1374 &None 1375 } else { 1376 &self.items.get(&sample.item_id).unwrap().data_buffer 1377 }; 1378 let data = sample.data(io, item_data_buffer)?; 1379 codec.get_next_image(data, sample.spatial_id, &mut tile.image, category)?; 1380 checked_incr!(self.tile_info[category.usize()].decoded_tile_count, 1); 1381 1382 if category == Category::Alpha && tile.image.yuv_range == YuvRange::Limited { 1383 tile.image.alpha_to_full_range()?; 1384 } 1385 tile.image.scale(tile.width, tile.height, category)?; 1386 1387 if self.tile_info[category.usize()].is_grid() { 1388 if tile_index == 0 { 1389 let grid = &self.tile_info[category.usize()].grid; 1390 Self::validate_grid_image_dimensions(&tile.image, grid)?; 1391 match category { 1392 Category::Color => { 1393 self.image.width = grid.width; 1394 self.image.height = grid.height; 1395 // Adopt the yuv_format and depth. 1396 self.image.yuv_format = tile.image.yuv_format; 1397 self.image.depth = tile.image.depth; 1398 self.image.allocate_planes(category)?; 1399 } 1400 Category::Alpha => { 1401 // Alpha is always just one plane and the depth has been validated 1402 // to be the same as the color planes' depth. 1403 self.image.allocate_planes(category)?; 1404 } 1405 Category::Gainmap => { 1406 self.gainmap.image.width = grid.width; 1407 self.gainmap.image.height = grid.height; 1408 // Adopt the yuv_format and depth. 1409 self.gainmap.image.yuv_format = tile.image.yuv_format; 1410 self.gainmap.image.depth = tile.image.depth; 1411 self.gainmap.image.allocate_planes(category)?; 1412 } 1413 } 1414 } 1415 if !tiles_slice1.is_empty() { 1416 let first_tile_image = &tiles_slice1[0].image; 1417 if tile.image.width != first_tile_image.width 1418 || tile.image.height != first_tile_image.height 1419 || tile.image.depth != first_tile_image.depth 1420 || tile.image.yuv_format != first_tile_image.yuv_format 1421 || tile.image.yuv_range != first_tile_image.yuv_range 1422 || tile.image.color_primaries != first_tile_image.color_primaries 1423 || tile.image.transfer_characteristics 1424 != first_tile_image.transfer_characteristics 1425 || tile.image.matrix_coefficients != first_tile_image.matrix_coefficients 1426 { 1427 return Err(AvifError::InvalidImageGrid( 1428 "grid image contains mismatched tiles".into(), 1429 )); 1430 } 1431 } 1432 match category { 1433 Category::Gainmap => self.gainmap.image.copy_from_tile( 1434 &tile.image, 1435 &self.tile_info[category.usize()], 1436 tile_index as u32, 1437 category, 1438 )?, 1439 _ => { 1440 self.image.copy_from_tile( 1441 &tile.image, 1442 &self.tile_info[category.usize()], 1443 tile_index as u32, 1444 category, 1445 )?; 1446 } 1447 } 1448 } else { 1449 // Non grid path, steal or copy planes from the only tile. 1450 match category { 1451 Category::Color => { 1452 self.image.width = tile.image.width; 1453 self.image.height = tile.image.height; 1454 self.image.depth = tile.image.depth; 1455 self.image.yuv_format = tile.image.yuv_format; 1456 self.image.steal_or_copy_from(&tile.image, category)?; 1457 } 1458 Category::Alpha => { 1459 if !self.image.has_same_properties(&tile.image) { 1460 return Err(AvifError::DecodeAlphaFailed); 1461 } 1462 self.image.steal_or_copy_from(&tile.image, category)?; 1463 } 1464 Category::Gainmap => { 1465 self.gainmap.image.width = tile.image.width; 1466 self.gainmap.image.height = tile.image.height; 1467 self.gainmap.image.depth = tile.image.depth; 1468 self.gainmap.image.yuv_format = tile.image.yuv_format; 1469 self.gainmap 1470 .image 1471 .steal_or_copy_from(&tile.image, category)?; 1472 } 1473 } 1474 } 1475 Ok(()) 1476 } 1477 decode_tiles(&mut self, image_index: usize) -> AvifResult<()>1478 fn decode_tiles(&mut self, image_index: usize) -> AvifResult<()> { 1479 let mut decoded_something = false; 1480 for category in self.settings.image_content_to_decode.categories() { 1481 let previous_decoded_tile_count = 1482 self.tile_info[category.usize()].decoded_tile_count as usize; 1483 let tile_count = self.tiles[category.usize()].len(); 1484 for tile_index in previous_decoded_tile_count..tile_count { 1485 self.decode_tile(image_index, category, tile_index)?; 1486 decoded_something = true; 1487 } 1488 } 1489 if decoded_something { 1490 Ok(()) 1491 } else { 1492 Err(AvifError::NoContent) 1493 } 1494 } 1495 next_image(&mut self) -> AvifResult<()>1496 pub fn next_image(&mut self) -> AvifResult<()> { 1497 if self.io.is_none() { 1498 return Err(AvifError::IoNotSet); 1499 } 1500 if !self.parsing_complete() { 1501 return Err(AvifError::NoContent); 1502 } 1503 if self.is_current_frame_fully_decoded() { 1504 for category in Category::ALL_USIZE { 1505 self.tile_info[category].decoded_tile_count = 0; 1506 } 1507 } 1508 1509 let next_image_index = checked_add!(self.image_index, 1)?; 1510 self.create_codecs()?; 1511 self.prepare_samples(next_image_index as usize)?; 1512 self.decode_tiles(next_image_index as usize)?; 1513 self.image_index = next_image_index; 1514 self.image_timing = self.nth_image_timing(self.image_index as u32)?; 1515 Ok(()) 1516 } 1517 is_current_frame_fully_decoded(&self) -> bool1518 fn is_current_frame_fully_decoded(&self) -> bool { 1519 if !self.parsing_complete() { 1520 return false; 1521 } 1522 for category in self.settings.image_content_to_decode.categories() { 1523 if !self.tile_info[category.usize()].is_fully_decoded() { 1524 return false; 1525 } 1526 } 1527 true 1528 } 1529 nth_image(&mut self, index: u32) -> AvifResult<()>1530 pub fn nth_image(&mut self, index: u32) -> AvifResult<()> { 1531 if !self.parsing_complete() { 1532 return Err(AvifError::NoContent); 1533 } 1534 if index >= self.image_count { 1535 return Err(AvifError::NoImagesRemaining); 1536 } 1537 let requested_index = i32_from_u32(index)?; 1538 if requested_index == checked_add!(self.image_index, 1)? { 1539 return self.next_image(); 1540 } 1541 if requested_index == self.image_index && self.is_current_frame_fully_decoded() { 1542 // Current frame which is already fully decoded has been requested. Do nothing. 1543 return Ok(()); 1544 } 1545 let nearest_keyframe = i32_from_u32(self.nearest_keyframe(index))?; 1546 if nearest_keyframe > checked_add!(self.image_index, 1)? 1547 || requested_index <= self.image_index 1548 { 1549 // Start decoding from the nearest keyframe. 1550 self.image_index = nearest_keyframe - 1; 1551 } 1552 loop { 1553 self.next_image()?; 1554 if requested_index == self.image_index { 1555 break; 1556 } 1557 } 1558 Ok(()) 1559 } 1560 image(&self) -> Option<&Image>1561 pub fn image(&self) -> Option<&Image> { 1562 if self.parsing_complete() { 1563 Some(&self.image) 1564 } else { 1565 None 1566 } 1567 } 1568 nth_image_timing(&self, n: u32) -> AvifResult<ImageTiming>1569 pub fn nth_image_timing(&self, n: u32) -> AvifResult<ImageTiming> { 1570 if !self.parsing_complete() { 1571 return Err(AvifError::NoContent); 1572 } 1573 if n > self.settings.image_count_limit { 1574 return Err(AvifError::NoImagesRemaining); 1575 } 1576 if self.color_track_id.is_none() { 1577 return Ok(self.image_timing); 1578 } 1579 let color_track_id = self.color_track_id.unwrap(); 1580 let color_track = self 1581 .tracks 1582 .iter() 1583 .find(|x| x.id == color_track_id) 1584 .ok_or(AvifError::NoContent)?; 1585 color_track.image_timing(n) 1586 } 1587 1588 // When next_image() or nth_image() returns AvifResult::WaitingOnIo, this function can be called 1589 // next to retrieve the number of top rows that can be immediately accessed from the luma plane 1590 // of decoder->image, and alpha if any. The corresponding rows from the chroma planes, 1591 // if any, can also be accessed (half rounded up if subsampled, same number of rows otherwise). 1592 // If a gain map is present, and image_content_to_decode contains ImageContentType::GainMap, 1593 // the gain map's planes can also be accessed in the same way. 1594 // The number of available gain map rows is at least: 1595 // decoder.decoded_row_count() * decoder.gainmap.image.height / decoder.image.height 1596 // When gain map scaling is needed, callers might choose to use a few less rows depending on how 1597 // many rows are needed by the scaling algorithm, to avoid the last row(s) changing when more 1598 // data becomes available. allow_incremental must be set to true before calling next_image() or 1599 // nth_image(). Returns decoder.image.height when the last call to next_image() or nth_image() 1600 // returned AvifResult::Ok. Returns 0 in all other cases. decoded_row_count(&self) -> u321601 pub fn decoded_row_count(&self) -> u32 { 1602 let mut min_row_count = self.image.height; 1603 for category in Category::ALL_USIZE { 1604 if self.tiles[category].is_empty() { 1605 continue; 1606 } 1607 let first_tile_height = self.tiles[category][0].height; 1608 let row_count = if category == Category::Gainmap.usize() 1609 && self.gainmap_present() 1610 && self.settings.image_content_to_decode.gainmap() 1611 && self.gainmap.image.height != 0 1612 && self.gainmap.image.height != self.image.height 1613 { 1614 if self.tile_info[category].is_fully_decoded() { 1615 self.image.height 1616 } else { 1617 let gainmap_row_count = self.tile_info[category] 1618 .decoded_row_count(self.gainmap.image.height, first_tile_height); 1619 // row_count fits for sure in 32 bits because heights do. 1620 let row_count = (gainmap_row_count as u64 * self.image.height as u64 1621 / self.gainmap.image.height as u64) 1622 as u32; 1623 1624 // Make sure it satisfies the C API guarantee. 1625 assert!( 1626 gainmap_row_count 1627 >= (row_count as f32 / self.image.height as f32 1628 * self.gainmap.image.height as f32) 1629 .round() as u32 1630 ); 1631 row_count 1632 } 1633 } else { 1634 self.tile_info[category].decoded_row_count(self.image.height, first_tile_height) 1635 }; 1636 min_row_count = std::cmp::min(min_row_count, row_count); 1637 } 1638 min_row_count 1639 } 1640 is_keyframe(&self, index: u32) -> bool1641 pub fn is_keyframe(&self, index: u32) -> bool { 1642 if !self.parsing_complete() { 1643 return false; 1644 } 1645 let index = index as usize; 1646 // All the tiles for the requested index must be a keyframe. 1647 for category in Category::ALL_USIZE { 1648 for tile in &self.tiles[category] { 1649 if index >= tile.input.samples.len() || !tile.input.samples[index].sync { 1650 return false; 1651 } 1652 } 1653 } 1654 true 1655 } 1656 nearest_keyframe(&self, mut index: u32) -> u321657 pub fn nearest_keyframe(&self, mut index: u32) -> u32 { 1658 if !self.parsing_complete() { 1659 return 0; 1660 } 1661 while index != 0 { 1662 if self.is_keyframe(index) { 1663 return index; 1664 } 1665 index -= 1; 1666 } 1667 assert!(self.is_keyframe(0)); 1668 0 1669 } 1670 nth_image_max_extent(&self, index: u32) -> AvifResult<Extent>1671 pub fn nth_image_max_extent(&self, index: u32) -> AvifResult<Extent> { 1672 if !self.parsing_complete() { 1673 return Err(AvifError::NoContent); 1674 } 1675 let mut extent = Extent::default(); 1676 let start_index = self.nearest_keyframe(index) as usize; 1677 let end_index = index as usize; 1678 for current_index in start_index..=end_index { 1679 for category in Category::ALL_USIZE { 1680 for tile in &self.tiles[category] { 1681 if current_index >= tile.input.samples.len() { 1682 return Err(AvifError::NoImagesRemaining); 1683 } 1684 let sample = &tile.input.samples[current_index]; 1685 let sample_extent = if sample.item_id != 0 { 1686 let item = self.items.get(&sample.item_id).unwrap(); 1687 item.max_extent(sample)? 1688 } else { 1689 Extent { 1690 offset: sample.offset, 1691 size: sample.size, 1692 } 1693 }; 1694 extent.merge(&sample_extent)?; 1695 } 1696 } 1697 } 1698 Ok(extent) 1699 } 1700 peek_compatible_file_type(data: &[u8]) -> bool1701 pub fn peek_compatible_file_type(data: &[u8]) -> bool { 1702 mp4box::peek_compatible_file_type(data).unwrap_or(false) 1703 } 1704 } 1705 1706 #[cfg(test)] 1707 mod tests { 1708 use super::*; 1709 use test_case::test_case; 1710 1711 #[test_case(10, 20, 50, 100, 10, 140 ; "case 1")] 1712 #[test_case(100, 20, 50, 100, 50, 100 ; "case 2")] merge_extents( offset1: u64, size1: usize, offset2: u64, size2: usize, expected_offset: u64, expected_size: usize, )1713 fn merge_extents( 1714 offset1: u64, 1715 size1: usize, 1716 offset2: u64, 1717 size2: usize, 1718 expected_offset: u64, 1719 expected_size: usize, 1720 ) { 1721 let mut e1 = Extent { 1722 offset: offset1, 1723 size: size1, 1724 }; 1725 let e2 = Extent { 1726 offset: offset2, 1727 size: size2, 1728 }; 1729 assert!(e1.merge(&e2).is_ok()); 1730 assert_eq!(e1.offset, expected_offset); 1731 assert_eq!(e1.size, expected_size); 1732 } 1733 } 1734