1 // Copyright 2023 The Pigweed Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 // use this file except in compliance with the License. You may obtain a copy of 5 // the License at 6 // 7 // https://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 // License for the specific language governing permissions and limitations under 13 // the License. 14 15 //! `pw_tokenizer` - Efficient string handling and printf style encoding. 16 //! 17 //! Logging is critical, but developers are often forced to choose between 18 //! additional logging or saving crucial flash space. The `pw_tokenizer` crate 19 //! helps address this by replacing printf-style strings with binary tokens 20 //! during compilation. This enables extensive logging with substantially less 21 //! memory usage. 22 //! 23 //! For a more in depth explanation of the systems design and motivations, 24 //! see [Pigweed's pw_tokenizer module documentation](https://pigweed.dev/pw_tokenizer/). 25 //! 26 //! # Examples 27 //! 28 //! Pigweed's tokenization database uses `printf` style strings internally so 29 //! those are supported directly. 30 //! 31 //! ``` 32 //! use pw_tokenizer::tokenize_printf_to_buffer; 33 //! 34 //! let mut buffer = [0u8; 1024]; 35 //! let len = tokenize_printf_to_buffer!(&mut buffer, "The answer is %d", 42)?; 36 //! 37 //! // 4 bytes used to encode the token and one to encode the value 42. This 38 //! // is a **3.5x** reduction in size compared to the raw string! 39 //! assert_eq!(len, 5); 40 //! # Ok::<(), pw_status::Error>(()) 41 //! ``` 42 //! 43 //! We also support Rust's `core::fmt` style syntax. These format strings are 44 //! converted to `printf` style at compile time to maintain compatibly with the 45 //! rest of the Pigweed tokenizer ecosystem. The below example produces the 46 //! same token and output as the above one. 47 //! 48 //! ``` 49 //! use pw_tokenizer::tokenize_core_fmt_to_buffer; 50 //! 51 //! let mut buffer = [0u8; 1024]; 52 //! let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is {}", 42 as i32)?; 53 //! assert_eq!(len, 5); 54 //! # Ok::<(), pw_status::Error>(()) 55 //! ``` 56 57 #![cfg_attr(not(feature = "std"), no_std)] 58 #![deny(missing_docs)] 59 60 use pw_status::Result; 61 62 #[doc(hidden)] 63 pub mod internal; 64 65 #[doc(hidden)] 66 // Creating a __private namespace allows us a way to get to the modules 67 // we need from macros by doing: 68 // use $crate::__private as __pw_tokenizer_crate; 69 // 70 // This is how proc macro generated code can reliably reference back to 71 // `pw_tokenizer` while still allowing a user to import it under a different 72 // name. 73 pub mod __private { 74 pub use crate::*; 75 pub use pw_bytes::concat_static_strs; 76 pub use pw_format_core::{PrintfFormatter, PrintfHexFormatter, PrintfUpperHexFormatter}; 77 pub use pw_status::Result; 78 pub use pw_stream::{Cursor, Seek, WriteInteger, WriteVarint}; 79 pub use pw_tokenizer_core::hash_string; 80 pub use pw_tokenizer_macro::{ 81 _token, _tokenize_core_fmt_to_buffer, _tokenize_core_fmt_to_writer, 82 _tokenize_printf_to_buffer, _tokenize_printf_to_writer, 83 }; 84 } 85 86 /// Return the [`u32`] token for the specified string and add it to the token 87 /// database. 88 /// 89 /// This is where the magic happens in `pw_tokenizer`! ... and by magic 90 /// we mean hiding information in a special linker section that ends up in the 91 /// final elf binary but does not get flashed to the device. 92 /// 93 /// Two things are accomplished here: 94 /// 1) The string is hashed into its stable `u32` token. This is the value that 95 /// is returned from the macro. 96 /// 2) A [token database entry](https://pigweed.dev/pw_tokenizer/design.html#binary-database-format) 97 /// is generated, assigned to a unique static symbol, placed in a linker 98 /// section named `pw_tokenizer.entries.<TOKEN_HASH>`. A 99 /// [linker script](https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main/pw_tokenizer/pw_tokenizer_linker_sections.ld) 100 /// is responsible for picking these symbols up and aggregating them into a 101 /// single `.pw_tokenizer.entries` section in the final binary. 102 /// 103 /// # Example 104 /// ``` 105 /// use pw_tokenizer::token; 106 /// 107 /// let token = token!("hello, \"world\""); 108 /// assert_eq!(token, 3537412730); 109 /// ``` 110 /// 111 /// Currently there is no support for encoding tokens to specific domains 112 /// or with "fixed lengths" per [`pw_tokenizer_core::hash_bytes_fixed`]. 113 #[macro_export] 114 macro_rules! token { 115 ($string:literal) => {{ 116 use $crate::__private as __pw_tokenizer_crate; 117 $crate::__private::_token!($string) 118 }}; 119 } 120 121 /// Tokenize a `core::fmt` style format string and arguments to an [`AsMut<u8>`] 122 /// buffer. The format string is converted in to a `printf` and added token to 123 /// the token database. 124 /// 125 /// See [`token`] for an explanation on how strings are tokenized and entries 126 /// are added to the token database. The token's domain is set to `""`. 127 /// 128 /// Returns a [`pw_status::Result<usize>`] the number of bytes written to the buffer. 129 /// 130 /// `tokenize_to_buffer!` supports concatenation of format strings as described 131 /// in [`pw_format::macros::FormatAndArgs`]. 132 /// 133 /// # Errors 134 /// - [`pw_status::Error::OutOfRange`] - Buffer is not large enough to fit 135 /// tokenized data. 136 /// - [`pw_status::Error::InvalidArgument`] - Invalid buffer was provided. 137 /// 138 /// # Example 139 /// 140 /// ``` 141 /// use pw_tokenizer::tokenize_core_fmt_to_buffer; 142 /// 143 /// // Tokenize a format string and argument into a buffer. 144 /// let mut buffer = [0u8; 1024]; 145 /// let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is {}", 42 as i32)?; 146 /// 147 /// // 4 bytes used to encode the token and one to encode the value 42. 148 /// assert_eq!(len, 5); 149 /// 150 /// // The format string can be composed of multiple strings literals using 151 /// // the custom`PW_FMT_CONCAT` operator. 152 /// let len = tokenize_core_fmt_to_buffer!(&mut buffer, "Hello " PW_FMT_CONCAT "Pigweed")?; 153 /// 154 /// // Only a single 4 byte token is emitted after concatenation of the string 155 /// // literals above. 156 /// assert_eq!(len, 4); 157 /// # Ok::<(), pw_status::Error>(()) 158 /// ``` 159 #[macro_export] 160 macro_rules! tokenize_core_fmt_to_buffer { 161 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{ 162 use $crate::__private as __pw_tokenizer_crate; 163 __pw_tokenizer_crate::_tokenize_core_fmt_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*) 164 }}; 165 } 166 167 /// Tokenize a printf format string and arguments to an [`AsMut<u8>`] buffer 168 /// and add the format string's token to the token database. 169 /// 170 /// See [`token`] for an explanation on how strings are tokenized and entries 171 /// are added to the token database. The token's domain is set to `""`. 172 /// 173 /// Returns a [`pw_status::Result<usize>`] the number of bytes written to the buffer. 174 /// 175 /// `tokenize_to_buffer!` supports concatenation of format strings as described 176 /// in [`pw_format::macros::FormatAndArgs`]. 177 /// 178 /// # Errors 179 /// - [`pw_status::Error::OutOfRange`] - Buffer is not large enough to fit 180 /// tokenized data. 181 /// - [`pw_status::Error::InvalidArgument`] - Invalid buffer was provided. 182 /// 183 /// # Example 184 /// 185 /// ``` 186 /// use pw_tokenizer::tokenize_printf_to_buffer; 187 /// 188 /// // Tokenize a format string and argument into a buffer. 189 /// let mut buffer = [0u8; 1024]; 190 /// let len = tokenize_printf_to_buffer!(&mut buffer, "The answer is %d", 42)?; 191 /// 192 /// // 4 bytes used to encode the token and one to encode the value 42. 193 /// assert_eq!(len, 5); 194 /// 195 /// // The format string can be composed of multiple strings literals using 196 /// // the custom`PW_FMT_CONCAT` operator. 197 /// let len = tokenize_printf_to_buffer!(&mut buffer, "Hello " PW_FMT_CONCAT "Pigweed")?; 198 /// 199 /// // Only a single 4 byte token is emitted after concatenation of the string 200 /// // literals above. 201 /// assert_eq!(len, 4); 202 /// # Ok::<(), pw_status::Error>(()) 203 /// ``` 204 #[macro_export] 205 macro_rules! tokenize_printf_to_buffer { 206 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{ 207 use $crate::__private as __pw_tokenizer_crate; 208 __pw_tokenizer_crate::_tokenize_printf_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*) 209 }}; 210 } 211 212 /// Deprecated alias for [`tokenize_printf_to_buffer!`]. 213 #[macro_export] 214 macro_rules! tokenize_to_buffer { 215 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{ 216 $crate::tokenize_printf_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*) 217 }}; 218 } 219 220 /// Tokenize a `core::fmt` format string and arguments to a [`MessageWriter`]. 221 /// The format string is converted in to a `printf` and added token to the token 222 /// database. 223 /// 224 /// `tokenize_core_fmt_to_writer!` and the accompanying [`MessageWriter`] trait 225 /// provide an optimized API for use cases like logging where the output of the 226 /// tokenization will be written to a shared/ambient resource like stdio, a 227 /// UART, or a shared buffer. 228 /// 229 /// See [`token`] for an explanation on how strings are tokenized and entries 230 /// are added to the token database. The token's domain is set to `""`. 231 /// 232 /// Returns a [`pw_status::Result<()>`]. 233 /// 234 /// `tokenize_core_fmt_to_writer!` supports concatenation of format strings as 235 /// described in [`pw_format::macros::FormatAndArgs`]. 236 /// 237 /// # Errors 238 /// - [`pw_status::Error::OutOfRange`] - [`MessageWriter`] does not have enough 239 /// space to fit tokenized data. 240 /// - others - `tokenize_core_fmt_to_writer!` will pass on any errors returned 241 /// by the [`MessageWriter`]. 242 /// 243 /// # Code Size 244 /// 245 /// This data was collected by examining the disassembly of a test program 246 /// built for a Cortex M0. 247 /// 248 /// | Tokenized Message | Per Call-site Cost (bytes) | 249 /// | --------------------| -------------------------- | 250 /// | no arguments | 10 | 251 /// | one `i32` argument | 18 | 252 /// 253 /// # Example 254 /// 255 /// ``` 256 /// use pw_status::Result; 257 /// use pw_stream::{Cursor, Write}; 258 /// use pw_tokenizer::{MessageWriter, tokenize_core_fmt_to_writer}; 259 /// 260 /// const BUFFER_LEN: usize = 32; 261 /// 262 /// // Declare a simple MessageWriter that uses a [`pw_status::Cursor`] to 263 /// // maintain an internal buffer. 264 /// struct TestMessageWriter { 265 /// cursor: Cursor<[u8; BUFFER_LEN]>, 266 /// } 267 /// 268 /// impl MessageWriter for TestMessageWriter { 269 /// fn new() -> Self { 270 /// Self { 271 /// cursor: Cursor::new([0u8; BUFFER_LEN]), 272 /// } 273 /// } 274 /// 275 /// fn write(&mut self, data: &[u8]) -> Result<()> { 276 /// self.cursor.write_all(data) 277 /// } 278 /// 279 /// fn remaining(&self) -> usize { 280 /// self.cursor.remaining() 281 /// } 282 /// 283 /// fn finalize(self) -> Result<()> { 284 /// let len = self.cursor.position(); 285 /// // 4 bytes used to encode the token and one to encode the value 42. 286 /// assert_eq!(len, 5); 287 /// Ok(()) 288 /// } 289 /// } 290 /// 291 /// // Tokenize a format string and argument into the writer. Note how we 292 /// // pass in the message writer's type, not an instance of it. 293 /// let len = tokenize_core_fmt_to_writer!(TestMessageWriter, "The answer is {}", 42 as i32)?; 294 /// # Ok::<(), pw_status::Error>(()) 295 /// ``` 296 #[macro_export] 297 macro_rules! tokenize_core_fmt_to_writer { 298 ($ty:ty, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{ 299 use $crate::__private as __pw_tokenizer_crate; 300 __pw_tokenizer_crate::_tokenize_core_fmt_to_writer!($ty, $($format_string)PW_FMT_CONCAT+, $($args),*) 301 }}; 302 } 303 304 /// Tokenize a `printf` format string and arguments to a [`MessageWriter`] and 305 /// add the format string's token to the token database. 306 /// 307 /// `tokenize_printf_fmt_to_writer!` and the accompanying [`MessageWriter`] trait 308 /// provide an optimized API for use cases like logging where the output of the 309 /// tokenization will be written to a shared/ambient resource like stdio, a 310 /// UART, or a shared buffer. 311 /// 312 /// See [`token`] for an explanation on how strings are tokenized and entries 313 /// are added to the token database. The token's domain is set to `""`. 314 /// 315 /// Returns a [`pw_status::Result<()>`]. 316 /// 317 /// `tokenize_core_fmt_to_writer!` supports concatenation of format strings as 318 /// described in [`pw_format::macros::FormatAndArgs`]. 319 /// 320 /// # Errors 321 /// - [`pw_status::Error::OutOfRange`] - [`MessageWriter`] does not have enough 322 /// space to fit tokenized data. 323 /// - others - `tokenize_printf_to_writer!` will pass on any errors returned 324 /// by the [`MessageWriter`]. 325 /// 326 /// # Code Size 327 /// 328 /// This data was collected by examining the disassembly of a test program 329 /// built for a Cortex M0. 330 /// 331 /// | Tokenized Message | Per Call-site Cost (bytes) | 332 /// | --------------------| -------------------------- | 333 /// | no arguments | 10 | 334 /// | one `i32` argument | 18 | 335 /// 336 /// # Example 337 /// 338 /// ``` 339 /// use pw_status::Result; 340 /// use pw_stream::{Cursor, Write}; 341 /// use pw_tokenizer::{MessageWriter, tokenize_printf_to_writer}; 342 /// 343 /// const BUFFER_LEN: usize = 32; 344 /// 345 /// // Declare a simple MessageWriter that uses a [`pw_status::Cursor`] to 346 /// // maintain an internal buffer. 347 /// struct TestMessageWriter { 348 /// cursor: Cursor<[u8; BUFFER_LEN]>, 349 /// } 350 /// 351 /// impl MessageWriter for TestMessageWriter { 352 /// fn new() -> Self { 353 /// Self { 354 /// cursor: Cursor::new([0u8; BUFFER_LEN]), 355 /// } 356 /// } 357 /// 358 /// fn write(&mut self, data: &[u8]) -> Result<()> { 359 /// self.cursor.write_all(data) 360 /// } 361 /// 362 /// fn remaining(&self) -> usize { 363 /// self.cursor.remaining() 364 /// } 365 /// 366 /// fn finalize(self) -> Result<()> { 367 /// let len = self.cursor.position(); 368 /// // 4 bytes used to encode the token and one to encode the value 42. 369 /// assert_eq!(len, 5); 370 /// Ok(()) 371 /// } 372 /// } 373 /// 374 /// // Tokenize a format string and argument into the writer. Note how we 375 /// // pass in the message writer's type, not an instance of it. 376 /// let len = tokenize_printf_to_writer!(TestMessageWriter, "The answer is %d", 42)?; 377 /// # Ok::<(), pw_status::Error>(()) 378 /// ``` 379 #[macro_export] 380 macro_rules! tokenize_printf_to_writer { 381 ($ty:ty, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{ 382 use $crate::__private as __pw_tokenizer_crate; 383 __pw_tokenizer_crate::_tokenize_printf_to_writer!($ty, $($format_string)PW_FMT_CONCAT+, $($args),*) 384 }}; 385 } 386 387 /// Deprecated alias for [`tokenize_printf_to_writer!`]. 388 #[macro_export] 389 macro_rules! tokenize_to_writer { 390 ($ty:ty, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{ 391 $crate::tokenize_printf_to_writer!($ty, $($format_string)PW_FMT_CONCAT+, $($args),*) 392 }}; 393 } 394 395 /// A trait used by [`tokenize_to_writer!`] to output tokenized messages. 396 /// 397 /// For more details on how this type is used, see the [`tokenize_to_writer!`] 398 /// documentation. 399 pub trait MessageWriter { 400 /// Returns a new instance of a `MessageWriter`. new() -> Self401 fn new() -> Self; 402 403 /// Append `data` to the message. write(&mut self, data: &[u8]) -> Result<()>404 fn write(&mut self, data: &[u8]) -> Result<()>; 405 406 /// Return the remaining space in this message instance. 407 /// 408 /// If there are no space constraints, return `usize::MAX`. remaining(&self) -> usize409 fn remaining(&self) -> usize; 410 411 /// Finalize message. 412 /// 413 /// `finalize()` is called when the tokenized message is complete. finalize(self) -> Result<()>414 fn finalize(self) -> Result<()>; 415 } 416 417 #[cfg(test)] 418 // Untyped prints code rely on as casts to annotate type information. 419 #[allow(clippy::unnecessary_cast)] 420 mod tests { 421 use super::*; 422 extern crate self as pw_tokenizer; 423 use pw_stream::{Cursor, Write}; 424 use std::cell::RefCell; 425 426 // This is not meant to be an exhaustive test of tokenization which is 427 // covered by `pw_tokenizer_core`'s unit tests. Rather, this is testing 428 // that the `tokenize!` macro connects to that correctly. 429 #[test] test_token()430 fn test_token() {} 431 432 macro_rules! tokenize_to_buffer_test { 433 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{ 434 if $printf_fmt != "" { 435 let mut buffer = [0u8; $buffer_len]; 436 let len = tokenize_printf_to_buffer!(&mut buffer, $printf_fmt, $($args),*).unwrap(); 437 assert_eq!( 438 &buffer[..len], 439 $expected_data, 440 "printf style input does not produce expected output", 441 ); 442 } 443 if $core_fmt != "" { 444 let mut buffer = [0u8; $buffer_len]; 445 let len = tokenize_core_fmt_to_buffer!(&mut buffer, $core_fmt, $($args),*).unwrap(); 446 assert_eq!( 447 &buffer[..len], 448 $expected_data, 449 "core::fmt style input does not produce expected output", 450 ); 451 } 452 }} 453 } 454 455 macro_rules! tokenize_to_writer_test { 456 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{ 457 // The `MessageWriter` API is used in places like logging where it 458 // accesses an shared/ambient resource (like stdio or an UART). To test 459 // it in a hermetic way we declare test specific `MessageWriter` that 460 // writes it's output to a scoped static variable that can be checked 461 // after the test is run. 462 463 // Since these tests are not multi-threaded, we can use a thread_local! 464 // instead of a mutex. 465 thread_local!(static TEST_OUTPUT: RefCell<Option<Vec<u8>>> = RefCell::new(None)); 466 467 struct TestMessageWriter { 468 cursor: Cursor<[u8; $buffer_len]>, 469 } 470 471 impl MessageWriter for TestMessageWriter { 472 fn new() -> Self { 473 Self { 474 cursor: Cursor::new([0u8; $buffer_len]), 475 } 476 } 477 478 fn write(&mut self, data: &[u8]) -> Result<()> { 479 self.cursor.write_all(data) 480 } 481 482 fn remaining(&self) -> usize { 483 self.cursor.remaining() 484 } 485 486 fn finalize(self) -> Result<()> { 487 let write_len = self.cursor.position(); 488 let data = self.cursor.into_inner(); 489 TEST_OUTPUT.with(|output| *output.borrow_mut() = Some(data[..write_len].to_vec())); 490 491 Ok(()) 492 } 493 } 494 495 if $printf_fmt != "" { 496 TEST_OUTPUT.with(|output| *output.borrow_mut() = None); 497 tokenize_printf_to_writer!(TestMessageWriter, $printf_fmt, $($args),*).unwrap(); 498 TEST_OUTPUT.with(|output| { 499 assert_eq!( 500 *output.borrow(), 501 Some($expected_data.to_vec()), 502 ) 503 }); 504 } 505 506 if $core_fmt != "" { 507 TEST_OUTPUT.with(|output| *output.borrow_mut() = None); 508 tokenize_core_fmt_to_writer!(TestMessageWriter, $core_fmt, $($args),*).unwrap(); 509 TEST_OUTPUT.with(|output| { 510 assert_eq!( 511 *output.borrow(), 512 Some($expected_data.to_vec()), 513 ) 514 }); 515 } 516 }} 517 } 518 519 macro_rules! tokenize_test { 520 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{ 521 tokenize_to_buffer_test!($expected_data, $buffer_len, $printf_fmt, $core_fmt, $($args),*); 522 tokenize_to_writer_test!($expected_data, $buffer_len, $printf_fmt, $core_fmt, $($args),*); 523 }}; 524 } 525 526 #[test] bare_string_encodes_correctly()527 fn bare_string_encodes_correctly() { 528 tokenize_test!( 529 &[0xe0, 0x92, 0xe0, 0xa], // expected buffer 530 64, // buffer size 531 "Hello Pigweed", // printf style 532 "Hello Pigweed", // core::fmt style 533 ); 534 } 535 536 #[test] test_decimal_format()537 fn test_decimal_format() { 538 // "as casts" are used for the integer arguments below. They are only 539 // need for the core::fmt style arguments but are added so that we can 540 // check that the printf and core::fmt style equivalents encode the same. 541 tokenize_test!( 542 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer 543 64, // buffer size 544 "The answer is %d!", // printf style 545 "The answer is {}!", // core::fmt style 546 1 as i32 547 ); 548 549 tokenize_test!( 550 &[0x36, 0xd0, 0xfb, 0x69, 0x1], // expected buffer 551 64, // buffer size 552 "No! The answer is %d!", // printf style 553 "No! The answer is {}!", // core::fmt style 554 -1 as i32 555 ); 556 557 tokenize_test!( 558 &[0xa4, 0xad, 0x50, 0x54, 0x0], // expected buffer 559 64, // buffer size 560 "I think you'll find that the answer is %d!", // printf style 561 "I think you'll find that the answer is {}!", // core::fmt style 562 0 as i32 563 ); 564 } 565 566 #[test] test_misc_integer_format()567 fn test_misc_integer_format() { 568 // %d, %i, %o, %u, %x, %X all encode integers the same. 569 tokenize_test!( 570 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer 571 64, // buffer size 572 "The answer is %d!", // printf style 573 "", // no equivalent core::fmt style 574 1 575 ); 576 577 // Because %i is an alias for %d, it gets converted to a %d by the 578 // `pw_format` macro infrastructure. 579 tokenize_test!( 580 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer 581 64, // buffer size 582 "The answer is %i!", // printf style 583 "", // no equivalent core::fmt style 584 1 585 ); 586 587 tokenize_test!( 588 &[0x5d, 0x70, 0x12, 0xb4, 0x2], // expected buffer 589 64, // buffer size 590 "The answer is %o!", // printf style 591 "", // no equivalent core::fmt style 592 1u32 593 ); 594 595 tokenize_test!( 596 &[0x63, 0x58, 0x5f, 0x8f, 0x2], // expected buffer 597 64, // buffer size 598 "The answer is %u!", // printf style 599 "", // no equivalent core::fmt style 600 1u32 601 ); 602 603 tokenize_test!( 604 &[0x66, 0xcc, 0x05, 0x7d, 0x2], // expected buffer 605 64, // buffer size 606 "The answer is %x!", // printf style 607 "", // no equivalent core::fmt style 608 1u32 609 ); 610 611 tokenize_test!( 612 &[0x46, 0x4c, 0x16, 0x96, 0x2], // expected buffer 613 64, // buffer size 614 "The answer is %X!", // printf style 615 "", // no equivalent core::fmt style 616 1u32 617 ); 618 } 619 620 #[test] test_string_format()621 fn test_string_format() { 622 tokenize_test!( 623 b"\x25\xf6\x2e\x66\x07Pigweed", // expected buffer 624 64, // buffer size 625 "Hello: %s!", // printf style 626 "", // no equivalent core::fmt style 627 "Pigweed" 628 ); 629 } 630 631 #[test] test_string_format_overflow()632 fn test_string_format_overflow() { 633 tokenize_test!( 634 b"\x25\xf6\x2e\x66\x83Pig", // expected buffer 635 8, // buffer size 636 "Hello: %s!", // printf style 637 "", // no equivalent core::fmt style 638 "Pigweed" 639 ); 640 } 641 642 #[test] test_char_format()643 fn test_char_format() { 644 tokenize_test!( 645 &[0x2e, 0x52, 0xac, 0xe4, 0x50], // expected buffer 646 64, // buffer size 647 "Hello: %cigweed", // printf style 648 "", // no equivalent core::fmt style 649 "P".as_bytes()[0] 650 ); 651 } 652 653 #[test] test_untyped_format()654 fn test_untyped_format() { 655 tokenize_test!( 656 &[0x63, 0x58, 0x5f, 0x8f, 0x2], // expected buffer 657 64, // buffer size 658 "The answer is %u!", // printf style 659 "The answer is {}!", // core::fmt style 660 1 as u32 661 ); 662 663 tokenize_test!( 664 &[0x36, 0xd0, 0xfb, 0x69, 0x1], // expected buffer 665 64, // buffer size 666 "No! The answer is %v!", // printf style 667 "No! The answer is {}!", // core::fmt style 668 -1 as i32 669 ); 670 671 tokenize_test!( 672 b"\x25\xf6\x2e\x66\x07Pigweed", // expected buffer 673 64, // buffer size 674 "Hello: %v!", // printf style 675 "Hello: {}!", // core::fmt style 676 "Pigweed" as &str 677 ); 678 } 679 680 #[test] test_field_width_and_zero_pad_format()681 fn test_field_width_and_zero_pad_format() { 682 tokenize_test!( 683 &[0x3a, 0xc2, 0x1a, 0x05, 0xfc, 0xab, 0x06], // expected buffer 684 64, // buffer size 685 "Lets go to the %x", // printf style 686 "Lets go to the {:x}", // core::fmt style 687 0xcafe as u32 688 ); 689 690 tokenize_test!( 691 &[0xf3, 0x16, 0x03, 0x99, 0xfc, 0xab, 0x06], // expected buffer 692 64, // buffer size 693 "Lets go to the %8x", // printf style 694 "Lets go to the {:8x}", // core::fmt style 695 0xcafe as u32 696 ); 697 698 tokenize_test!( 699 &[0x44, 0xce, 0xa3, 0x7e, 0xfc, 0xab, 0x06], // expected buffer 700 64, // buffer size 701 "Lets go to the %08x", // printf style 702 "Lets go to the {:08x}", // core::fmt style 703 0xcafe as u32 704 ); 705 } 706 707 #[test] tokenizer_supports_concatenated_printf_format_strings()708 fn tokenizer_supports_concatenated_printf_format_strings() { 709 // Since the no argument and some arguments cases are handled differently 710 // by `tokenize_to_buffer!` we need to test both. 711 let mut buffer = [0u8; 64]; 712 let len = 713 tokenize_printf_to_buffer!(&mut buffer, "Hello" PW_FMT_CONCAT " Pigweed").unwrap(); 714 assert_eq!(&buffer[..len], &[0xe0, 0x92, 0xe0, 0xa]); 715 716 let len = tokenize_printf_to_buffer!(&mut buffer, "Hello: " PW_FMT_CONCAT "%cigweed", 717 "P".as_bytes()[0]) 718 .unwrap(); 719 assert_eq!(&buffer[..len], &[0x2e, 0x52, 0xac, 0xe4, 0x50]); 720 } 721 722 #[test] tokenizer_supports_concatenated_core_fmt_format_strings()723 fn tokenizer_supports_concatenated_core_fmt_format_strings() { 724 // Since the no argument and some arguments cases are handled differently 725 // by `tokenize_to_buffer!` we need to test both. 726 let mut buffer = [0u8; 64]; 727 let len = 728 tokenize_core_fmt_to_buffer!(&mut buffer, "Hello" PW_FMT_CONCAT " Pigweed").unwrap(); 729 assert_eq!(&buffer[..len], &[0xe0, 0x92, 0xe0, 0xa]); 730 731 let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is " PW_FMT_CONCAT "{}!", 732 1 as i32) 733 .unwrap(); 734 assert_eq!(&buffer[..len], &[0x52, 0x1c, 0xb0, 0x4c, 0x2]); 735 } 736 } 737