1 #![warn(rust_2018_idioms)]
2
3 use tokio_util::codec::{AnyDelimiterCodec, BytesCodec, Decoder, Encoder, LinesCodec};
4
5 use bytes::{BufMut, Bytes, BytesMut};
6
7 #[test]
bytes_decoder()8 fn bytes_decoder() {
9 let mut codec = BytesCodec::new();
10 let buf = &mut BytesMut::new();
11 buf.put_slice(b"abc");
12 assert_eq!("abc", codec.decode(buf).unwrap().unwrap());
13 assert_eq!(None, codec.decode(buf).unwrap());
14 assert_eq!(None, codec.decode(buf).unwrap());
15 buf.put_slice(b"a");
16 assert_eq!("a", codec.decode(buf).unwrap().unwrap());
17 }
18
19 #[test]
bytes_encoder()20 fn bytes_encoder() {
21 let mut codec = BytesCodec::new();
22
23 // Default capacity of BytesMut
24 #[cfg(target_pointer_width = "64")]
25 const INLINE_CAP: usize = 4 * 8 - 1;
26 #[cfg(target_pointer_width = "32")]
27 const INLINE_CAP: usize = 4 * 4 - 1;
28
29 let mut buf = BytesMut::new();
30 codec
31 .encode(Bytes::from_static(&[0; INLINE_CAP + 1]), &mut buf)
32 .unwrap();
33
34 // Default capacity of Framed Read
35 const INITIAL_CAPACITY: usize = 8 * 1024;
36
37 let mut buf = BytesMut::with_capacity(INITIAL_CAPACITY);
38 codec
39 .encode(Bytes::from_static(&[0; INITIAL_CAPACITY + 1]), &mut buf)
40 .unwrap();
41 codec
42 .encode(BytesMut::from(&b"hello"[..]), &mut buf)
43 .unwrap();
44 }
45
46 #[test]
lines_decoder()47 fn lines_decoder() {
48 let mut codec = LinesCodec::new();
49 let buf = &mut BytesMut::new();
50 buf.reserve(200);
51 buf.put_slice(b"line 1\nline 2\r\nline 3\n\r\n\r");
52 assert_eq!("line 1", codec.decode(buf).unwrap().unwrap());
53 assert_eq!("line 2", codec.decode(buf).unwrap().unwrap());
54 assert_eq!("line 3", codec.decode(buf).unwrap().unwrap());
55 assert_eq!("", codec.decode(buf).unwrap().unwrap());
56 assert_eq!(None, codec.decode(buf).unwrap());
57 assert_eq!(None, codec.decode_eof(buf).unwrap());
58 buf.put_slice(b"k");
59 assert_eq!(None, codec.decode(buf).unwrap());
60 assert_eq!("\rk", codec.decode_eof(buf).unwrap().unwrap());
61 assert_eq!(None, codec.decode(buf).unwrap());
62 assert_eq!(None, codec.decode_eof(buf).unwrap());
63 }
64
65 #[test]
lines_decoder_invalid_utf8()66 fn lines_decoder_invalid_utf8() {
67 let mut codec = LinesCodec::new();
68 let buf = &mut BytesMut::new();
69 buf.reserve(200);
70 buf.put_slice(b"line 1\xc3\x28");
71 assert_eq!(None, codec.decode(buf).unwrap());
72 assert!(codec.decode_eof(buf).is_err());
73 assert_eq!(None, codec.decode_eof(buf).unwrap());
74 buf.put_slice(b"line 22222222222222\n");
75 assert_eq!("line 22222222222222", codec.decode(buf).unwrap().unwrap());
76 }
77
78 #[test]
lines_decoder_max_length()79 fn lines_decoder_max_length() {
80 const MAX_LENGTH: usize = 6;
81
82 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
83 let buf = &mut BytesMut::new();
84
85 buf.reserve(200);
86 buf.put_slice(b"line 1 is too long\nline 2\nline 3\r\nline 4\n\r\n\r");
87
88 assert!(codec.decode(buf).is_err());
89
90 let line = codec.decode(buf).unwrap().unwrap();
91 assert!(line.len() <= MAX_LENGTH, "{line:?}.len() <= {MAX_LENGTH:?}");
92 assert_eq!("line 2", line);
93
94 assert!(codec.decode(buf).is_err());
95
96 let line = codec.decode(buf).unwrap().unwrap();
97 assert!(line.len() <= MAX_LENGTH, "{line:?}.len() <= {MAX_LENGTH:?}");
98 assert_eq!("line 4", line);
99
100 let line = codec.decode(buf).unwrap().unwrap();
101 assert!(line.len() <= MAX_LENGTH, "{line:?}.len() <= {MAX_LENGTH:?}");
102 assert_eq!("", line);
103
104 assert_eq!(None, codec.decode(buf).unwrap());
105 assert_eq!(None, codec.decode_eof(buf).unwrap());
106 buf.put_slice(b"k");
107 assert_eq!(None, codec.decode(buf).unwrap());
108
109 let line = codec.decode_eof(buf).unwrap().unwrap();
110 assert!(line.len() <= MAX_LENGTH, "{line:?}.len() <= {MAX_LENGTH:?}");
111 assert_eq!("\rk", line);
112
113 assert_eq!(None, codec.decode(buf).unwrap());
114 assert_eq!(None, codec.decode_eof(buf).unwrap());
115
116 // Line that's one character too long. This could cause an out of bounds
117 // error if we peek at the next characters using slice indexing.
118 buf.put_slice(b"aaabbbc");
119 assert!(codec.decode(buf).is_err());
120 }
121
122 #[test]
lines_decoder_max_length_underrun()123 fn lines_decoder_max_length_underrun() {
124 const MAX_LENGTH: usize = 6;
125
126 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
127 let buf = &mut BytesMut::new();
128
129 buf.reserve(200);
130 buf.put_slice(b"line ");
131 assert_eq!(None, codec.decode(buf).unwrap());
132 buf.put_slice(b"too l");
133 assert!(codec.decode(buf).is_err());
134 buf.put_slice(b"ong\n");
135 assert_eq!(None, codec.decode(buf).unwrap());
136
137 buf.put_slice(b"line 2");
138 assert_eq!(None, codec.decode(buf).unwrap());
139 buf.put_slice(b"\n");
140 assert_eq!("line 2", codec.decode(buf).unwrap().unwrap());
141 }
142
143 #[test]
lines_decoder_max_length_bursts()144 fn lines_decoder_max_length_bursts() {
145 const MAX_LENGTH: usize = 10;
146
147 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
148 let buf = &mut BytesMut::new();
149
150 buf.reserve(200);
151 buf.put_slice(b"line ");
152 assert_eq!(None, codec.decode(buf).unwrap());
153 buf.put_slice(b"too l");
154 assert_eq!(None, codec.decode(buf).unwrap());
155 buf.put_slice(b"ong\n");
156 assert!(codec.decode(buf).is_err());
157 }
158
159 #[test]
lines_decoder_max_length_big_burst()160 fn lines_decoder_max_length_big_burst() {
161 const MAX_LENGTH: usize = 10;
162
163 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
164 let buf = &mut BytesMut::new();
165
166 buf.reserve(200);
167 buf.put_slice(b"line ");
168 assert_eq!(None, codec.decode(buf).unwrap());
169 buf.put_slice(b"too long!\n");
170 assert!(codec.decode(buf).is_err());
171 }
172
173 #[test]
lines_decoder_max_length_newline_between_decodes()174 fn lines_decoder_max_length_newline_between_decodes() {
175 const MAX_LENGTH: usize = 5;
176
177 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
178 let buf = &mut BytesMut::new();
179
180 buf.reserve(200);
181 buf.put_slice(b"hello");
182 assert_eq!(None, codec.decode(buf).unwrap());
183
184 buf.put_slice(b"\nworld");
185 assert_eq!("hello", codec.decode(buf).unwrap().unwrap());
186 }
187
188 // Regression test for [infinite loop bug](https://github.com/tokio-rs/tokio/issues/1483)
189 #[test]
lines_decoder_discard_repeat()190 fn lines_decoder_discard_repeat() {
191 const MAX_LENGTH: usize = 1;
192
193 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
194 let buf = &mut BytesMut::new();
195
196 buf.reserve(200);
197 buf.put_slice(b"aa");
198 assert!(codec.decode(buf).is_err());
199 buf.put_slice(b"a");
200 assert_eq!(None, codec.decode(buf).unwrap());
201 }
202
203 // Regression test for [subsequent calls to LinesCodec decode does not return the desired results bug](https://github.com/tokio-rs/tokio/issues/3555)
204 #[test]
lines_decoder_max_length_underrun_twice()205 fn lines_decoder_max_length_underrun_twice() {
206 const MAX_LENGTH: usize = 11;
207
208 let mut codec = LinesCodec::new_with_max_length(MAX_LENGTH);
209 let buf = &mut BytesMut::new();
210
211 buf.reserve(200);
212 buf.put_slice(b"line ");
213 assert_eq!(None, codec.decode(buf).unwrap());
214 buf.put_slice(b"too very l");
215 assert!(codec.decode(buf).is_err());
216 buf.put_slice(b"aaaaaaaaaaaaaaaaaaaaaaa");
217 assert_eq!(None, codec.decode(buf).unwrap());
218 buf.put_slice(b"ong\nshort\n");
219 assert_eq!("short", codec.decode(buf).unwrap().unwrap());
220 }
221
222 #[test]
lines_encoder()223 fn lines_encoder() {
224 let mut codec = LinesCodec::new();
225 let mut buf = BytesMut::new();
226
227 codec.encode("line 1", &mut buf).unwrap();
228 assert_eq!("line 1\n", buf);
229
230 codec.encode("line 2", &mut buf).unwrap();
231 assert_eq!("line 1\nline 2\n", buf);
232 }
233
234 #[test]
any_delimiters_decoder_any_character()235 fn any_delimiters_decoder_any_character() {
236 let mut codec = AnyDelimiterCodec::new(b",;\n\r".to_vec(), b",".to_vec());
237 let buf = &mut BytesMut::new();
238 buf.reserve(200);
239 buf.put_slice(b"chunk 1,chunk 2;chunk 3\n\r");
240 assert_eq!("chunk 1", codec.decode(buf).unwrap().unwrap());
241 assert_eq!("chunk 2", codec.decode(buf).unwrap().unwrap());
242 assert_eq!("chunk 3", codec.decode(buf).unwrap().unwrap());
243 assert_eq!("", codec.decode(buf).unwrap().unwrap());
244 assert_eq!(None, codec.decode(buf).unwrap());
245 assert_eq!(None, codec.decode_eof(buf).unwrap());
246 buf.put_slice(b"k");
247 assert_eq!(None, codec.decode(buf).unwrap());
248 assert_eq!("k", codec.decode_eof(buf).unwrap().unwrap());
249 assert_eq!(None, codec.decode(buf).unwrap());
250 assert_eq!(None, codec.decode_eof(buf).unwrap());
251 }
252
253 #[test]
any_delimiters_decoder_max_length()254 fn any_delimiters_decoder_max_length() {
255 const MAX_LENGTH: usize = 7;
256
257 let mut codec =
258 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
259 let buf = &mut BytesMut::new();
260
261 buf.reserve(200);
262 buf.put_slice(b"chunk 1 is too long\nchunk 2\nchunk 3\r\nchunk 4\n\r\n");
263
264 assert!(codec.decode(buf).is_err());
265
266 let chunk = codec.decode(buf).unwrap().unwrap();
267 assert!(
268 chunk.len() <= MAX_LENGTH,
269 "{chunk:?}.len() <= {MAX_LENGTH:?}"
270 );
271 assert_eq!("chunk 2", chunk);
272
273 let chunk = codec.decode(buf).unwrap().unwrap();
274 assert!(
275 chunk.len() <= MAX_LENGTH,
276 "{chunk:?}.len() <= {MAX_LENGTH:?}"
277 );
278 assert_eq!("chunk 3", chunk);
279
280 // \r\n cause empty chunk
281 let chunk = codec.decode(buf).unwrap().unwrap();
282 assert!(
283 chunk.len() <= MAX_LENGTH,
284 "{chunk:?}.len() <= {MAX_LENGTH:?}"
285 );
286 assert_eq!("", chunk);
287
288 let chunk = codec.decode(buf).unwrap().unwrap();
289 assert!(
290 chunk.len() <= MAX_LENGTH,
291 "{chunk:?}.len() <= {MAX_LENGTH:?}"
292 );
293 assert_eq!("chunk 4", chunk);
294
295 let chunk = codec.decode(buf).unwrap().unwrap();
296 assert!(
297 chunk.len() <= MAX_LENGTH,
298 "{chunk:?}.len() <= {MAX_LENGTH:?}"
299 );
300 assert_eq!("", chunk);
301
302 let chunk = codec.decode(buf).unwrap().unwrap();
303 assert!(
304 chunk.len() <= MAX_LENGTH,
305 "{chunk:?}.len() <= {MAX_LENGTH:?}"
306 );
307 assert_eq!("", chunk);
308
309 assert_eq!(None, codec.decode(buf).unwrap());
310 assert_eq!(None, codec.decode_eof(buf).unwrap());
311 buf.put_slice(b"k");
312 assert_eq!(None, codec.decode(buf).unwrap());
313
314 let chunk = codec.decode_eof(buf).unwrap().unwrap();
315 assert!(
316 chunk.len() <= MAX_LENGTH,
317 "{chunk:?}.len() <= {MAX_LENGTH:?}"
318 );
319 assert_eq!("k", chunk);
320
321 assert_eq!(None, codec.decode(buf).unwrap());
322 assert_eq!(None, codec.decode_eof(buf).unwrap());
323
324 // Delimiter that's one character too long. This could cause an out of bounds
325 // error if we peek at the next characters using slice indexing.
326 buf.put_slice(b"aaabbbcc");
327 assert!(codec.decode(buf).is_err());
328 }
329
330 #[test]
any_delimiter_decoder_max_length_underrun()331 fn any_delimiter_decoder_max_length_underrun() {
332 const MAX_LENGTH: usize = 7;
333
334 let mut codec =
335 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
336 let buf = &mut BytesMut::new();
337
338 buf.reserve(200);
339 buf.put_slice(b"chunk ");
340 assert_eq!(None, codec.decode(buf).unwrap());
341 buf.put_slice(b"too l");
342 assert!(codec.decode(buf).is_err());
343 buf.put_slice(b"ong\n");
344 assert_eq!(None, codec.decode(buf).unwrap());
345
346 buf.put_slice(b"chunk 2");
347 assert_eq!(None, codec.decode(buf).unwrap());
348 buf.put_slice(b",");
349 assert_eq!("chunk 2", codec.decode(buf).unwrap().unwrap());
350 }
351
352 #[test]
any_delimiter_decoder_max_length_underrun_twice()353 fn any_delimiter_decoder_max_length_underrun_twice() {
354 const MAX_LENGTH: usize = 11;
355
356 let mut codec =
357 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
358 let buf = &mut BytesMut::new();
359
360 buf.reserve(200);
361 buf.put_slice(b"chunk ");
362 assert_eq!(None, codec.decode(buf).unwrap());
363 buf.put_slice(b"too very l");
364 assert!(codec.decode(buf).is_err());
365 buf.put_slice(b"aaaaaaaaaaaaaaaaaaaaaaa");
366 assert_eq!(None, codec.decode(buf).unwrap());
367 buf.put_slice(b"ong\nshort\n");
368 assert_eq!("short", codec.decode(buf).unwrap().unwrap());
369 }
370 #[test]
any_delimiter_decoder_max_length_bursts()371 fn any_delimiter_decoder_max_length_bursts() {
372 const MAX_LENGTH: usize = 11;
373
374 let mut codec =
375 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
376 let buf = &mut BytesMut::new();
377
378 buf.reserve(200);
379 buf.put_slice(b"chunk ");
380 assert_eq!(None, codec.decode(buf).unwrap());
381 buf.put_slice(b"too l");
382 assert_eq!(None, codec.decode(buf).unwrap());
383 buf.put_slice(b"ong\n");
384 assert!(codec.decode(buf).is_err());
385 }
386
387 #[test]
any_delimiter_decoder_max_length_big_burst()388 fn any_delimiter_decoder_max_length_big_burst() {
389 const MAX_LENGTH: usize = 11;
390
391 let mut codec =
392 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
393 let buf = &mut BytesMut::new();
394
395 buf.reserve(200);
396 buf.put_slice(b"chunk ");
397 assert_eq!(None, codec.decode(buf).unwrap());
398 buf.put_slice(b"too long!\n");
399 assert!(codec.decode(buf).is_err());
400 }
401
402 #[test]
any_delimiter_decoder_max_length_delimiter_between_decodes()403 fn any_delimiter_decoder_max_length_delimiter_between_decodes() {
404 const MAX_LENGTH: usize = 5;
405
406 let mut codec =
407 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
408 let buf = &mut BytesMut::new();
409
410 buf.reserve(200);
411 buf.put_slice(b"hello");
412 assert_eq!(None, codec.decode(buf).unwrap());
413
414 buf.put_slice(b",world");
415 assert_eq!("hello", codec.decode(buf).unwrap().unwrap());
416 }
417
418 #[test]
any_delimiter_decoder_discard_repeat()419 fn any_delimiter_decoder_discard_repeat() {
420 const MAX_LENGTH: usize = 1;
421
422 let mut codec =
423 AnyDelimiterCodec::new_with_max_length(b",;\n\r".to_vec(), b",".to_vec(), MAX_LENGTH);
424 let buf = &mut BytesMut::new();
425
426 buf.reserve(200);
427 buf.put_slice(b"aa");
428 assert!(codec.decode(buf).is_err());
429 buf.put_slice(b"a");
430 assert_eq!(None, codec.decode(buf).unwrap());
431 }
432
433 #[test]
any_delimiter_encoder()434 fn any_delimiter_encoder() {
435 let mut codec = AnyDelimiterCodec::new(b",".to_vec(), b";--;".to_vec());
436 let mut buf = BytesMut::new();
437
438 codec.encode("chunk 1", &mut buf).unwrap();
439 assert_eq!("chunk 1;--;", buf);
440
441 codec.encode("chunk 2", &mut buf).unwrap();
442 assert_eq!("chunk 1;--;chunk 2;--;", buf);
443 }
444