1#[cfg(test)]
2use alloc::vec;
3
4#[cfg(test)]
5use alloc::vec::Vec;
6
7#[cfg(test)]
8extern crate std;
9
10#[cfg(all(test, not(feature = "std")))]
11impl crate::io_nostd::Read for std::fs::File {
12 fn read(&mut self, buf: &mut [u8]) -> Result<usize, crate::io_nostd::Error> {
13 std::io::Read::read(self, buf).map_err(|e| {
14 if e.get_ref().is_none() {
15 crate::io_nostd::Error::from(crate::io_nostd::ErrorKind::Other)
16 } else {
17 crate::io_nostd::Error::new(
18 crate::io_nostd::ErrorKind::Other,
19 alloc::boxed::Box::new(e.into_inner().unwrap()),
20 )
21 }
22 })
23 }
24}
25
26#[cfg(all(test, feature = "std"))]
27#[allow(dead_code)]
28fn assure_error_impl() {
29 // not a real test just there to throw an compiler error if Error is not derived correctly
30
31 use crate::frame_decoder::FrameDecoderError;
32 let _err: &dyn std::error::Error = &FrameDecoderError::NotYetInitialized;
33}
34
35#[cfg(all(test, feature = "std"))]
36#[allow(dead_code)]
37fn assure_decoder_send_sync() {
38 // not a real test just there to throw an compiler error if FrameDecoder is Send + Sync
39
40 use crate::frame_decoder::FrameDecoder;
41 let decoder = FrameDecoder::new();
42 std::thread::spawn(move || {
43 drop(decoder);
44 });
45}
46
47#[test]
48fn skippable_frame() {
49 use crate::frame;
50
51 let mut content = vec![];
52 content.extend_from_slice(&0x184D2A50u32.to_le_bytes());
53 content.extend_from_slice(&300u32.to_le_bytes());
54 assert_eq!(8, content.len());
55 let err = frame::read_frame_header(content.as_slice());
56 assert!(matches!(
57 err,
58 Err(frame::ReadFrameHeaderError::SkipFrame {
59 magic_number: 0x184D2A50u32,
60 length: 300
61 })
62 ));
63
64 content.clear();
65 content.extend_from_slice(&0x184D2A5Fu32.to_le_bytes());
66 content.extend_from_slice(&0xFFFFFFFFu32.to_le_bytes());
67 assert_eq!(8, content.len());
68 let err = frame::read_frame_header(content.as_slice());
69 assert!(matches!(
70 err,
71 Err(frame::ReadFrameHeaderError::SkipFrame {
72 magic_number: 0x184D2A5Fu32,
73 length: 0xFFFFFFFF
74 })
75 ));
76}
77
78#[cfg(test)]
79#[test]
80fn test_frame_header_reading() {
81 use crate::frame;
82 use std::fs;
83
84 let mut content = fs::File::open("./decodecorpus_files/z000088.zst").unwrap();
85 let (_frame, _) = frame::read_frame_header(&mut content).unwrap();
86}
87
88#[test]
89fn test_block_header_reading() {
90 use crate::decoding;
91 use crate::frame;
92 use std::fs;
93
94 let mut content = fs::File::open("./decodecorpus_files/z000088.zst").unwrap();
95 let (_frame, _) = frame::read_frame_header(&mut content).unwrap();
96
97 let mut block_dec = decoding::block_decoder::new();
98 let block_header = block_dec.read_block_header(&mut content).unwrap();
99 let _ = block_header; //TODO validate blockheader in a smart way
100}
101
102#[test]
103fn test_frame_decoder() {
104 use crate::frame_decoder;
105 use std::fs;
106
107 let mut content = fs::File::open("./decodecorpus_files/z000088.zst").unwrap();
108
109 struct NullWriter(());
110 impl std::io::Write for NullWriter {
111 fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
112 Ok(buf.len())
113 }
114 fn flush(&mut self) -> Result<(), std::io::Error> {
115 Ok(())
116 }
117 }
118 let mut _null_target = NullWriter(());
119
120 let mut frame_dec = frame_decoder::FrameDecoder::new();
121 frame_dec.reset(&mut content).unwrap();
122 frame_dec
123 .decode_blocks(&mut content, frame_decoder::BlockDecodingStrategy::All)
124 .unwrap();
125}
126
127#[test]
128fn test_decode_from_to() {
129 use crate::frame_decoder;
130 use std::fs::File;
131 use std::io::Read;
132 let f = File::open("./decodecorpus_files/z000088.zst").unwrap();
133 let mut frame_dec = frame_decoder::FrameDecoder::new();
134
135 let content: Vec<u8> = f.bytes().map(|x| x.unwrap()).collect();
136
137 let mut target = vec![0u8; 1024 * 1024];
138
139 // first part
140 let source1 = &content[..50 * 1024];
141 let (read1, written1) = frame_dec
142 .decode_from_to(source1, target.as_mut_slice())
143 .unwrap();
144
145 //second part explicitely without checksum
146 let source2 = &content[read1..content.len() - 4];
147 let (read2, written2) = frame_dec
148 .decode_from_to(source2, &mut target[written1..])
149 .unwrap();
150
151 //must have decoded until checksum
152 assert!(read1 + read2 == content.len() - 4);
153
154 //insert checksum separatly to test that this is handled correctly
155 let chksum_source = &content[read1 + read2..];
156 let (read3, written3) = frame_dec
157 .decode_from_to(chksum_source, &mut target[written1 + written2..])
158 .unwrap();
159
160 //this must result in these values because just the checksum was processed
161 assert!(read3 == 4);
162 assert!(written3 == 0);
163
164 let read = read1 + read2 + read3;
165 let written = written1 + written2;
166
167 let result = &target.as_slice()[..written];
168
169 if read != content.len() {
170 panic!(
171 "Byte counter: {} was wrong. Should be: {}",
172 read,
173 content.len()
174 );
175 }
176
177 match frame_dec.get_checksum_from_data() {
178 Some(chksum) => {
179 #[cfg(feature = "hash")]
180 if frame_dec.get_calculated_checksum().unwrap() != chksum {
181 std::println!(
182 "Checksum did not match! From data: {}, calculated while decoding: {}\n",
183 chksum,
184 frame_dec.get_calculated_checksum().unwrap()
185 );
186 } else {
187 std::println!("Checksums are ok!\n");
188 }
189 #[cfg(not(feature = "hash"))]
190 std::println!(
191 "Checksum feature not enabled, skipping. From data: {}\n",
192 chksum
193 );
194 }
195 None => std::println!("No checksums to test\n"),
196 }
197
198 let original_f = File::open("./decodecorpus_files/z000088").unwrap();
199 let original: Vec<u8> = original_f.bytes().map(|x| x.unwrap()).collect();
200
201 if original.len() != result.len() {
202 panic!(
203 "Result has wrong length: {}, should be: {}",
204 result.len(),
205 original.len()
206 );
207 }
208
209 let mut counter = 0;
210 let min = if original.len() < result.len() {
211 original.len()
212 } else {
213 result.len()
214 };
215 for idx in 0..min {
216 if original[idx] != result[idx] {
217 counter += 1;
218 //std::println!(
219 // "Original {:3} not equal to result {:3} at byte: {}",
220 // original[idx], result[idx], idx,
221 //);
222 }
223 }
224 if counter > 0 {
225 panic!("Result differs in at least {} bytes from original", counter);
226 }
227}
228
229#[test]
230fn test_specific_file() {
231 use crate::frame_decoder;
232 use std::fs;
233 use std::io::Read;
234
235 let path = "./decodecorpus_files/z000068.zst";
236 let mut content = fs::File::open(path).unwrap();
237
238 struct NullWriter(());
239 impl std::io::Write for NullWriter {
240 fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
241 Ok(buf.len())
242 }
243 fn flush(&mut self) -> Result<(), std::io::Error> {
244 Ok(())
245 }
246 }
247 let mut _null_target = NullWriter(());
248
249 let mut frame_dec = frame_decoder::FrameDecoder::new();
250 frame_dec.reset(&mut content).unwrap();
251 frame_dec
252 .decode_blocks(&mut content, frame_decoder::BlockDecodingStrategy::All)
253 .unwrap();
254 let result = frame_dec.collect().unwrap();
255
256 let original_f = fs::File::open("./decodecorpus_files/z000088").unwrap();
257 let original: Vec<u8> = original_f.bytes().map(|x| x.unwrap()).collect();
258
259 std::println!("Results for file: {}", path);
260
261 if original.len() != result.len() {
262 std::println!(
263 "Result has wrong length: {}, should be: {}",
264 result.len(),
265 original.len()
266 );
267 }
268
269 let mut counter = 0;
270 let min = if original.len() < result.len() {
271 original.len()
272 } else {
273 result.len()
274 };
275 for idx in 0..min {
276 if original[idx] != result[idx] {
277 counter += 1;
278 //std::println!(
279 // "Original {:3} not equal to result {:3} at byte: {}",
280 // original[idx], result[idx], idx,
281 //);
282 }
283 }
284 if counter > 0 {
285 std::println!("Result differs in at least {} bytes from original", counter);
286 }
287}
288
289#[test]
290#[cfg(feature = "std")]
291fn test_streaming() {
292 use std::fs;
293 use std::io::Read;
294
295 let mut content = fs::File::open("./decodecorpus_files/z000088.zst").unwrap();
296 let mut stream = crate::streaming_decoder::StreamingDecoder::new(&mut content).unwrap();
297
298 let mut result = Vec::new();
299 Read::read_to_end(&mut stream, &mut result).unwrap();
300
301 let original_f = fs::File::open("./decodecorpus_files/z000088").unwrap();
302 let original: Vec<u8> = original_f.bytes().map(|x| x.unwrap()).collect();
303
304 if original.len() != result.len() {
305 panic!(
306 "Result has wrong length: {}, should be: {}",
307 result.len(),
308 original.len()
309 );
310 }
311
312 let mut counter = 0;
313 let min = if original.len() < result.len() {
314 original.len()
315 } else {
316 result.len()
317 };
318 for idx in 0..min {
319 if original[idx] != result[idx] {
320 counter += 1;
321 //std::println!(
322 // "Original {:3} not equal to result {:3} at byte: {}",
323 // original[idx], result[idx], idx,
324 //);
325 }
326 }
327 if counter > 0 {
328 panic!("Result differs in at least {} bytes from original", counter);
329 }
330
331 // Test resetting to a new file while keeping the old decoder
332
333 let mut content = fs::File::open("./decodecorpus_files/z000068.zst").unwrap();
334 let mut stream = crate::streaming_decoder::StreamingDecoder::new_with_decoder(
335 &mut content,
336 stream.into_frame_decoder(),
337 )
338 .unwrap();
339
340 let mut result = Vec::new();
341 Read::read_to_end(&mut stream, &mut result).unwrap();
342
343 let original_f = fs::File::open("./decodecorpus_files/z000068").unwrap();
344 let original: Vec<u8> = original_f.bytes().map(|x| x.unwrap()).collect();
345
346 std::println!("Results for file:");
347
348 if original.len() != result.len() {
349 panic!(
350 "Result has wrong length: {}, should be: {}",
351 result.len(),
352 original.len()
353 );
354 }
355
356 let mut counter = 0;
357 let min = if original.len() < result.len() {
358 original.len()
359 } else {
360 result.len()
361 };
362 for idx in 0..min {
363 if original[idx] != result[idx] {
364 counter += 1;
365 //std::println!(
366 // "Original {:3} not equal to result {:3} at byte: {}",
367 // original[idx], result[idx], idx,
368 //);
369 }
370 }
371 if counter > 0 {
372 panic!("Result differs in at least {} bytes from original", counter);
373 }
374}
375
376#[test]
377fn test_incremental_read() {
378 use crate::frame_decoder::FrameDecoder;
379
380 let mut unread_compressed_content =
381 include_bytes!("../../decodecorpus_files/abc.txt.zst").as_slice();
382
383 let mut frame_dec = FrameDecoder::new();
384 frame_dec.reset(&mut unread_compressed_content).unwrap();
385
386 let mut output = [0u8; 3];
387 let (_, written) = frame_dec
388 .decode_from_to(unread_compressed_content, &mut output)
389 .unwrap();
390
391 assert_eq!(written, 3);
392 assert_eq!(output.map(char::from), ['a', 'b', 'c']);
393
394 assert!(frame_dec.is_finished());
395 let written = frame_dec.collect_to_writer(&mut &mut output[..]).unwrap();
396 assert_eq!(written, 3);
397 assert_eq!(output.map(char::from), ['d', 'e', 'f']);
398}
399
400#[test]
401#[cfg(not(feature = "std"))]
402fn test_streaming_no_std() {
403 use crate::io::Read;
404
405 let content = include_bytes!("../../decodecorpus_files/z000088.zst");
406 let mut content = content.as_slice();
407 let mut stream = crate::streaming_decoder::StreamingDecoder::new(&mut content).unwrap();
408
409 let original = include_bytes!("../../decodecorpus_files/z000088");
410 let mut result = vec![0; original.len()];
411 Read::read_exact(&mut stream, &mut result).unwrap();
412
413 if original.len() != result.len() {
414 panic!(
415 "Result has wrong length: {}, should be: {}",
416 result.len(),
417 original.len()
418 );
419 }
420
421 let mut counter = 0;
422 let min = if original.len() < result.len() {
423 original.len()
424 } else {
425 result.len()
426 };
427 for idx in 0..min {
428 if original[idx] != result[idx] {
429 counter += 1;
430 //std::println!(
431 // "Original {:3} not equal to result {:3} at byte: {}",
432 // original[idx], result[idx], idx,
433 //);
434 }
435 }
436 if counter > 0 {
437 panic!("Result differs in at least {} bytes from original", counter);
438 }
439
440 // Test resetting to a new file while keeping the old decoder
441
442 let content = include_bytes!("../../decodecorpus_files/z000068.zst");
443 let mut content = content.as_slice();
444 let mut stream = crate::streaming_decoder::StreamingDecoder::new_with_decoder(
445 &mut content,
446 stream.into_frame_decoder(),
447 )
448 .unwrap();
449
450 let original = include_bytes!("../../decodecorpus_files/z000068");
451 let mut result = vec![0; original.len()];
452 Read::read_exact(&mut stream, &mut result).unwrap();
453
454 std::println!("Results for file:");
455
456 if original.len() != result.len() {
457 panic!(
458 "Result has wrong length: {}, should be: {}",
459 result.len(),
460 original.len()
461 );
462 }
463
464 let mut counter = 0;
465 let min = if original.len() < result.len() {
466 original.len()
467 } else {
468 result.len()
469 };
470 for idx in 0..min {
471 if original[idx] != result[idx] {
472 counter += 1;
473 //std::println!(
474 // "Original {:3} not equal to result {:3} at byte: {}",
475 // original[idx], result[idx], idx,
476 //);
477 }
478 }
479 if counter > 0 {
480 panic!("Result differs in at least {} bytes from original", counter);
481 }
482}
483
484#[test]
485fn test_decode_all() {
486 use crate::frame_decoder::{FrameDecoder, FrameDecoderError};
487
488 let skip_frame = |input: &mut Vec<u8>, length: usize| {
489 input.extend_from_slice(&0x184D2A50u32.to_le_bytes());
490 input.extend_from_slice(&(length as u32).to_le_bytes());
491 input.resize(input.len() + length, 0);
492 };
493
494 let mut original = Vec::new();
495 let mut input = Vec::new();
496
497 skip_frame(&mut input, 300);
498 input.extend_from_slice(include_bytes!("../../decodecorpus_files/z000089.zst"));
499 original.extend_from_slice(include_bytes!("../../decodecorpus_files/z000089"));
500 skip_frame(&mut input, 400);
501 input.extend_from_slice(include_bytes!("../../decodecorpus_files/z000090.zst"));
502 original.extend_from_slice(include_bytes!("../../decodecorpus_files/z000090"));
503 skip_frame(&mut input, 500);
504
505 let mut decoder = FrameDecoder::new();
506
507 // decode_all with correct buffers.
508 let mut output = vec![0; original.len()];
509 let result = decoder.decode_all(&input, &mut output).unwrap();
510 assert_eq!(result, original.len());
511 assert_eq!(output, original);
512
513 // decode_all with smaller output length.
514 let mut output = vec![0; original.len() - 1];
515 let result = decoder.decode_all(&input, &mut output);
516 assert!(
517 matches!(result, Err(FrameDecoderError::TargetTooSmall)),
518 "{:?}",
519 result
520 );
521
522 // decode_all with larger output length.
523 let mut output = vec![0; original.len() + 1];
524 let result = decoder.decode_all(&input, &mut output).unwrap();
525 assert_eq!(result, original.len());
526 assert_eq!(&output[..result], original);
527
528 // decode_all with truncated regular frame.
529 let mut output = vec![0; original.len()];
530 let result = decoder.decode_all(&input[..input.len() - 600], &mut output);
531 assert!(
532 matches!(result, Err(FrameDecoderError::FailedToReadBlockBody(_))),
533 "{:?}",
534 result
535 );
536
537 // decode_all with truncated skip frame.
538 let mut output = vec![0; original.len()];
539 let result = decoder.decode_all(&input[..input.len() - 1], &mut output);
540 assert!(
541 matches!(result, Err(FrameDecoderError::FailedToSkipFrame)),
542 "{:?}",
543 result
544 );
545
546 // decode_all_to_vec with correct output capacity.
547 let mut output = Vec::new();
548 output.reserve_exact(original.len());
549 decoder.decode_all_to_vec(&input, &mut output).unwrap();
550 assert_eq!(output, original);
551
552 // decode_all_to_vec with smaller output capacity.
553 let mut output = Vec::new();
554 output.reserve_exact(original.len() - 1);
555 let result = decoder.decode_all_to_vec(&input, &mut output);
556 assert!(
557 matches!(result, Err(FrameDecoderError::TargetTooSmall)),
558 "{:?}",
559 result
560 );
561
562 // decode_all_to_vec with larger output capacity.
563 let mut output = Vec::new();
564 output.reserve_exact(original.len() + 1);
565 decoder.decode_all_to_vec(&input, &mut output).unwrap();
566 assert_eq!(output, original);
567}
568
569pub mod bit_reader;
570pub mod decode_corpus;
571pub mod dict_test;
572pub mod fuzz_regressions;
573