1#[test]
2fn test_decode_corpus_files() {
3 extern crate std;
4 use crate::frame_decoder;
5 use alloc::borrow::ToOwned;
6 use alloc::string::{String, ToString};
7 use alloc::vec::Vec;
8 use std::fs;
9 use std::io::Read;
10 use std::println;
11
12 let mut success_counter = 0;
13 let mut fail_counter_diff = 0;
14 let mut fail_counter_size = 0;
15 let mut fail_counter_bytes_read = 0;
16 #[cfg_attr(not(feature = "hash"), allow(unused_mut))]
17 let mut fail_counter_chksum = 0;
18 let mut total_counter = 0;
19 let mut failed: Vec<String> = Vec::new();
20
21 let mut speeds = Vec::new();
22 let mut speeds_read = Vec::new();
23
24 let mut files: Vec<_> = fs::read_dir("./decodecorpus_files").unwrap().collect();
25 if fs::read_dir("./local_corpus_files").is_ok() {
26 files.extend(fs::read_dir("./local_corpus_files").unwrap());
27 }
28
29 files.sort_by_key(|x| match x {
30 Err(_) => "".to_owned(),
31 Ok(entry) => entry.path().to_str().unwrap().to_owned(),
32 });
33
34 let mut frame_dec = frame_decoder::FrameDecoder::new();
35
36 for file in files {
37 let f = file.unwrap();
38 let metadata = f.metadata().unwrap();
39 let file_size = metadata.len();
40
41 let p = String::from(f.path().to_str().unwrap());
42 if !p.ends_with(".zst") {
43 continue;
44 }
45 println!("Trying file: {}", p);
46
47 let mut content = fs::File::open(f.path()).unwrap();
48
49 frame_dec.reset(&mut content).unwrap();
50
51 let start_time = std::time::Instant::now();
52 /////DECODING
53 frame_dec
54 .decode_blocks(&mut content, frame_decoder::BlockDecodingStrategy::All)
55 .unwrap();
56 let result = frame_dec.collect().unwrap();
57 let end_time = start_time.elapsed();
58
59 match frame_dec.get_checksum_from_data() {
60 Some(chksum) => {
61 #[cfg(feature = "hash")]
62 if frame_dec.get_calculated_checksum().unwrap() != chksum {
63 println!(
64 "Checksum did not match! From data: {}, calculated while decoding: {}\n",
65 chksum,
66 frame_dec.get_calculated_checksum().unwrap()
67 );
68 fail_counter_chksum += 1;
69 failed.push(p.clone().to_string());
70 } else {
71 println!("Checksums are ok!\n");
72 }
73 #[cfg(not(feature = "hash"))]
74 println!(
75 "Checksum feature not enabled, skipping. From data: {}\n",
76 chksum
77 );
78 }
79 None => println!("No checksums to test\n"),
80 }
81
82 let mut original_p = p.clone();
83 original_p.truncate(original_p.len() - 4);
84 let original_f = fs::File::open(original_p).unwrap();
85 let original: Vec<u8> = original_f.bytes().map(|x| x.unwrap()).collect();
86
87 println!("Results for file: {}", p.clone());
88 let mut success = true;
89
90 if original.len() != result.len() {
91 println!(
92 "Result has wrong length: {}, should be: {}",
93 result.len(),
94 original.len()
95 );
96 success = false;
97 fail_counter_size += 1;
98 }
99
100 if frame_dec.bytes_read_from_source() != file_size {
101 println!(
102 "Framedecoder counted wrong amount of bytes: {}, should be: {}",
103 frame_dec.bytes_read_from_source(),
104 file_size
105 );
106 success = false;
107 fail_counter_bytes_read += 1;
108 }
109
110 let mut counter = 0;
111 let min = if original.len() < result.len() {
112 original.len()
113 } else {
114 result.len()
115 };
116 for idx in 0..min {
117 if original[idx] != result[idx] {
118 counter += 1;
119 //println!(
120 // "Original {} not equal to result {} at byte: {}",
121 // original[idx], result[idx], idx,
122 //);
123 }
124 }
125
126 if counter > 0 {
127 println!("Result differs in at least {} bytes from original", counter);
128 success = false;
129 fail_counter_diff += 1;
130 }
131
132 if success {
133 success_counter += 1;
134 } else {
135 failed.push(p.clone().to_string());
136 }
137 total_counter += 1;
138
139 let dur = end_time.as_micros() as usize;
140 let speed = result.len() / if dur == 0 { 1 } else { dur };
141 let speed_read = file_size as usize / if dur == 0 { 1 } else { dur };
142 println!("SPEED: {}", speed);
143 println!("SPEED_read: {}", speed_read);
144 speeds.push(speed);
145 speeds_read.push(speed_read);
146 }
147
148 println!("###################");
149 println!("Summary:");
150 println!("###################");
151 println!(
152 "Total: {}, Success: {}, WrongSize: {}, WrongBytecount: {}, WrongChecksum: {}, Diffs: {}",
153 total_counter,
154 success_counter,
155 fail_counter_size,
156 fail_counter_bytes_read,
157 fail_counter_chksum,
158 fail_counter_diff
159 );
160 println!("Failed files: ");
161 for f in &failed {
162 println!("{}", f);
163 }
164
165 let speed_len = speeds.len();
166 let sum_speed: usize = speeds.into_iter().sum();
167 let avg_speed = sum_speed / speed_len;
168 let avg_speed_bps = avg_speed * 1_000_000;
169 if avg_speed_bps < 1000 {
170 println!("Average speed: {} B/s", avg_speed_bps);
171 } else if avg_speed_bps < 1_000_000 {
172 println!("Average speed: {} KB/s", avg_speed_bps / 1000);
173 } else {
174 println!("Average speed: {} MB/s", avg_speed_bps / 1_000_000);
175 }
176
177 let speed_read_len = speeds_read.len();
178 let sum_speed_read: usize = speeds_read.into_iter().sum();
179 let avg_speed_read = sum_speed_read / speed_read_len;
180 let avg_speed_read_bps = avg_speed_read * 1_000_000;
181 if avg_speed_read_bps < 1000 {
182 println!("Average speed reading: {} B/s", avg_speed_read_bps);
183 } else if avg_speed_bps < 1_000_000 {
184 println!("Average speed reading: {} KB/s", avg_speed_read_bps / 1000);
185 } else {
186 println!(
187 "Average speed reading: {} MB/s",
188 avg_speed_read_bps / 1_000_000
189 );
190 }
191
192 assert!(failed.is_empty());
193}
194