1#[test]
2fn test_decode_corpus_files() {
3 extern crate std;
4 use crate::frame_decoder;
5 use alloc::borrow::ToOwned;
6 use alloc::string::{String, ToString};
7 use alloc::vec::Vec;
8 use std::fs;
9 use std::io::Read;
10 use std::println;
11
12 let mut success_counter = 0;
13 let mut fail_counter_diff = 0;
14 let mut fail_counter_size = 0;
15 let mut fail_counter_bytes_read = 0;
16 let mut fail_counter_chksum = 0;
17 let mut total_counter = 0;
18 let mut failed: Vec<String> = Vec::new();
19
20 let mut speeds = Vec::new();
21 let mut speeds_read = Vec::new();
22
23 let mut files: Vec<_> = fs::read_dir("./decodecorpus_files").unwrap().collect();
24 if fs::read_dir("./local_corpus_files").is_ok() {
25 files.extend(fs::read_dir("./local_corpus_files").unwrap());
26 }
27
28 files.sort_by_key(|x| match x {
29 Err(_) => "".to_owned(),
30 Ok(entry) => entry.path().to_str().unwrap().to_owned(),
31 });
32
33 let mut frame_dec = frame_decoder::FrameDecoder::new();
34
35 for file in files {
36 let f = file.unwrap();
37 let metadata = f.metadata().unwrap();
38 let file_size = metadata.len();
39
40 let p = String::from(f.path().to_str().unwrap());
41 if !p.ends_with(".zst") {
42 continue;
43 }
44 println!("Trying file: {}", p);
45
46 let mut content = fs::File::open(f.path()).unwrap();
47
48 frame_dec.reset(&mut content).unwrap();
49
50 let start_time = std::time::Instant::now();
51 /////DECODING
52 frame_dec
53 .decode_blocks(&mut content, frame_decoder::BlockDecodingStrategy::All)
54 .unwrap();
55 let result = frame_dec.collect().unwrap();
56 let end_time = start_time.elapsed();
57
58 match frame_dec.get_checksum_from_data() {
59 Some(chksum) => {
60 if frame_dec.get_calculated_checksum().unwrap() != chksum {
61 println!(
62 "Checksum did not match! From data: {}, calculated while decoding: {}\n",
63 chksum,
64 frame_dec.get_calculated_checksum().unwrap()
65 );
66 fail_counter_chksum += 1;
67 failed.push(p.clone().to_string());
68 } else {
69 println!("Checksums are ok!\n");
70 }
71 }
72 None => println!("No checksums to test\n"),
73 }
74
75 let mut original_p = p.clone();
76 original_p.truncate(original_p.len() - 4);
77 let original_f = fs::File::open(original_p).unwrap();
78 let original: Vec<u8> = original_f.bytes().map(|x| x.unwrap()).collect();
79
80 println!("Results for file: {}", p.clone());
81 let mut success = true;
82
83 if original.len() != result.len() {
84 println!(
85 "Result has wrong length: {}, should be: {}",
86 result.len(),
87 original.len()
88 );
89 success = false;
90 fail_counter_size += 1;
91 }
92
93 if frame_dec.bytes_read_from_source() != file_size {
94 println!(
95 "Framedecoder counted wrong amount of bytes: {}, should be: {}",
96 frame_dec.bytes_read_from_source(),
97 file_size
98 );
99 success = false;
100 fail_counter_bytes_read += 1;
101 }
102
103 let mut counter = 0;
104 let min = if original.len() < result.len() {
105 original.len()
106 } else {
107 result.len()
108 };
109 for idx in 0..min {
110 if original[idx] != result[idx] {
111 counter += 1;
112 //println!(
113 // "Original {} not equal to result {} at byte: {}",
114 // original[idx], result[idx], idx,
115 //);
116 }
117 }
118
119 if counter > 0 {
120 println!("Result differs in at least {} bytes from original", counter);
121 success = false;
122 fail_counter_diff += 1;
123 }
124
125 if success {
126 success_counter += 1;
127 } else {
128 failed.push(p.clone().to_string());
129 }
130 total_counter += 1;
131
132 let dur = end_time.as_micros() as usize;
133 let speed = result.len() / if dur == 0 { 1 } else { dur };
134 let speed_read = file_size as usize / if dur == 0 { 1 } else { dur };
135 println!("SPEED: {}", speed);
136 println!("SPEED_read: {}", speed_read);
137 speeds.push(speed);
138 speeds_read.push(speed_read);
139 }
140
141 println!("###################");
142 println!("Summary:");
143 println!("###################");
144 println!(
145 "Total: {}, Success: {}, WrongSize: {}, WrongBytecount: {}, WrongChecksum: {}, Diffs: {}",
146 total_counter,
147 success_counter,
148 fail_counter_size,
149 fail_counter_bytes_read,
150 fail_counter_chksum,
151 fail_counter_diff
152 );
153 println!("Failed files: ");
154 for f in &failed {
155 println!("{}", f);
156 }
157
158 let speed_len = speeds.len();
159 let sum_speed: usize = speeds.into_iter().sum();
160 let avg_speed = sum_speed / speed_len;
161 let avg_speed_bps = avg_speed * 1_000_000;
162 if avg_speed_bps < 1000 {
163 println!("Average speed: {} B/s", avg_speed_bps);
164 } else if avg_speed_bps < 1_000_000 {
165 println!("Average speed: {} KB/s", avg_speed_bps / 1000);
166 } else {
167 println!("Average speed: {} MB/s", avg_speed_bps / 1_000_000);
168 }
169
170 let speed_read_len = speeds_read.len();
171 let sum_speed_read: usize = speeds_read.into_iter().sum();
172 let avg_speed_read = sum_speed_read / speed_read_len;
173 let avg_speed_read_bps = avg_speed_read * 1_000_000;
174 if avg_speed_read_bps < 1000 {
175 println!("Average speed reading: {} B/s", avg_speed_read_bps);
176 } else if avg_speed_bps < 1_000_000 {
177 println!("Average speed reading: {} KB/s", avg_speed_read_bps / 1000);
178 } else {
179 println!(
180 "Average speed reading: {} MB/s",
181 avg_speed_read_bps / 1_000_000
182 );
183 }
184
185 assert!(failed.is_empty());
186}
187