1#[test]
2fn test_dict_parsing() {
3 use crate::decoding::dictionary::Dictionary;
4 use alloc::vec;
5 let mut raw = vec![0u8; 8];
6
7 // correct magic num
8 raw[0] = 0x37;
9 raw[1] = 0xA4;
10 raw[2] = 0x30;
11 raw[3] = 0xEC;
12
13 //dict-id
14 let dict_id = 0x47232101;
15 raw[4] = 0x01;
16 raw[5] = 0x21;
17 raw[6] = 0x23;
18 raw[7] = 0x47;
19
20 // tables copied from ./dict_tests/dictionary
21 let raw_tables = &[
22 54, 16, 192, 155, 4, 0, 207, 59, 239, 121, 158, 116, 220, 93, 114, 229, 110, 41, 249, 95,
23 165, 255, 83, 202, 254, 68, 74, 159, 63, 161, 100, 151, 137, 21, 184, 183, 189, 100, 235,
24 209, 251, 174, 91, 75, 91, 185, 19, 39, 75, 146, 98, 177, 249, 14, 4, 35, 0, 0, 0, 40, 40,
25 20, 10, 12, 204, 37, 196, 1, 173, 122, 0, 4, 0, 128, 1, 2, 2, 25, 32, 27, 27, 22, 24, 26,
26 18, 12, 12, 15, 16, 11, 69, 37, 225, 48, 20, 12, 6, 2, 161, 80, 40, 20, 44, 137, 145, 204,
27 46, 0, 0, 0, 0, 0, 116, 253, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
28 ];
29 raw.extend(&raw_tables[..]);
30
31 //offset history 3,10,0x00ABCDEF
32 raw.extend(vec![3, 0, 0, 0]);
33 raw.extend(vec![10, 0, 0, 0]);
34 raw.extend(vec![0xEF, 0xCD, 0xAB, 0]);
35
36 //just some random bytes
37 let raw_content = vec![
38 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 123, 3, 234, 23, 234, 34, 23, 234, 34, 34, 234, 234,
39 ];
40 raw.extend(&raw_content);
41
42 let dict = Dictionary::decode_dict(&raw).unwrap();
43
44 if dict.id != dict_id {
45 panic!(
46 "Dict-id did not get parsed correctly. Is: {}, Should be: {}",
47 dict.id, dict_id
48 );
49 }
50
51 if !dict.dict_content.eq(&raw_content) {
52 panic!(
53 "dict content did not get parsed correctly. Is: {:?}, Should be: {:?}",
54 dict.dict_content, raw_content
55 );
56 }
57
58 if !dict.offset_hist.eq(&[3, 10, 0x00ABCDEF]) {
59 panic!(
60 "offset history did not get parsed correctly. Is: {:?}, Should be: {:?}",
61 dict.offset_hist,
62 [3, 10, 0x00ABCDEF]
63 );
64 }
65
66 // test magic num checking
67 raw[0] = 1;
68 raw[1] = 1;
69 raw[2] = 1;
70 raw[3] = 1;
71 match Dictionary::decode_dict(&raw) {
72 Ok(_) => panic!("The dict got decoded but the magic num was incorrect!"),
73 Err(_) => { /* This is what should happen*/ }
74 }
75}
76
77#[test]
78fn test_dict_decoding() {
79 extern crate std;
80 use crate::frame_decoder;
81 use alloc::borrow::ToOwned;
82 use alloc::string::{String, ToString};
83 use alloc::vec::Vec;
84 use std::fs;
85 use std::io::Read;
86 use std::println;
87
88 let mut success_counter = 0;
89 let mut fail_counter_diff = 0;
90 let mut fail_counter_size = 0;
91 let mut fail_counter_bytes_read = 0;
92 let mut total_counter = 0;
93 let mut failed: Vec<String> = Vec::new();
94
95 let mut speeds = Vec::new();
96 let mut speeds_read = Vec::new();
97
98 let mut files: Vec<_> = fs::read_dir("./dict_tests/files").unwrap().collect();
99 let dict = fs::File::open("./dict_tests/dictionary").unwrap();
100 let dict: Vec<u8> = dict.bytes().map(|x| x.unwrap()).collect();
101
102 files.sort_by_key(|x| match x {
103 Err(_) => "".to_owned(),
104 Ok(entry) => entry.path().to_str().unwrap().to_owned(),
105 });
106
107 let mut frame_dec = frame_decoder::FrameDecoder::new();
108 let dict = crate::decoding::dictionary::Dictionary::decode_dict(&dict).unwrap();
109 frame_dec.add_dict(dict).unwrap();
110
111 for file in files {
112 let f = file.unwrap();
113 let metadata = f.metadata().unwrap();
114 let file_size = metadata.len();
115
116 let p = String::from(f.path().to_str().unwrap());
117 if !p.ends_with(".zst") {
118 continue;
119 }
120 println!("Trying file: {}", p);
121
122 let mut content = fs::File::open(f.path()).unwrap();
123
124 frame_dec.reset(&mut content).unwrap();
125
126 let start_time = std::time::Instant::now();
127 /////DECODING
128 frame_dec
129 .decode_blocks(&mut content, frame_decoder::BlockDecodingStrategy::All)
130 .unwrap();
131 let result = frame_dec.collect().unwrap();
132 let end_time = start_time.elapsed();
133
134 match frame_dec.get_checksum_from_data() {
135 Some(chksum) => {
136 if frame_dec.get_calculated_checksum().unwrap() != chksum {
137 println!(
138 "Checksum did not match! From data: {}, calculated while decoding: {}\n",
139 chksum,
140 frame_dec.get_calculated_checksum().unwrap()
141 );
142 } else {
143 println!("Checksums are ok!\n");
144 }
145 }
146 None => println!("No checksums to test\n"),
147 }
148
149 let mut original_p = p.clone();
150 original_p.truncate(original_p.len() - 4);
151 let original_f = fs::File::open(original_p).unwrap();
152 let original: Vec<u8> = original_f.bytes().map(|x| x.unwrap()).collect();
153
154 println!("Results for file: {}", p.clone());
155 let mut success = true;
156
157 if original.len() != result.len() {
158 println!(
159 "Result has wrong length: {}, should be: {}",
160 result.len(),
161 original.len()
162 );
163 success = false;
164 fail_counter_size += 1;
165 }
166
167 if frame_dec.bytes_read_from_source() != file_size {
168 println!(
169 "Framedecoder counted wrong amount of bytes: {}, should be: {}",
170 frame_dec.bytes_read_from_source(),
171 file_size
172 );
173 success = false;
174 fail_counter_bytes_read += 1;
175 }
176
177 let mut counter = 0;
178 let min = if original.len() < result.len() {
179 original.len()
180 } else {
181 result.len()
182 };
183 for idx in 0..min {
184 if original[idx] != result[idx] {
185 counter += 1;
186 //println!(
187 // "Original {} not equal to result {} at byte: {}",
188 // original[idx], result[idx], idx,
189 //);
190 }
191 }
192
193 if counter > 0 {
194 println!("Result differs in at least {} bytes from original", counter);
195 success = false;
196 fail_counter_diff += 1;
197 }
198
199 if success {
200 success_counter += 1;
201 } else {
202 failed.push(p.clone().to_string());
203 }
204 total_counter += 1;
205
206 let dur = end_time.as_micros() as usize;
207 let speed = result.len() / if dur == 0 { 1 } else { dur };
208 let speed_read = file_size as usize / if dur == 0 { 1 } else { dur };
209 println!("SPEED: {}", speed);
210 println!("SPEED_read: {}", speed_read);
211 speeds.push(speed);
212 speeds_read.push(speed_read);
213 }
214
215 println!("###################");
216 println!("Summary:");
217 println!("###################");
218 println!(
219 "Total: {}, Success: {}, WrongSize: {}, WrongBytecount: {}, Diffs: {}",
220 total_counter,
221 success_counter,
222 fail_counter_size,
223 fail_counter_bytes_read,
224 fail_counter_diff
225 );
226 println!("Failed files: ");
227 for f in &failed {
228 println!("{}", f);
229 }
230
231 let speed_len = speeds.len();
232 let sum_speed: usize = speeds.into_iter().sum();
233 let avg_speed = sum_speed / speed_len;
234 let avg_speed_bps = avg_speed * 1_000_000;
235 if avg_speed_bps < 1000 {
236 println!("Average speed: {} B/s", avg_speed_bps);
237 } else if avg_speed_bps < 1_000_000 {
238 println!("Average speed: {} KB/s", avg_speed_bps / 1000);
239 } else {
240 println!("Average speed: {} MB/s", avg_speed_bps / 1_000_000);
241 }
242
243 let speed_read_len = speeds_read.len();
244 let sum_speed_read: usize = speeds_read.into_iter().sum();
245 let avg_speed_read = sum_speed_read / speed_read_len;
246 let avg_speed_read_bps = avg_speed_read * 1_000_000;
247 if avg_speed_read_bps < 1000 {
248 println!("Average speed reading: {} B/s", avg_speed_read_bps);
249 } else if avg_speed_bps < 1_000_000 {
250 println!("Average speed reading: {} KB/s", avg_speed_read_bps / 1000);
251 } else {
252 println!(
253 "Average speed reading: {} MB/s",
254 avg_speed_read_bps / 1_000_000
255 );
256 }
257
258 assert!(failed.is_empty());
259}
260