1#[test]
2fn test_dict_parsing() {
3 use crate::decoding::dictionary::Dictionary;
4 use alloc::vec;
5 let mut raw = vec![0u8; 8];
6
7 // correct magic num
8 raw[0] = 0x37;
9 raw[1] = 0xA4;
10 raw[2] = 0x30;
11 raw[3] = 0xEC;
12
13 //dict-id
14 let dict_id = 0x47232101;
15 raw[4] = 0x01;
16 raw[5] = 0x21;
17 raw[6] = 0x23;
18 raw[7] = 0x47;
19
20 // tables copied from ./dict_tests/dictionary
21 let raw_tables = &[
22 54, 16, 192, 155, 4, 0, 207, 59, 239, 121, 158, 116, 220, 93, 114, 229, 110, 41, 249, 95,
23 165, 255, 83, 202, 254, 68, 74, 159, 63, 161, 100, 151, 137, 21, 184, 183, 189, 100, 235,
24 209, 251, 174, 91, 75, 91, 185, 19, 39, 75, 146, 98, 177, 249, 14, 4, 35, 0, 0, 0, 40, 40,
25 20, 10, 12, 204, 37, 196, 1, 173, 122, 0, 4, 0, 128, 1, 2, 2, 25, 32, 27, 27, 22, 24, 26,
26 18, 12, 12, 15, 16, 11, 69, 37, 225, 48, 20, 12, 6, 2, 161, 80, 40, 20, 44, 137, 145, 204,
27 46, 0, 0, 0, 0, 0, 116, 253, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
28 ];
29 raw.extend(&raw_tables[..]);
30
31 //offset history 3,10,0x00ABCDEF
32 raw.extend(vec![3, 0, 0, 0]);
33 raw.extend(vec![10, 0, 0, 0]);
34 raw.extend(vec![0xEF, 0xCD, 0xAB, 0]);
35
36 //just some random bytes
37 let raw_content = vec![
38 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 123, 3, 234, 23, 234, 34, 23, 234, 34, 34, 234, 234,
39 ];
40 raw.extend(&raw_content);
41
42 let dict = Dictionary::decode_dict(&raw).unwrap();
43
44 if dict.id != dict_id {
45 panic!(
46 "Dict-id did not get parsed correctly. Is: {}, Should be: {}",
47 dict.id, dict_id
48 );
49 }
50
51 if !dict.dict_content.eq(&raw_content) {
52 panic!(
53 "dict content did not get parsed correctly. Is: {:?}, Should be: {:?}",
54 dict.dict_content, raw_content
55 );
56 }
57
58 if !dict.offset_hist.eq(&[3, 10, 0x00ABCDEF]) {
59 panic!(
60 "offset history did not get parsed correctly. Is: {:?}, Should be: {:?}",
61 dict.offset_hist,
62 [3, 10, 0x00ABCDEF]
63 );
64 }
65
66 // test magic num checking
67 raw[0] = 1;
68 raw[1] = 1;
69 raw[2] = 1;
70 raw[3] = 1;
71 match Dictionary::decode_dict(&raw) {
72 Ok(_) => panic!("The dict got decoded but the magic num was incorrect!"),
73 Err(_) => { /* This is what should happen*/ }
74 }
75}
76
77#[test]
78fn test_dict_decoding() {
79 extern crate std;
80 use crate::frame_decoder;
81 use alloc::borrow::ToOwned;
82 use alloc::string::{String, ToString};
83 use alloc::vec::Vec;
84 use std::fs;
85 use std::io::Read;
86 use std::println;
87
88 let mut success_counter = 0;
89 let mut fail_counter_diff = 0;
90 let mut fail_counter_size = 0;
91 let mut fail_counter_bytes_read = 0;
92 let mut total_counter = 0;
93 let mut failed: Vec<String> = Vec::new();
94
95 let mut speeds = Vec::new();
96 let mut speeds_read = Vec::new();
97
98 let mut files: Vec<_> = fs::read_dir("./dict_tests/files").unwrap().collect();
99 let dict = fs::File::open("./dict_tests/dictionary").unwrap();
100 let dict: Vec<u8> = dict.bytes().map(|x| x.unwrap()).collect();
101
102 files.sort_by_key(|x| match x {
103 Err(_) => "".to_owned(),
104 Ok(entry) => entry.path().to_str().unwrap().to_owned(),
105 });
106
107 let mut frame_dec = frame_decoder::FrameDecoder::new();
108 let dict = crate::decoding::dictionary::Dictionary::decode_dict(&dict).unwrap();
109 frame_dec.add_dict(dict).unwrap();
110
111 for file in files {
112 let f = file.unwrap();
113 let metadata = f.metadata().unwrap();
114 let file_size = metadata.len();
115
116 let p = String::from(f.path().to_str().unwrap());
117 if !p.ends_with(".zst") {
118 continue;
119 }
120 println!("Trying file: {}", p);
121
122 let mut content = fs::File::open(f.path()).unwrap();
123
124 frame_dec.reset(&mut content).unwrap();
125
126 let start_time = std::time::Instant::now();
127 /////DECODING
128 frame_dec
129 .decode_blocks(&mut content, frame_decoder::BlockDecodingStrategy::All)
130 .unwrap();
131 let result = frame_dec.collect().unwrap();
132 let end_time = start_time.elapsed();
133
134 match frame_dec.get_checksum_from_data() {
135 Some(chksum) => {
136 #[cfg(feature = "hash")]
137 if frame_dec.get_calculated_checksum().unwrap() != chksum {
138 println!(
139 "Checksum did not match! From data: {}, calculated while decoding: {}\n",
140 chksum,
141 frame_dec.get_calculated_checksum().unwrap()
142 );
143 } else {
144 println!("Checksums are ok!\n");
145 }
146 #[cfg(not(feature = "hash"))]
147 println!(
148 "Checksum feature not enabled, skipping. From data: {}\n",
149 chksum
150 );
151 }
152 None => println!("No checksums to test\n"),
153 }
154
155 let mut original_p = p.clone();
156 original_p.truncate(original_p.len() - 4);
157 let original_f = fs::File::open(original_p).unwrap();
158 let original: Vec<u8> = original_f.bytes().map(|x| x.unwrap()).collect();
159
160 println!("Results for file: {}", p.clone());
161 let mut success = true;
162
163 if original.len() != result.len() {
164 println!(
165 "Result has wrong length: {}, should be: {}",
166 result.len(),
167 original.len()
168 );
169 success = false;
170 fail_counter_size += 1;
171 }
172
173 if frame_dec.bytes_read_from_source() != file_size {
174 println!(
175 "Framedecoder counted wrong amount of bytes: {}, should be: {}",
176 frame_dec.bytes_read_from_source(),
177 file_size
178 );
179 success = false;
180 fail_counter_bytes_read += 1;
181 }
182
183 let mut counter = 0;
184 let min = if original.len() < result.len() {
185 original.len()
186 } else {
187 result.len()
188 };
189 for idx in 0..min {
190 if original[idx] != result[idx] {
191 counter += 1;
192 //println!(
193 // "Original {} not equal to result {} at byte: {}",
194 // original[idx], result[idx], idx,
195 //);
196 }
197 }
198
199 if counter > 0 {
200 println!("Result differs in at least {} bytes from original", counter);
201 success = false;
202 fail_counter_diff += 1;
203 }
204
205 if success {
206 success_counter += 1;
207 } else {
208 failed.push(p.clone().to_string());
209 }
210 total_counter += 1;
211
212 let dur = end_time.as_micros() as usize;
213 let speed = result.len() / if dur == 0 { 1 } else { dur };
214 let speed_read = file_size as usize / if dur == 0 { 1 } else { dur };
215 println!("SPEED: {}", speed);
216 println!("SPEED_read: {}", speed_read);
217 speeds.push(speed);
218 speeds_read.push(speed_read);
219 }
220
221 println!("###################");
222 println!("Summary:");
223 println!("###################");
224 println!(
225 "Total: {}, Success: {}, WrongSize: {}, WrongBytecount: {}, Diffs: {}",
226 total_counter,
227 success_counter,
228 fail_counter_size,
229 fail_counter_bytes_read,
230 fail_counter_diff
231 );
232 println!("Failed files: ");
233 for f in &failed {
234 println!("{}", f);
235 }
236
237 let speed_len = speeds.len();
238 let sum_speed: usize = speeds.into_iter().sum();
239 let avg_speed = sum_speed / speed_len;
240 let avg_speed_bps = avg_speed * 1_000_000;
241 if avg_speed_bps < 1000 {
242 println!("Average speed: {} B/s", avg_speed_bps);
243 } else if avg_speed_bps < 1_000_000 {
244 println!("Average speed: {} KB/s", avg_speed_bps / 1000);
245 } else {
246 println!("Average speed: {} MB/s", avg_speed_bps / 1_000_000);
247 }
248
249 let speed_read_len = speeds_read.len();
250 let sum_speed_read: usize = speeds_read.into_iter().sum();
251 let avg_speed_read = sum_speed_read / speed_read_len;
252 let avg_speed_read_bps = avg_speed_read * 1_000_000;
253 if avg_speed_read_bps < 1000 {
254 println!("Average speed reading: {} B/s", avg_speed_read_bps);
255 } else if avg_speed_bps < 1_000_000 {
256 println!("Average speed reading: {} KB/s", avg_speed_read_bps / 1000);
257 } else {
258 println!(
259 "Average speed reading: {} MB/s",
260 avg_speed_read_bps / 1_000_000
261 );
262 }
263
264 assert!(failed.is_empty());
265}
266