1#[cfg(feature = "csv_output")]
2use crate::csv_report::FileCsvReport;
3use crate::stats::bivariate::regression::Slope;
4use crate::stats::univariate::outliers::tukey::LabeledSample;
5use crate::{html::Html, stats::bivariate::Data};
6
7use crate::estimate::{ChangeDistributions, ChangeEstimates, Distributions, Estimate, Estimates};
8use crate::format;
9use crate::measurement::ValueFormatter;
10use crate::stats::univariate::Sample;
11use crate::stats::Distribution;
12use crate::{PlotConfiguration, Throughput};
13use anes::{Attribute, ClearLine, Color, ResetAttributes, SetAttribute, SetForegroundColor};
14use std::cmp;
15use std::collections::HashSet;
16use std::fmt;
17use std::io::stderr;
18use std::io::Write;
19use std::path::{Path, PathBuf};
20
21const MAX_DIRECTORY_NAME_LEN: usize = 64;
22const MAX_TITLE_LEN: usize = 100;
23
24pub(crate) struct ComparisonData {
25 pub p_value: f64,
26 pub t_distribution: Distribution<f64>,
27 pub t_value: f64,
28 pub relative_estimates: ChangeEstimates,
29 pub relative_distributions: ChangeDistributions,
30 pub significance_threshold: f64,
31 pub noise_threshold: f64,
32 pub base_iter_counts: Vec<f64>,
33 pub base_sample_times: Vec<f64>,
34 pub base_avg_times: Vec<f64>,
35 pub base_estimates: Estimates,
36}
37
38pub(crate) struct MeasurementData<'a> {
39 pub data: Data<'a, f64, f64>,
40 pub avg_times: LabeledSample<'a, f64>,
41 pub absolute_estimates: Estimates,
42 pub distributions: Distributions,
43 pub comparison: Option<ComparisonData>,
44 pub throughput: Option<Throughput>,
45}
46impl<'a> MeasurementData<'a> {
47 pub fn iter_counts(&self) -> &Sample<f64> {
48 self.data.x()
49 }
50
51 #[cfg(feature = "csv_output")]
52 pub fn sample_times(&self) -> &Sample<f64> {
53 self.data.y()
54 }
55}
56
57#[derive(Debug, Clone, Copy, Eq, PartialEq)]
58pub enum ValueType {
59 Bytes,
60 Elements,
61 Value,
62}
63
64#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
65pub struct BenchmarkId {
66 pub group_id: String,
67 pub function_id: Option<String>,
68 pub value_str: Option<String>,
69 pub throughput: Option<Throughput>,
70 full_id: String,
71 directory_name: String,
72 title: String,
73}
74
75fn truncate_to_character_boundary(s: &mut String, max_len: usize) {
76 let mut boundary = cmp::min(max_len, s.len());
77 while !s.is_char_boundary(boundary) {
78 boundary -= 1;
79 }
80 s.truncate(boundary);
81}
82
83pub fn make_filename_safe(string: &str) -> String {
84 let mut string = string.replace(
85 &['?', '"', '/', '\\', '*', '<', '>', ':', '|', '^'][..],
86 "_",
87 );
88
89 // Truncate to last character boundary before max length...
90 truncate_to_character_boundary(&mut string, MAX_DIRECTORY_NAME_LEN);
91
92 if cfg!(target_os = "windows") {
93 {
94 string = string
95 // On Windows, spaces in the end of the filename are ignored and will be trimmed.
96 //
97 // Without trimming ourselves, creating a directory `dir ` will silently create
98 // `dir` instead, but then operations on files like `dir /file` will fail.
99 //
100 // Also note that it's important to do this *after* trimming to MAX_DIRECTORY_NAME_LEN,
101 // otherwise it can trim again to a name with a trailing space.
102 .trim_end()
103 // On Windows, file names are not case-sensitive, so lowercase everything.
104 .to_lowercase();
105 }
106 }
107
108 string
109}
110
111impl BenchmarkId {
112 pub fn new(
113 group_id: String,
114 function_id: Option<String>,
115 value_str: Option<String>,
116 throughput: Option<Throughput>,
117 ) -> BenchmarkId {
118 let full_id = match (&function_id, &value_str) {
119 (Some(func), Some(val)) => format!("{}/{}/{}", group_id, func, val),
120 (Some(func), &None) => format!("{}/{}", group_id, func),
121 (&None, Some(val)) => format!("{}/{}", group_id, val),
122 (&None, &None) => group_id.clone(),
123 };
124
125 let mut title = full_id.clone();
126 truncate_to_character_boundary(&mut title, MAX_TITLE_LEN);
127 if title != full_id {
128 title.push_str("...");
129 }
130
131 let directory_name = match (&function_id, &value_str) {
132 (Some(func), Some(val)) => format!(
133 "{}/{}/{}",
134 make_filename_safe(&group_id),
135 make_filename_safe(func),
136 make_filename_safe(val)
137 ),
138 (Some(func), &None) => format!(
139 "{}/{}",
140 make_filename_safe(&group_id),
141 make_filename_safe(func)
142 ),
143 (&None, Some(val)) => format!(
144 "{}/{}",
145 make_filename_safe(&group_id),
146 make_filename_safe(val)
147 ),
148 (&None, &None) => make_filename_safe(&group_id),
149 };
150
151 BenchmarkId {
152 group_id,
153 function_id,
154 value_str,
155 throughput,
156 full_id,
157 directory_name,
158 title,
159 }
160 }
161
162 pub fn id(&self) -> &str {
163 &self.full_id
164 }
165
166 pub fn as_title(&self) -> &str {
167 &self.title
168 }
169
170 pub fn as_directory_name(&self) -> &str {
171 &self.directory_name
172 }
173
174 pub fn as_number(&self) -> Option<f64> {
175 match self.throughput {
176 Some(Throughput::Bytes(n))
177 | Some(Throughput::Elements(n))
178 | Some(Throughput::BytesDecimal(n)) => Some(n as f64),
179 None => self
180 .value_str
181 .as_ref()
182 .and_then(|string| string.parse::<f64>().ok()),
183 }
184 }
185
186 pub fn value_type(&self) -> Option<ValueType> {
187 match self.throughput {
188 Some(Throughput::Bytes(_)) => Some(ValueType::Bytes),
189 Some(Throughput::BytesDecimal(_)) => Some(ValueType::Bytes),
190 Some(Throughput::Elements(_)) => Some(ValueType::Elements),
191 None => self
192 .value_str
193 .as_ref()
194 .and_then(|string| string.parse::<f64>().ok())
195 .map(|_| ValueType::Value),
196 }
197 }
198
199 pub fn ensure_directory_name_unique(&mut self, existing_directories: &HashSet<String>) {
200 if !existing_directories.contains(self.as_directory_name()) {
201 return;
202 }
203
204 let mut counter = 2;
205 loop {
206 let new_dir_name = format!("{}_{}", self.as_directory_name(), counter);
207 if !existing_directories.contains(&new_dir_name) {
208 self.directory_name = new_dir_name;
209 return;
210 }
211 counter += 1;
212 }
213 }
214
215 pub fn ensure_title_unique(&mut self, existing_titles: &HashSet<String>) {
216 if !existing_titles.contains(self.as_title()) {
217 return;
218 }
219
220 let mut counter = 2;
221 loop {
222 let new_title = format!("{} #{}", self.as_title(), counter);
223 if !existing_titles.contains(&new_title) {
224 self.title = new_title;
225 return;
226 }
227 counter += 1;
228 }
229 }
230}
231impl fmt::Display for BenchmarkId {
232 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
233 f.write_str(self.as_title())
234 }
235}
236impl fmt::Debug for BenchmarkId {
237 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
238 fn format_opt(opt: &Option<String>) -> String {
239 match *opt {
240 Some(ref string) => format!("\"{}\"", string),
241 None => "None".to_owned(),
242 }
243 }
244
245 write!(
246 f,
247 "BenchmarkId {{ group_id: \"{}\", function_id: {}, value_str: {}, throughput: {:?} }}",
248 self.group_id,
249 format_opt(&self.function_id),
250 format_opt(&self.value_str),
251 self.throughput,
252 )
253 }
254}
255
256pub struct ReportContext {
257 pub output_directory: PathBuf,
258 pub plot_config: PlotConfiguration,
259}
260impl ReportContext {
261 pub fn report_path<P: AsRef<Path> + ?Sized>(&self, id: &BenchmarkId, file_name: &P) -> PathBuf {
262 let mut path = self.output_directory.clone();
263 path.push(id.as_directory_name());
264 path.push("report");
265 path.push(file_name);
266 path
267 }
268}
269
270pub(crate) trait Report {
271 fn test_start(&self, _id: &BenchmarkId, _context: &ReportContext) {}
272 fn test_pass(&self, _id: &BenchmarkId, _context: &ReportContext) {}
273
274 fn benchmark_start(&self, _id: &BenchmarkId, _context: &ReportContext) {}
275 fn profile(&self, _id: &BenchmarkId, _context: &ReportContext, _profile_ns: f64) {}
276 fn warmup(&self, _id: &BenchmarkId, _context: &ReportContext, _warmup_ns: f64) {}
277 fn terminated(&self, _id: &BenchmarkId, _context: &ReportContext) {}
278 fn analysis(&self, _id: &BenchmarkId, _context: &ReportContext) {}
279 fn measurement_start(
280 &self,
281 _id: &BenchmarkId,
282 _context: &ReportContext,
283 _sample_count: u64,
284 _estimate_ns: f64,
285 _iter_count: u64,
286 ) {
287 }
288 fn measurement_complete(
289 &self,
290 _id: &BenchmarkId,
291 _context: &ReportContext,
292 _measurements: &MeasurementData<'_>,
293 _formatter: &dyn ValueFormatter,
294 ) {
295 }
296 fn summarize(
297 &self,
298 _context: &ReportContext,
299 _all_ids: &[BenchmarkId],
300 _formatter: &dyn ValueFormatter,
301 ) {
302 }
303 fn final_summary(&self, _context: &ReportContext) {}
304 fn group_separator(&self) {}
305}
306
307pub(crate) struct Reports {
308 pub(crate) cli_enabled: bool,
309 pub(crate) cli: CliReport,
310 pub(crate) bencher_enabled: bool,
311 pub(crate) bencher: BencherReport,
312 pub(crate) csv_enabled: bool,
313 pub(crate) html: Option<Html>,
314}
315macro_rules! reports_impl {
316 (fn $name:ident(&self, $($argn:ident: $argt:ty),*)) => {
317 fn $name(&self, $($argn: $argt),* ) {
318 if self.cli_enabled {
319 self.cli.$name($($argn),*);
320 }
321 if self.bencher_enabled {
322 self.bencher.$name($($argn),*);
323 }
324 #[cfg(feature = "csv_output")]
325 if self.csv_enabled {
326 FileCsvReport.$name($($argn),*);
327 }
328 if let Some(reporter) = &self.html {
329 reporter.$name($($argn),*);
330 }
331 }
332 };
333}
334
335impl Report for Reports {
336 reports_impl!(fn test_start(&self, id: &BenchmarkId, context: &ReportContext));
337 reports_impl!(fn test_pass(&self, id: &BenchmarkId, context: &ReportContext));
338 reports_impl!(fn benchmark_start(&self, id: &BenchmarkId, context: &ReportContext));
339 reports_impl!(fn profile(&self, id: &BenchmarkId, context: &ReportContext, profile_ns: f64));
340 reports_impl!(fn warmup(&self, id: &BenchmarkId, context: &ReportContext, warmup_ns: f64));
341 reports_impl!(fn terminated(&self, id: &BenchmarkId, context: &ReportContext));
342 reports_impl!(fn analysis(&self, id: &BenchmarkId, context: &ReportContext));
343 reports_impl!(fn measurement_start(
344 &self,
345 id: &BenchmarkId,
346 context: &ReportContext,
347 sample_count: u64,
348 estimate_ns: f64,
349 iter_count: u64
350 ));
351 reports_impl!(
352 fn measurement_complete(
353 &self,
354 id: &BenchmarkId,
355 context: &ReportContext,
356 measurements: &MeasurementData<'_>,
357 formatter: &dyn ValueFormatter
358 ));
359 reports_impl!(
360 fn summarize(
361 &self,
362 context: &ReportContext,
363 all_ids: &[BenchmarkId],
364 formatter: &dyn ValueFormatter
365 ));
366
367 reports_impl!(fn final_summary(&self, context: &ReportContext));
368 reports_impl!(fn group_separator(&self, ));
369}
370
371#[derive(Debug, Clone, Copy, Eq, PartialEq)]
372pub(crate) enum CliVerbosity {
373 Quiet,
374 Normal,
375 Verbose,
376}
377
378pub(crate) struct CliReport {
379 pub enable_text_overwrite: bool,
380 pub enable_text_coloring: bool,
381 pub verbosity: CliVerbosity,
382}
383impl CliReport {
384 pub fn new(
385 enable_text_overwrite: bool,
386 enable_text_coloring: bool,
387 verbosity: CliVerbosity,
388 ) -> CliReport {
389 CliReport {
390 enable_text_overwrite,
391 enable_text_coloring,
392 verbosity,
393 }
394 }
395
396 fn text_overwrite(&self) {
397 if self.enable_text_overwrite {
398 eprint!("\r{}", ClearLine::All)
399 }
400 }
401
402 // Passing a String is the common case here.
403 #[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_pass_by_value))]
404 fn print_overwritable(&self, s: String) {
405 if self.enable_text_overwrite {
406 eprint!("{}", s);
407 stderr().flush().unwrap();
408 } else {
409 eprintln!("{}", s);
410 }
411 }
412
413 fn with_color(&self, color: Color, s: &str) -> String {
414 if self.enable_text_coloring {
415 format!("{}{}{}", SetForegroundColor(color), s, ResetAttributes)
416 } else {
417 String::from(s)
418 }
419 }
420
421 fn green(&self, s: &str) -> String {
422 self.with_color(Color::DarkGreen, s)
423 }
424
425 fn yellow(&self, s: &str) -> String {
426 self.with_color(Color::DarkYellow, s)
427 }
428
429 fn red(&self, s: &str) -> String {
430 self.with_color(Color::DarkRed, s)
431 }
432
433 fn bold(&self, s: String) -> String {
434 if self.enable_text_coloring {
435 format!("{}{}{}", SetAttribute(Attribute::Bold), s, ResetAttributes)
436 } else {
437 s
438 }
439 }
440
441 fn faint(&self, s: String) -> String {
442 if self.enable_text_coloring {
443 format!("{}{}{}", SetAttribute(Attribute::Faint), s, ResetAttributes)
444 } else {
445 s
446 }
447 }
448
449 pub fn outliers(&self, sample: &LabeledSample<'_, f64>) {
450 let (los, lom, _, him, his) = sample.count();
451 let noutliers = los + lom + him + his;
452 let sample_size = sample.len();
453
454 if noutliers == 0 {
455 return;
456 }
457
458 let percent = |n: usize| 100. * n as f64 / sample_size as f64;
459
460 println!(
461 "{}",
462 self.yellow(&format!(
463 "Found {} outliers among {} measurements ({:.2}%)",
464 noutliers,
465 sample_size,
466 percent(noutliers)
467 ))
468 );
469
470 let print = |n, label| {
471 if n != 0 {
472 println!(" {} ({:.2}%) {}", n, percent(n), label);
473 }
474 };
475
476 print(los, "low severe");
477 print(lom, "low mild");
478 print(him, "high mild");
479 print(his, "high severe");
480 }
481}
482impl Report for CliReport {
483 fn test_start(&self, id: &BenchmarkId, _: &ReportContext) {
484 println!("Testing {}", id);
485 }
486 fn test_pass(&self, _: &BenchmarkId, _: &ReportContext) {
487 println!("Success");
488 }
489
490 fn benchmark_start(&self, id: &BenchmarkId, _: &ReportContext) {
491 self.print_overwritable(format!("Benchmarking {}", id));
492 }
493
494 fn profile(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64) {
495 self.text_overwrite();
496 self.print_overwritable(format!(
497 "Benchmarking {}: Profiling for {}",
498 id,
499 format::time(warmup_ns)
500 ));
501 }
502
503 fn warmup(&self, id: &BenchmarkId, _: &ReportContext, warmup_ns: f64) {
504 self.text_overwrite();
505 self.print_overwritable(format!(
506 "Benchmarking {}: Warming up for {}",
507 id,
508 format::time(warmup_ns)
509 ));
510 }
511
512 fn terminated(&self, id: &BenchmarkId, _: &ReportContext) {
513 self.text_overwrite();
514 println!("Benchmarking {}: Complete (Analysis Disabled)", id);
515 }
516
517 fn analysis(&self, id: &BenchmarkId, _: &ReportContext) {
518 self.text_overwrite();
519 self.print_overwritable(format!("Benchmarking {}: Analyzing", id));
520 }
521
522 fn measurement_start(
523 &self,
524 id: &BenchmarkId,
525 _: &ReportContext,
526 sample_count: u64,
527 estimate_ns: f64,
528 iter_count: u64,
529 ) {
530 self.text_overwrite();
531 let iter_string = if matches!(self.verbosity, CliVerbosity::Verbose) {
532 format!("{} iterations", iter_count)
533 } else {
534 format::iter_count(iter_count)
535 };
536
537 self.print_overwritable(format!(
538 "Benchmarking {}: Collecting {} samples in estimated {} ({})",
539 id,
540 sample_count,
541 format::time(estimate_ns),
542 iter_string
543 ));
544 }
545
546 fn measurement_complete(
547 &self,
548 id: &BenchmarkId,
549 _: &ReportContext,
550 meas: &MeasurementData<'_>,
551 formatter: &dyn ValueFormatter,
552 ) {
553 self.text_overwrite();
554
555 let typical_estimate = &meas.absolute_estimates.typical();
556
557 {
558 let mut id = id.as_title().to_owned();
559
560 if id.len() > 23 {
561 println!("{}", self.green(&id));
562 id.clear();
563 }
564 let id_len = id.len();
565
566 println!(
567 "{}{}time: [{} {} {}]",
568 self.green(&id),
569 " ".repeat(24 - id_len),
570 self.faint(
571 formatter.format_value(typical_estimate.confidence_interval.lower_bound)
572 ),
573 self.bold(formatter.format_value(typical_estimate.point_estimate)),
574 self.faint(
575 formatter.format_value(typical_estimate.confidence_interval.upper_bound)
576 )
577 );
578 }
579
580 if let Some(ref throughput) = meas.throughput {
581 println!(
582 "{}thrpt: [{} {} {}]",
583 " ".repeat(24),
584 self.faint(formatter.format_throughput(
585 throughput,
586 typical_estimate.confidence_interval.upper_bound
587 )),
588 self.bold(formatter.format_throughput(throughput, typical_estimate.point_estimate)),
589 self.faint(formatter.format_throughput(
590 throughput,
591 typical_estimate.confidence_interval.lower_bound
592 )),
593 )
594 }
595
596 if !matches!(self.verbosity, CliVerbosity::Quiet) {
597 if let Some(ref comp) = meas.comparison {
598 let different_mean = comp.p_value < comp.significance_threshold;
599 let mean_est = &comp.relative_estimates.mean;
600 let point_estimate = mean_est.point_estimate;
601 let mut point_estimate_str = format::change(point_estimate, true);
602 // The change in throughput is related to the change in timing. Reducing the timing by
603 // 50% increases the throughput by 100%.
604 let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0;
605 let mut thrpt_point_estimate_str =
606 format::change(to_thrpt_estimate(point_estimate), true);
607 let explanation_str: String;
608
609 if !different_mean {
610 explanation_str = "No change in performance detected.".to_owned();
611 } else {
612 let comparison = compare_to_threshold(mean_est, comp.noise_threshold);
613 match comparison {
614 ComparisonResult::Improved => {
615 point_estimate_str = self.green(&self.bold(point_estimate_str));
616 thrpt_point_estimate_str =
617 self.green(&self.bold(thrpt_point_estimate_str));
618 explanation_str =
619 format!("Performance has {}.", self.green("improved"));
620 }
621 ComparisonResult::Regressed => {
622 point_estimate_str = self.red(&self.bold(point_estimate_str));
623 thrpt_point_estimate_str =
624 self.red(&self.bold(thrpt_point_estimate_str));
625 explanation_str = format!("Performance has {}.", self.red("regressed"));
626 }
627 ComparisonResult::NonSignificant => {
628 explanation_str = "Change within noise threshold.".to_owned();
629 }
630 }
631 }
632
633 if meas.throughput.is_some() {
634 println!("{}change:", " ".repeat(17));
635
636 println!(
637 "{}time: [{} {} {}] (p = {:.2} {} {:.2})",
638 " ".repeat(24),
639 self.faint(format::change(
640 mean_est.confidence_interval.lower_bound,
641 true
642 )),
643 point_estimate_str,
644 self.faint(format::change(
645 mean_est.confidence_interval.upper_bound,
646 true
647 )),
648 comp.p_value,
649 if different_mean { "<" } else { ">" },
650 comp.significance_threshold
651 );
652 println!(
653 "{}thrpt: [{} {} {}]",
654 " ".repeat(24),
655 self.faint(format::change(
656 to_thrpt_estimate(mean_est.confidence_interval.upper_bound),
657 true
658 )),
659 thrpt_point_estimate_str,
660 self.faint(format::change(
661 to_thrpt_estimate(mean_est.confidence_interval.lower_bound),
662 true
663 )),
664 );
665 } else {
666 println!(
667 "{}change: [{} {} {}] (p = {:.2} {} {:.2})",
668 " ".repeat(24),
669 self.faint(format::change(
670 mean_est.confidence_interval.lower_bound,
671 true
672 )),
673 point_estimate_str,
674 self.faint(format::change(
675 mean_est.confidence_interval.upper_bound,
676 true
677 )),
678 comp.p_value,
679 if different_mean { "<" } else { ">" },
680 comp.significance_threshold
681 );
682 }
683
684 println!("{}{}", " ".repeat(24), explanation_str);
685 }
686 }
687
688 if !matches!(self.verbosity, CliVerbosity::Quiet) {
689 self.outliers(&meas.avg_times);
690 }
691
692 if matches!(self.verbosity, CliVerbosity::Verbose) {
693 let format_short_estimate = |estimate: &Estimate| -> String {
694 format!(
695 "[{} {}]",
696 formatter.format_value(estimate.confidence_interval.lower_bound),
697 formatter.format_value(estimate.confidence_interval.upper_bound)
698 )
699 };
700
701 let data = &meas.data;
702 if let Some(slope_estimate) = meas.absolute_estimates.slope.as_ref() {
703 println!(
704 "{:<7}{} {:<15}[{:0.7} {:0.7}]",
705 "slope",
706 format_short_estimate(slope_estimate),
707 "R^2",
708 Slope(slope_estimate.confidence_interval.lower_bound).r_squared(data),
709 Slope(slope_estimate.confidence_interval.upper_bound).r_squared(data),
710 );
711 }
712 println!(
713 "{:<7}{} {:<15}{}",
714 "mean",
715 format_short_estimate(&meas.absolute_estimates.mean),
716 "std. dev.",
717 format_short_estimate(&meas.absolute_estimates.std_dev),
718 );
719 println!(
720 "{:<7}{} {:<15}{}",
721 "median",
722 format_short_estimate(&meas.absolute_estimates.median),
723 "med. abs. dev.",
724 format_short_estimate(&meas.absolute_estimates.median_abs_dev),
725 );
726 }
727 }
728
729 fn group_separator(&self) {
730 println!();
731 }
732}
733
734pub struct BencherReport;
735impl Report for BencherReport {
736 fn measurement_start(
737 &self,
738 id: &BenchmarkId,
739 _context: &ReportContext,
740 _sample_count: u64,
741 _estimate_ns: f64,
742 _iter_count: u64,
743 ) {
744 print!("test {} ... ", id);
745 }
746
747 fn measurement_complete(
748 &self,
749 _id: &BenchmarkId,
750 _: &ReportContext,
751 meas: &MeasurementData<'_>,
752 formatter: &dyn ValueFormatter,
753 ) {
754 let mut values = [
755 meas.absolute_estimates.median.point_estimate,
756 meas.absolute_estimates.std_dev.point_estimate,
757 ];
758 let unit = formatter.scale_for_machines(&mut values);
759
760 println!(
761 "bench: {:>11} {}/iter (+/- {})",
762 format::integer(values[0]),
763 unit,
764 format::integer(values[1])
765 );
766 }
767
768 fn group_separator(&self) {
769 println!();
770 }
771}
772
773enum ComparisonResult {
774 Improved,
775 Regressed,
776 NonSignificant,
777}
778
779fn compare_to_threshold(estimate: &Estimate, noise: f64) -> ComparisonResult {
780 let ci = &estimate.confidence_interval;
781 let lb = ci.lower_bound;
782 let ub = ci.upper_bound;
783
784 if lb < -noise && ub < -noise {
785 ComparisonResult::Improved
786 } else if lb > noise && ub > noise {
787 ComparisonResult::Regressed
788 } else {
789 ComparisonResult::NonSignificant
790 }
791}
792
793#[cfg(test)]
794mod test {
795 use super::*;
796
797 #[test]
798 fn test_make_filename_safe_replaces_characters() {
799 let input = "?/\\*\"";
800 let safe = make_filename_safe(input);
801 assert_eq!("_____", &safe);
802 }
803
804 #[test]
805 fn test_make_filename_safe_truncates_long_strings() {
806 let input = "this is a very long string. it is too long to be safe as a directory name, and so it needs to be truncated. what a long string this is.";
807 let safe = make_filename_safe(input);
808 assert!(input.len() > MAX_DIRECTORY_NAME_LEN);
809 assert_eq!(&input[0..MAX_DIRECTORY_NAME_LEN], &safe);
810 }
811
812 #[test]
813 fn test_make_filename_safe_respects_character_boundaries() {
814 let input = "✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓";
815 let safe = make_filename_safe(input);
816 assert!(safe.len() < MAX_DIRECTORY_NAME_LEN);
817 }
818
819 #[test]
820 fn test_benchmark_id_make_directory_name_unique() {
821 let existing_id = BenchmarkId::new(
822 "group".to_owned(),
823 Some("function".to_owned()),
824 Some("value".to_owned()),
825 None,
826 );
827 let mut directories = HashSet::new();
828 directories.insert(existing_id.as_directory_name().to_owned());
829
830 let mut new_id = existing_id.clone();
831 new_id.ensure_directory_name_unique(&directories);
832 assert_eq!("group/function/value_2", new_id.as_directory_name());
833 directories.insert(new_id.as_directory_name().to_owned());
834
835 new_id = existing_id;
836 new_id.ensure_directory_name_unique(&directories);
837 assert_eq!("group/function/value_3", new_id.as_directory_name());
838 directories.insert(new_id.as_directory_name().to_owned());
839 }
840 #[test]
841 fn test_benchmark_id_make_long_directory_name_unique() {
842 let long_name = (0..MAX_DIRECTORY_NAME_LEN).map(|_| 'a').collect::<String>();
843 let existing_id = BenchmarkId::new(long_name, None, None, None);
844 let mut directories = HashSet::new();
845 directories.insert(existing_id.as_directory_name().to_owned());
846
847 let mut new_id = existing_id.clone();
848 new_id.ensure_directory_name_unique(&directories);
849 assert_ne!(existing_id.as_directory_name(), new_id.as_directory_name());
850 }
851}
852