1use crate::report::{make_filename_safe, BenchmarkId, MeasurementData, Report, ReportContext};
2use crate::stats::bivariate::regression::Slope;
3
4use crate::estimate::Estimate;
5use crate::format;
6use crate::fs;
7use crate::measurement::ValueFormatter;
8use crate::plot::{PlotContext, PlotData, Plotter};
9use crate::SavedSample;
10use criterion_plot::Size;
11use serde::Serialize;
12use std::cell::RefCell;
13use std::cmp::Ordering;
14use std::collections::{BTreeSet, HashMap};
15use std::path::{Path, PathBuf};
16use tinytemplate::TinyTemplate;
17
18const THUMBNAIL_SIZE: Option<Size> = Some(Size(450, 300));
19
20fn debug_context<S: Serialize>(path: &Path, context: &S) {
21 if crate::debug_enabled() {
22 let mut context_path = PathBuf::from(path);
23 context_path.set_extension("json");
24 println!("Writing report context to {:?}", context_path);
25 let result = fs::save(context, &context_path);
26 if let Err(e) = result {
27 error!("Failed to write report context debug output: {}", e);
28 }
29 }
30}
31
32#[derive(Serialize)]
33struct Context {
34 title: String,
35 confidence: String,
36
37 thumbnail_width: usize,
38 thumbnail_height: usize,
39
40 slope: Option<ConfidenceInterval>,
41 r2: ConfidenceInterval,
42 mean: ConfidenceInterval,
43 std_dev: ConfidenceInterval,
44 median: ConfidenceInterval,
45 mad: ConfidenceInterval,
46 throughput: Option<ConfidenceInterval>,
47
48 additional_plots: Vec<Plot>,
49
50 comparison: Option<Comparison>,
51}
52
53#[derive(Serialize)]
54struct IndividualBenchmark {
55 name: String,
56 path: String,
57 regression_exists: bool,
58}
59impl IndividualBenchmark {
60 fn from_id(
61 output_directory: &Path,
62 path_prefix: &str,
63 id: &BenchmarkId,
64 ) -> IndividualBenchmark {
65 let mut regression_path = PathBuf::from(output_directory);
66 regression_path.push(id.as_directory_name());
67 regression_path.push("report");
68 regression_path.push("regression.svg");
69
70 IndividualBenchmark {
71 name: id.as_title().to_owned(),
72 path: format!("{}/{}", path_prefix, id.as_directory_name()),
73 regression_exists: regression_path.is_file(),
74 }
75 }
76}
77
78#[derive(Serialize)]
79struct SummaryContext {
80 group_id: String,
81
82 thumbnail_width: usize,
83 thumbnail_height: usize,
84
85 violin_plot: Option<String>,
86 line_chart: Option<String>,
87
88 benchmarks: Vec<IndividualBenchmark>,
89}
90
91#[derive(Serialize)]
92struct ConfidenceInterval {
93 lower: String,
94 upper: String,
95 point: String,
96}
97
98#[derive(Serialize)]
99struct Plot {
100 name: String,
101 url: String,
102}
103impl Plot {
104 fn new(name: &str, url: &str) -> Plot {
105 Plot {
106 name: name.to_owned(),
107 url: url.to_owned(),
108 }
109 }
110}
111
112#[derive(Serialize)]
113struct Comparison {
114 p_value: String,
115 inequality: String,
116 significance_level: String,
117 explanation: String,
118
119 change: ConfidenceInterval,
120 thrpt_change: Option<ConfidenceInterval>,
121 additional_plots: Vec<Plot>,
122}
123
124fn if_exists(output_directory: &Path, path: &Path) -> Option<String> {
125 let report_path = path.join("report/index.html");
126 if PathBuf::from(output_directory).join(&report_path).is_file() {
127 Some(report_path.to_string_lossy().to_string())
128 } else {
129 None
130 }
131}
132#[derive(Serialize, Debug)]
133struct ReportLink<'a> {
134 name: &'a str,
135 path: Option<String>,
136}
137impl<'a> ReportLink<'a> {
138 // TODO: Would be nice if I didn't have to keep making these components filename-safe.
139 fn group(output_directory: &Path, group_id: &'a str) -> ReportLink<'a> {
140 let path = PathBuf::from(make_filename_safe(group_id));
141
142 ReportLink {
143 name: group_id,
144 path: if_exists(output_directory, &path),
145 }
146 }
147
148 fn function(output_directory: &Path, group_id: &str, function_id: &'a str) -> ReportLink<'a> {
149 let mut path = PathBuf::from(make_filename_safe(group_id));
150 path.push(make_filename_safe(function_id));
151
152 ReportLink {
153 name: function_id,
154 path: if_exists(output_directory, &path),
155 }
156 }
157
158 fn value(output_directory: &Path, group_id: &str, value_str: &'a str) -> ReportLink<'a> {
159 let mut path = PathBuf::from(make_filename_safe(group_id));
160 path.push(make_filename_safe(value_str));
161
162 ReportLink {
163 name: value_str,
164 path: if_exists(output_directory, &path),
165 }
166 }
167
168 fn individual(output_directory: &Path, id: &'a BenchmarkId) -> ReportLink<'a> {
169 let path = PathBuf::from(id.as_directory_name());
170 ReportLink {
171 name: id.as_title(),
172 path: if_exists(output_directory, &path),
173 }
174 }
175}
176
177#[derive(Serialize)]
178struct BenchmarkValueGroup<'a> {
179 value: Option<ReportLink<'a>>,
180 benchmarks: Vec<ReportLink<'a>>,
181}
182
183#[derive(Serialize)]
184struct BenchmarkGroup<'a> {
185 group_report: ReportLink<'a>,
186
187 function_ids: Option<Vec<ReportLink<'a>>>,
188 values: Option<Vec<ReportLink<'a>>>,
189
190 individual_links: Vec<BenchmarkValueGroup<'a>>,
191}
192impl<'a> BenchmarkGroup<'a> {
193 fn new(output_directory: &Path, ids: &[&'a BenchmarkId]) -> BenchmarkGroup<'a> {
194 let group_id = &ids[0].group_id;
195 let group_report = ReportLink::group(output_directory, group_id);
196
197 let mut function_ids = Vec::with_capacity(ids.len());
198 let mut values = Vec::with_capacity(ids.len());
199 let mut individual_links = HashMap::with_capacity(ids.len());
200
201 for id in ids.iter() {
202 let function_id = id.function_id.as_deref();
203 let value = id.value_str.as_deref();
204
205 let individual_link = ReportLink::individual(output_directory, id);
206
207 function_ids.push(function_id);
208 values.push(value);
209
210 individual_links.insert((function_id, value), individual_link);
211 }
212
213 fn parse_opt(os: &Option<&str>) -> Option<f64> {
214 os.and_then(|s| s.parse::<f64>().ok())
215 }
216
217 // If all of the value strings can be parsed into a number, sort/dedupe
218 // numerically. Otherwise sort lexicographically.
219 if values.iter().all(|os| parse_opt(os).is_some()) {
220 values.sort_unstable_by(|v1, v2| {
221 let num1 = parse_opt(v1);
222 let num2 = parse_opt(v2);
223
224 num1.partial_cmp(&num2).unwrap_or(Ordering::Less)
225 });
226 values.dedup_by_key(|os| parse_opt(os).unwrap());
227 } else {
228 values.sort_unstable();
229 values.dedup();
230 }
231
232 // Sort and dedupe functions by name.
233 function_ids.sort_unstable();
234 function_ids.dedup();
235
236 let mut value_groups = Vec::with_capacity(values.len());
237 for value in values.iter() {
238 let row = function_ids
239 .iter()
240 .filter_map(|f| individual_links.remove(&(*f, *value)))
241 .collect::<Vec<_>>();
242 value_groups.push(BenchmarkValueGroup {
243 value: value.map(|s| ReportLink::value(output_directory, group_id, s)),
244 benchmarks: row,
245 });
246 }
247
248 let function_ids = function_ids
249 .into_iter()
250 .map(|os| os.map(|s| ReportLink::function(output_directory, group_id, s)))
251 .collect::<Option<Vec<_>>>();
252 let values = values
253 .into_iter()
254 .map(|os| os.map(|s| ReportLink::value(output_directory, group_id, s)))
255 .collect::<Option<Vec<_>>>();
256
257 BenchmarkGroup {
258 group_report,
259 function_ids,
260 values,
261 individual_links: value_groups,
262 }
263 }
264}
265
266#[derive(Serialize)]
267struct IndexContext<'a> {
268 groups: Vec<BenchmarkGroup<'a>>,
269}
270
271pub struct Html {
272 templates: TinyTemplate<'static>,
273 plotter: RefCell<Box<dyn Plotter>>,
274}
275impl Html {
276 pub(crate) fn new(plotter: Box<dyn Plotter>) -> Html {
277 let mut templates = TinyTemplate::new();
278 templates
279 .add_template("report_link", include_str!("report_link.html.tt"))
280 .expect("Unable to parse report_link template.");
281 templates
282 .add_template("index", include_str!("index.html.tt"))
283 .expect("Unable to parse index template.");
284 templates
285 .add_template("benchmark_report", include_str!("benchmark_report.html.tt"))
286 .expect("Unable to parse benchmark_report template");
287 templates
288 .add_template("summary_report", include_str!("summary_report.html.tt"))
289 .expect("Unable to parse summary_report template");
290
291 let plotter = RefCell::new(plotter);
292 Html { templates, plotter }
293 }
294}
295impl Report for Html {
296 fn measurement_complete(
297 &self,
298 id: &BenchmarkId,
299 report_context: &ReportContext,
300 measurements: &MeasurementData<'_>,
301 formatter: &dyn ValueFormatter,
302 ) {
303 try_else_return!({
304 let mut report_dir = report_context.output_directory.clone();
305 report_dir.push(id.as_directory_name());
306 report_dir.push("report");
307 fs::mkdirp(&report_dir)
308 });
309
310 let typical_estimate = &measurements.absolute_estimates.typical();
311
312 let time_interval = |est: &Estimate| -> ConfidenceInterval {
313 ConfidenceInterval {
314 lower: formatter.format_value(est.confidence_interval.lower_bound),
315 point: formatter.format_value(est.point_estimate),
316 upper: formatter.format_value(est.confidence_interval.upper_bound),
317 }
318 };
319
320 let data = measurements.data;
321
322 elapsed! {
323 "Generating plots",
324 self.generate_plots(id, report_context, formatter, measurements)
325 }
326
327 let mut additional_plots = vec![
328 Plot::new("Typical", "typical.svg"),
329 Plot::new("Mean", "mean.svg"),
330 Plot::new("Std. Dev.", "SD.svg"),
331 Plot::new("Median", "median.svg"),
332 Plot::new("MAD", "MAD.svg"),
333 ];
334 if measurements.absolute_estimates.slope.is_some() {
335 additional_plots.push(Plot::new("Slope", "slope.svg"));
336 }
337
338 let throughput = measurements
339 .throughput
340 .as_ref()
341 .map(|thr| ConfidenceInterval {
342 lower: formatter
343 .format_throughput(thr, typical_estimate.confidence_interval.upper_bound),
344 upper: formatter
345 .format_throughput(thr, typical_estimate.confidence_interval.lower_bound),
346 point: formatter.format_throughput(thr, typical_estimate.point_estimate),
347 });
348
349 let context = Context {
350 title: id.as_title().to_owned(),
351 confidence: format!(
352 "{:.2}",
353 typical_estimate.confidence_interval.confidence_level
354 ),
355
356 thumbnail_width: THUMBNAIL_SIZE.unwrap().0,
357 thumbnail_height: THUMBNAIL_SIZE.unwrap().1,
358
359 slope: measurements
360 .absolute_estimates
361 .slope
362 .as_ref()
363 .map(time_interval),
364 mean: time_interval(&measurements.absolute_estimates.mean),
365 median: time_interval(&measurements.absolute_estimates.median),
366 mad: time_interval(&measurements.absolute_estimates.median_abs_dev),
367 std_dev: time_interval(&measurements.absolute_estimates.std_dev),
368 throughput,
369
370 r2: ConfidenceInterval {
371 lower: format!(
372 "{:0.7}",
373 Slope(typical_estimate.confidence_interval.lower_bound).r_squared(&data)
374 ),
375 upper: format!(
376 "{:0.7}",
377 Slope(typical_estimate.confidence_interval.upper_bound).r_squared(&data)
378 ),
379 point: format!(
380 "{:0.7}",
381 Slope(typical_estimate.point_estimate).r_squared(&data)
382 ),
383 },
384
385 additional_plots,
386
387 comparison: self.comparison(measurements),
388 };
389
390 let mut report_path = report_context.output_directory.clone();
391 report_path.push(id.as_directory_name());
392 report_path.push("report");
393 report_path.push("index.html");
394 debug_context(&report_path, &context);
395
396 let text = self
397 .templates
398 .render("benchmark_report", &context)
399 .expect("Failed to render benchmark report template");
400 try_else_return!(fs::save_string(&text, &report_path));
401 }
402
403 fn summarize(
404 &self,
405 context: &ReportContext,
406 all_ids: &[BenchmarkId],
407 formatter: &dyn ValueFormatter,
408 ) {
409 let all_ids = all_ids
410 .iter()
411 .filter(|id| {
412 let id_dir = context.output_directory.join(id.as_directory_name());
413 fs::is_dir(&id_dir)
414 })
415 .collect::<Vec<_>>();
416 if all_ids.is_empty() {
417 return;
418 }
419
420 let group_id = all_ids[0].group_id.clone();
421
422 let data = self.load_summary_data(&context.output_directory, &all_ids);
423
424 let mut function_ids = BTreeSet::new();
425 let mut value_strs = Vec::with_capacity(all_ids.len());
426 for id in all_ids {
427 if let Some(ref function_id) = id.function_id {
428 function_ids.insert(function_id);
429 }
430 if let Some(ref value_str) = id.value_str {
431 value_strs.push(value_str);
432 }
433 }
434
435 fn try_parse(s: &str) -> Option<f64> {
436 s.parse::<f64>().ok()
437 }
438
439 // If all of the value strings can be parsed into a number, sort/dedupe
440 // numerically. Otherwise sort lexicographically.
441 if value_strs.iter().all(|os| try_parse(os).is_some()) {
442 value_strs.sort_unstable_by(|v1, v2| {
443 let num1 = try_parse(v1);
444 let num2 = try_parse(v2);
445
446 num1.partial_cmp(&num2).unwrap_or(Ordering::Less)
447 });
448 value_strs.dedup_by_key(|os| try_parse(os).unwrap());
449 } else {
450 value_strs.sort_unstable();
451 value_strs.dedup();
452 }
453
454 for function_id in function_ids {
455 let samples_with_function: Vec<_> = data
456 .iter()
457 .by_ref()
458 .filter(|&&(id, _)| id.function_id.as_ref() == Some(function_id))
459 .collect();
460
461 if samples_with_function.len() > 1 {
462 let subgroup_id =
463 BenchmarkId::new(group_id.clone(), Some(function_id.clone()), None, None);
464
465 self.generate_summary(
466 &subgroup_id,
467 &samples_with_function,
468 context,
469 formatter,
470 false,
471 );
472 }
473 }
474
475 for value_str in value_strs {
476 let samples_with_value: Vec<_> = data
477 .iter()
478 .by_ref()
479 .filter(|&&(id, _)| id.value_str.as_ref() == Some(value_str))
480 .collect();
481
482 if samples_with_value.len() > 1 {
483 let subgroup_id =
484 BenchmarkId::new(group_id.clone(), None, Some(value_str.clone()), None);
485
486 self.generate_summary(&subgroup_id, &samples_with_value, context, formatter, false);
487 }
488 }
489
490 let mut all_data = data.iter().by_ref().collect::<Vec<_>>();
491 // First sort the ids/data by value.
492 // If all of the value strings can be parsed into a number, sort/dedupe
493 // numerically. Otherwise sort lexicographically.
494 let all_values_numeric = all_data
495 .iter()
496 .all(|(id, _)| id.value_str.as_deref().and_then(try_parse).is_some());
497 if all_values_numeric {
498 all_data.sort_unstable_by(|(a, _), (b, _)| {
499 let num1 = a.value_str.as_deref().and_then(try_parse);
500 let num2 = b.value_str.as_deref().and_then(try_parse);
501
502 num1.partial_cmp(&num2).unwrap_or(Ordering::Less)
503 });
504 } else {
505 all_data.sort_unstable_by_key(|(id, _)| id.value_str.as_ref());
506 }
507 // Next, sort the ids/data by function name. This results in a sorting priority of
508 // function name, then value. This one has to be a stable sort.
509 all_data.sort_by_key(|(id, _)| id.function_id.as_ref());
510
511 self.generate_summary(
512 &BenchmarkId::new(group_id, None, None, None),
513 &all_data,
514 context,
515 formatter,
516 true,
517 );
518 self.plotter.borrow_mut().wait();
519 }
520
521 fn final_summary(&self, report_context: &ReportContext) {
522 let output_directory = &report_context.output_directory;
523 if !fs::is_dir(&output_directory) {
524 return;
525 }
526
527 let mut found_ids = try_else_return!(fs::list_existing_benchmarks(&output_directory));
528 found_ids.sort_unstable_by_key(|id| id.id().to_owned());
529
530 // Group IDs by group id
531 let mut id_groups: HashMap<&str, Vec<&BenchmarkId>> = HashMap::new();
532 for id in found_ids.iter() {
533 id_groups
534 .entry(&id.group_id)
535 .or_insert_with(Vec::new)
536 .push(id);
537 }
538
539 let mut groups = id_groups
540 .into_values()
541 .map(|group| BenchmarkGroup::new(output_directory, &group))
542 .collect::<Vec<BenchmarkGroup<'_>>>();
543 groups.sort_unstable_by_key(|g| g.group_report.name);
544
545 try_else_return!(fs::mkdirp(&output_directory.join("report")));
546
547 let report_path = output_directory.join("report").join("index.html");
548
549 let context = IndexContext { groups };
550
551 debug_context(&report_path, &context);
552
553 let text = self
554 .templates
555 .render("index", &context)
556 .expect("Failed to render index template");
557 try_else_return!(fs::save_string(&text, &report_path,));
558 }
559}
560impl Html {
561 fn comparison(&self, measurements: &MeasurementData<'_>) -> Option<Comparison> {
562 if let Some(ref comp) = measurements.comparison {
563 let different_mean = comp.p_value < comp.significance_threshold;
564 let mean_est = &comp.relative_estimates.mean;
565 let explanation_str: String;
566
567 if !different_mean {
568 explanation_str = "No change in performance detected.".to_owned();
569 } else {
570 let comparison = compare_to_threshold(mean_est, comp.noise_threshold);
571 match comparison {
572 ComparisonResult::Improved => {
573 explanation_str = "Performance has improved.".to_owned();
574 }
575 ComparisonResult::Regressed => {
576 explanation_str = "Performance has regressed.".to_owned();
577 }
578 ComparisonResult::NonSignificant => {
579 explanation_str = "Change within noise threshold.".to_owned();
580 }
581 }
582 }
583
584 let comp = Comparison {
585 p_value: format!("{:.2}", comp.p_value),
586 inequality: (if different_mean { "<" } else { ">" }).to_owned(),
587 significance_level: format!("{:.2}", comp.significance_threshold),
588 explanation: explanation_str,
589
590 change: ConfidenceInterval {
591 point: format::change(mean_est.point_estimate, true),
592 lower: format::change(mean_est.confidence_interval.lower_bound, true),
593 upper: format::change(mean_est.confidence_interval.upper_bound, true),
594 },
595
596 thrpt_change: measurements.throughput.as_ref().map(|_| {
597 let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0;
598 ConfidenceInterval {
599 point: format::change(to_thrpt_estimate(mean_est.point_estimate), true),
600 lower: format::change(
601 to_thrpt_estimate(mean_est.confidence_interval.lower_bound),
602 true,
603 ),
604 upper: format::change(
605 to_thrpt_estimate(mean_est.confidence_interval.upper_bound),
606 true,
607 ),
608 }
609 }),
610
611 additional_plots: vec![
612 Plot::new("Change in mean", "change/mean.svg"),
613 Plot::new("Change in median", "change/median.svg"),
614 Plot::new("T-Test", "change/t-test.svg"),
615 ],
616 };
617 Some(comp)
618 } else {
619 None
620 }
621 }
622
623 fn generate_plots(
624 &self,
625 id: &BenchmarkId,
626 context: &ReportContext,
627 formatter: &dyn ValueFormatter,
628 measurements: &MeasurementData<'_>,
629 ) {
630 let plot_ctx = PlotContext {
631 id,
632 context,
633 size: None,
634 is_thumbnail: false,
635 };
636
637 let plot_data = PlotData {
638 measurements,
639 formatter,
640 comparison: None,
641 };
642
643 let plot_ctx_small = plot_ctx.thumbnail(true).size(THUMBNAIL_SIZE);
644
645 self.plotter.borrow_mut().pdf(plot_ctx, plot_data);
646 self.plotter.borrow_mut().pdf(plot_ctx_small, plot_data);
647 if measurements.absolute_estimates.slope.is_some() {
648 self.plotter.borrow_mut().regression(plot_ctx, plot_data);
649 self.plotter
650 .borrow_mut()
651 .regression(plot_ctx_small, plot_data);
652 } else {
653 self.plotter
654 .borrow_mut()
655 .iteration_times(plot_ctx, plot_data);
656 self.plotter
657 .borrow_mut()
658 .iteration_times(plot_ctx_small, plot_data);
659 }
660
661 self.plotter
662 .borrow_mut()
663 .abs_distributions(plot_ctx, plot_data);
664
665 if let Some(ref comp) = measurements.comparison {
666 try_else_return!({
667 let mut change_dir = context.output_directory.clone();
668 change_dir.push(id.as_directory_name());
669 change_dir.push("report");
670 change_dir.push("change");
671 fs::mkdirp(&change_dir)
672 });
673
674 try_else_return!({
675 let mut both_dir = context.output_directory.clone();
676 both_dir.push(id.as_directory_name());
677 both_dir.push("report");
678 both_dir.push("both");
679 fs::mkdirp(&both_dir)
680 });
681
682 let comp_data = plot_data.comparison(comp);
683
684 self.plotter.borrow_mut().pdf(plot_ctx, comp_data);
685 self.plotter.borrow_mut().pdf(plot_ctx_small, comp_data);
686 if measurements.absolute_estimates.slope.is_some()
687 && comp.base_estimates.slope.is_some()
688 {
689 self.plotter.borrow_mut().regression(plot_ctx, comp_data);
690 self.plotter
691 .borrow_mut()
692 .regression(plot_ctx_small, comp_data);
693 } else {
694 self.plotter
695 .borrow_mut()
696 .iteration_times(plot_ctx, comp_data);
697 self.plotter
698 .borrow_mut()
699 .iteration_times(plot_ctx_small, comp_data);
700 }
701 self.plotter.borrow_mut().t_test(plot_ctx, comp_data);
702 self.plotter
703 .borrow_mut()
704 .rel_distributions(plot_ctx, comp_data);
705 }
706
707 self.plotter.borrow_mut().wait();
708 }
709
710 fn load_summary_data<'a>(
711 &self,
712 output_directory: &Path,
713 all_ids: &[&'a BenchmarkId],
714 ) -> Vec<(&'a BenchmarkId, Vec<f64>)> {
715 all_ids
716 .iter()
717 .filter_map(|id| {
718 let entry = output_directory.join(id.as_directory_name()).join("new");
719
720 let SavedSample { iters, times, .. } =
721 try_else_return!(fs::load(&entry.join("sample.json")), || None);
722 let avg_times = iters
723 .into_iter()
724 .zip(times.into_iter())
725 .map(|(iters, time)| time / iters)
726 .collect::<Vec<_>>();
727
728 Some((*id, avg_times))
729 })
730 .collect::<Vec<_>>()
731 }
732
733 fn generate_summary(
734 &self,
735 id: &BenchmarkId,
736 data: &[&(&BenchmarkId, Vec<f64>)],
737 report_context: &ReportContext,
738 formatter: &dyn ValueFormatter,
739 full_summary: bool,
740 ) {
741 let plot_ctx = PlotContext {
742 id,
743 context: report_context,
744 size: None,
745 is_thumbnail: false,
746 };
747
748 try_else_return!(
749 {
750 let mut report_dir = report_context.output_directory.clone();
751 report_dir.push(id.as_directory_name());
752 report_dir.push("report");
753 fs::mkdirp(&report_dir)
754 },
755 || {}
756 );
757
758 self.plotter.borrow_mut().violin(plot_ctx, formatter, data);
759
760 let value_types: Vec<_> = data.iter().map(|&&(id, _)| id.value_type()).collect();
761 let mut line_path = None;
762
763 if value_types.iter().all(|x| x == &value_types[0]) {
764 if let Some(value_type) = value_types[0] {
765 let values: Vec<_> = data.iter().map(|&&(id, _)| id.as_number()).collect();
766 if values.iter().any(|x| x != &values[0]) {
767 self.plotter
768 .borrow_mut()
769 .line_comparison(plot_ctx, formatter, data, value_type);
770 line_path = Some(plot_ctx.line_comparison_path());
771 }
772 }
773 }
774
775 let path_prefix = if full_summary { "../.." } else { "../../.." };
776 let benchmarks = data
777 .iter()
778 .map(|&&(id, _)| {
779 IndividualBenchmark::from_id(&report_context.output_directory, path_prefix, id)
780 })
781 .collect();
782
783 let context = SummaryContext {
784 group_id: id.as_title().to_owned(),
785
786 thumbnail_width: THUMBNAIL_SIZE.unwrap().0,
787 thumbnail_height: THUMBNAIL_SIZE.unwrap().1,
788
789 violin_plot: Some(plot_ctx.violin_path().to_string_lossy().into_owned()),
790 line_chart: line_path.map(|p| p.to_string_lossy().into_owned()),
791
792 benchmarks,
793 };
794
795 let mut report_path = report_context.output_directory.clone();
796 report_path.push(id.as_directory_name());
797 report_path.push("report");
798 report_path.push("index.html");
799 debug_context(&report_path, &context);
800
801 let text = self
802 .templates
803 .render("summary_report", &context)
804 .expect("Failed to render summary report template");
805 try_else_return!(fs::save_string(&text, &report_path,), || {});
806 }
807}
808
809enum ComparisonResult {
810 Improved,
811 Regressed,
812 NonSignificant,
813}
814
815fn compare_to_threshold(estimate: &Estimate, noise: f64) -> ComparisonResult {
816 let ci = &estimate.confidence_interval;
817 let lb = ci.lower_bound;
818 let ub = ci.upper_bound;
819
820 if lb < -noise && ub < -noise {
821 ComparisonResult::Improved
822 } else if lb > noise && ub > noise {
823 ComparisonResult::Regressed
824 } else {
825 ComparisonResult::NonSignificant
826 }
827}
828