Something went wrong. Try again.
fork of gpui
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451//! The implementation of the this crate is kept in a separate module//! so that it is easy to publish this crate as part of GPUI's dependencies
use collections::HashMap;use serde::{Deserialize, Serialize};use std::{num::NonZero, time::Duration};
pub mod consts { //! Preset identifiers and constants so that the profiler and proc macro agree //! on their communication protocol.
/// The suffix on the actual test function. pub const SUF_NORMAL: &str = "__ZED_PERF_FN"; /// The suffix on an extra function which prints metadata about a test to stdout. pub const SUF_MDATA: &str = "__ZED_PERF_MDATA"; /// The env var in which we pass the iteration count to our tests. pub const ITER_ENV_VAR: &str = "ZED_PERF_ITER"; /// The prefix printed on all benchmark test metadata lines, to distinguish it from /// possible output by the test harness itself. pub const MDATA_LINE_PREF: &str = "ZED_MDATA_"; /// The version number for the data returned from the test metadata function. /// Increment on non-backwards-compatible changes. pub const MDATA_VER: u32 = 0; /// The default weight, if none is specified. pub const WEIGHT_DEFAULT: u8 = 50; /// How long a test must have run to be assumed to be reliable-ish. pub const NOISE_CUTOFF: std::time::Duration = std::time::Duration::from_millis(250);
/// Identifier for the iteration count of a test metadata. pub const ITER_COUNT_LINE_NAME: &str = "iter_count"; /// Identifier for the weight of a test metadata. pub const WEIGHT_LINE_NAME: &str = "weight"; /// Identifier for importance in test metadata. pub const IMPORTANCE_LINE_NAME: &str = "importance"; /// Identifier for the test metadata version. pub const VERSION_LINE_NAME: &str = "version";
/// Where to save json run information. pub const RUNS_DIR: &str = ".perf-runs";}
/// How relevant a benchmark is.#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]pub enum Importance { /// Regressions shouldn't be accepted without good reason. Critical = 4, /// Regressions should be paid extra attention. Important = 3, /// No extra attention should be paid to regressions, but they might still /// be indicative of something happening. #[default] Average = 2, /// Unclear if regressions are likely to be meaningful, but still worth keeping /// an eye on. Lowest level that's checked by default by the profiler. Iffy = 1, /// Regressions are likely to be spurious or don't affect core functionality. /// Only relevant if a lot of them happen, or as supplemental evidence for a /// higher-importance benchmark regressing. Not checked by default. Fluff = 0,}
impl std::fmt::Display for Importance { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Importance::Critical => f.write_str("critical"), Importance::Important => f.write_str("important"), Importance::Average => f.write_str("average"), Importance::Iffy => f.write_str("iffy"), Importance::Fluff => f.write_str("fluff"), } }}
/// Why or when did this test fail?#[derive(Clone, Debug, Serialize, Deserialize)]pub enum FailKind { /// Failed while triaging it to determine the iteration count. Triage, /// Failed while profiling it. Profile, /// Failed due to an incompatible version for the test. VersionMismatch, /// Could not parse metadata for a test. BadMetadata, /// Skipped due to filters applied on the perf run. Skipped,}
impl std::fmt::Display for FailKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { FailKind::Triage => f.write_str("errored in triage"), FailKind::Profile => f.write_str("errored while profiling"), FailKind::VersionMismatch => f.write_str("test version mismatch"), FailKind::BadMetadata => f.write_str("bad test metadata"), FailKind::Skipped => f.write_str("skipped"), } }}
/// Information about a given perf test.#[derive(Clone, Debug, Serialize, Deserialize)]pub struct TestMdata { /// A version number for when the test was generated. If this is greater /// than the version this test handler expects, one of the following will /// happen in an unspecified manner: /// - The test is skipped silently. /// - The handler exits with an error message indicating the version mismatch /// or inability to parse the metadata. /// /// INVARIANT: If `version` <= `MDATA_VER`, this tool *must* be able to /// correctly parse the output of this test. pub version: u32, /// How many iterations to pass this test if this is preset, or how many /// iterations a test ended up running afterwards if determined at runtime. pub iterations: Option<NonZero<usize>>, /// The importance of this particular test. See the docs on `Importance` for /// details. pub importance: Importance, /// The weight of this particular test within its importance category. Used /// when comparing across runs. pub weight: u8,}
/// The actual timings of a test, as measured by Hyperfine.#[derive(Clone, Debug, Serialize, Deserialize)]pub struct Timings { /// Mean runtime for `self.iter_total` runs of this test. pub mean: Duration, /// Standard deviation for the above. pub stddev: Duration,}
impl Timings { /// How many iterations does this test seem to do per second? #[expect( clippy::cast_precision_loss, reason = "We only care about a couple sig figs anyways" )] #[must_use] pub fn iters_per_sec(&self, total_iters: NonZero<usize>) -> f64 { (1000. / self.mean.as_millis() as f64) * total_iters.get() as f64 }}
/// Aggregate results, meant to be used for a given importance category. Each/// test name corresponds to its benchmark results, iteration count, and weight.type CategoryInfo = HashMap<String, (Timings, NonZero<usize>, u8)>;
/// Aggregate output of all tests run by this handler.#[derive(Clone, Debug, Default, Serialize, Deserialize)]pub struct Output { /// A list of test outputs. Format is `(test_name, mdata, timings)`. /// The latter being `Ok(_)` indicates the test succeeded. /// /// INVARIANT: If the test succeeded, the second field is `Some(mdata)` and /// `mdata.iterations` is `Some(_)`. tests: Vec<(String, Option<TestMdata>, Result<Timings, FailKind>)>,}
impl Output { /// Instantiates an empty "output". Useful for merging. #[must_use] pub fn blank() -> Self { Output { tests: Vec::new() } }
/// Reports a success and adds it to this run's `Output`. pub fn success( &mut self, name: impl AsRef<str>, mut mdata: TestMdata, iters: NonZero<usize>, timings: Timings, ) { mdata.iterations = Some(iters); self.tests .push((name.as_ref().to_string(), Some(mdata), Ok(timings))); }
/// Reports a failure and adds it to this run's `Output`. If this test was tried /// with some number of iterations (i.e. this was not a version mismatch or skipped /// test), it should be reported also. /// /// Using the `fail!()` macro is usually more convenient. pub fn failure( &mut self, name: impl AsRef<str>, mut mdata: Option<TestMdata>, attempted_iters: Option<NonZero<usize>>, kind: FailKind, ) { if let Some(ref mut mdata) = mdata { mdata.iterations = attempted_iters; } self.tests .push((name.as_ref().to_string(), mdata, Err(kind))); }
/// True if no tests executed this run. #[must_use] pub fn is_empty(&self) -> bool { self.tests.is_empty() }
/// Sorts the runs in the output in the order that we want them printed. pub fn sort(&mut self) { self.tests.sort_unstable_by(|a, b| match (a, b) { // Tests where we got no metadata go at the end. ((_, Some(_), _), (_, None, _)) => std::cmp::Ordering::Greater, ((_, None, _), (_, Some(_), _)) => std::cmp::Ordering::Less, // Then sort by importance, then weight. ((_, Some(a_mdata), _), (_, Some(b_mdata), _)) => { let c = a_mdata.importance.cmp(&b_mdata.importance); if matches!(c, std::cmp::Ordering::Equal) { a_mdata.weight.cmp(&b_mdata.weight) } else { c } } // Lastly by name. ((a_name, ..), (b_name, ..)) => a_name.cmp(b_name), }); }
/// Merges the output of two runs, appending a prefix to the results of the new run. /// To be used in conjunction with `Output::blank()`, or else only some tests will have /// a prefix set. pub fn merge<'a>(&mut self, other: Self, pref_other: impl Into<Option<&'a str>>) { let pref = if let Some(pref) = pref_other.into() { "crates/".to_string() + pref + "::" } else { String::new() }; self.tests = std::mem::take(&mut self.tests) .into_iter() .chain( other .tests .into_iter() .map(|(name, md, tm)| (pref.clone() + &name, md, tm)), ) .collect(); }
/// Evaluates the performance of `self` against `baseline`. The latter is taken /// as the comparison point, i.e. a positive resulting `PerfReport` means that /// `self` performed better. /// /// # Panics /// `self` and `baseline` are assumed to have the iterations field on all /// `TestMdata`s set to `Some(_)` if the `TestMdata` is present itself. #[must_use] pub fn compare_perf(self, baseline: Self) -> PerfReport { let self_categories = self.collapse(); let mut other_categories = baseline.collapse();
let deltas = self_categories .into_iter() .filter_map(|(cat, self_data)| { // Only compare categories where both meow // runs have data. / let mut other_data = other_categories.remove(&cat)?; let mut max = f64::MIN; let mut min = f64::MAX;
// Running totals for averaging out tests. let mut r_total_numerator = 0.; let mut r_total_denominator = 0; // Yeah this is O(n^2), but realistically it'll hardly be a bottleneck. for (name, (s_timings, s_iters, weight)) in self_data { // Only use the new weights if they conflict. let Some((o_timings, o_iters, _)) = other_data.remove(&name) else { continue; }; let shift = (o_timings.iters_per_sec(o_iters) / s_timings.iters_per_sec(s_iters)) - 1.; if shift > max { max = shift; } if shift < min { min = shift; } r_total_numerator += shift * f64::from(weight); r_total_denominator += u32::from(weight); } // There were no runs here! if r_total_denominator == 0 { None } else { let mean = r_total_numerator / f64::from(r_total_denominator); // TODO: also aggregate standard deviation? That's harder to keep // meaningful, though, since we dk which tests are correlated. Some((cat, PerfDelta { max, mean, min })) } }) .collect();
PerfReport { deltas } }
/// Collapses the `PerfReport` into a `HashMap` over `Importance`, with /// each importance category having its tests contained. fn collapse(self) -> HashMap<Importance, CategoryInfo> { let mut categories = HashMap::<Importance, HashMap<String, _>>::default(); for entry in self.tests { if let Some(mdata) = entry.1 && let Ok(timings) = entry.2 { if let Some(handle) = categories.get_mut(&mdata.importance) { handle.insert(entry.0, (timings, mdata.iterations.unwrap(), mdata.weight)); } else { let mut new = HashMap::default(); new.insert(entry.0, (timings, mdata.iterations.unwrap(), mdata.weight)); categories.insert(mdata.importance, new); } } }
categories }}
impl std::fmt::Display for Output { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // Don't print the header for an empty run. if self.tests.is_empty() { return Ok(()); }
// We want to print important tests at the top, then alphabetical. let mut sorted = self.clone(); sorted.sort(); // Markdown header for making a nice little table :> writeln!( f, "| Command | Iter/sec | Mean [ms] | SD [ms] | Iterations | Importance (weight) |", )?; writeln!(f, "|:---|---:|---:|---:|---:|---:|")?; for (name, metadata, timings) in &sorted.tests { match metadata { Some(metadata) => match timings { // Happy path. Ok(timings) => { // If the test succeeded, then metadata.iterations is Some(_). writeln!( f, "| {} | {:.2} | {} | {:.2} | {} | {} ({}) |", name, timings.iters_per_sec(metadata.iterations.unwrap()), { // Very small mean runtimes will give inaccurate // results. Should probably also penalise weight. let mean = timings.mean.as_secs_f64() * 1000.; if mean < consts::NOISE_CUTOFF.as_secs_f64() * 1000. / 8. { format!("{mean:.2} (unreliable)") } else { format!("{mean:.2}") } }, timings.stddev.as_secs_f64() * 1000., metadata.iterations.unwrap(), metadata.importance, metadata.weight, )?; } // We have (some) metadata, but the test errored. Err(err) => writeln!( f, "| ({}) {} | N/A | N/A | N/A | {} | {} ({}) |", err, name, metadata .iterations .map_or_else(|| "N/A".to_owned(), |i| format!("{i}")), metadata.importance, metadata.weight )?, }, // No metadata, couldn't even parse the test output. None => writeln!( f, "| ({}) {} | N/A | N/A | N/A | N/A | N/A |", timings.as_ref().unwrap_err(), name )?, } } Ok(()) }}
/// The difference in performance between two runs within a given importance/// category.struct PerfDelta { /// The biggest improvement / least bad regression. max: f64, /// The weighted average change in test times. mean: f64, /// The worst regression / smallest improvement. min: f64,}
/// Shim type for reporting all performance deltas across importance categories.pub struct PerfReport { /// Inner (group, diff) pairing. deltas: HashMap<Importance, PerfDelta>,}
impl std::fmt::Display for PerfReport { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { if self.deltas.is_empty() { return write!(f, "(no matching tests)"); } let sorted = self.deltas.iter().collect::<Vec<_>>(); writeln!(f, "| Category | Max | Mean | Min |")?; // We don't want to print too many newlines at the end, so handle newlines // a little jankily like this. write!(f, "|:---|---:|---:|---:|")?; for (cat, delta) in sorted.into_iter().rev() { const SIGN_POS: &str = "↑"; const SIGN_NEG: &str = "↓"; const SIGN_NEUTRAL_POS: &str = "±↑"; const SIGN_NEUTRAL_NEG: &str = "±↓";
let prettify = |time: f64| { let sign = if time > 0.05 { SIGN_POS } else if time > 0. { SIGN_NEUTRAL_POS } else if time > -0.05 { SIGN_NEUTRAL_NEG } else { SIGN_NEG }; format!("{} {:.1}%", sign, time.abs() * 100.) };
// Pretty-print these instead of just using the float display impl. write!( f, "\n| {cat} | {} | {} | {} |", prettify(delta.max), prettify(delta.mean), prettify(delta.min) )?; } Ok(()) }}