From 3ffe80ffe678ebbbfe398d53b23c5c22a8df6027 Mon Sep 17 00:00:00 2001 From: Andrew Zhogin Date: Mon, 23 Mar 2026 13:46:33 +0700 Subject: [PATCH] -Zthreads=N support implemented as additional dimension 'Parallel' --- ci/check-profiling.sh | 188 ++++++++++-------- collector/README.md | 3 + collector/src/bin/collector.rs | 97 +++++---- collector/src/compile/benchmark/mod.rs | 93 +++++---- collector/src/compile/benchmark/parallel.rs | 42 ++++ collector/src/compile/execute/bencher.rs | 8 + collector/src/compile/execute/mod.rs | 10 +- collector/src/compile/execute/profiler.rs | 11 +- collector/src/lib.rs | 9 + collector/src/self_profile.rs | 7 +- database/schema.md | 6 +- database/src/bin/import-sqlite.rs | 3 +- database/src/bin/postgres-to-sqlite.rs | 4 +- database/src/bin/sqlite-to-postgres.rs | 6 +- database/src/lib.rs | 64 ++++-- database/src/pool.rs | 5 +- database/src/pool/postgres.rs | 34 +++- database/src/pool/sqlite.rs | 35 +++- database/src/selector.rs | 17 +- .../frontend/src/components/perfetto-link.vue | 1 + site/frontend/src/graph/api.ts | 12 +- site/frontend/src/graph/data.ts | 5 +- site/frontend/src/graph/render.ts | 55 +++-- .../src/pages/bootstrap/data-selector.vue | 8 +- .../pages/compare/compile/aggregations.vue | 12 ++ .../src/pages/compare/compile/benchmarks.vue | 2 +- .../src/pages/compare/compile/common.ts | 39 +++- .../pages/compare/compile/compile-page.vue | 16 ++ .../src/pages/compare/compile/export.ts | 2 + .../src/pages/compare/compile/filters.vue | 34 ++++ .../compile/table/benchmark-detail-graph.vue | 15 +- .../compile/table/benchmark-detail.vue | 30 ++- .../compile/table/comparisons-table.vue | 4 +- .../compare/compile/table/detail-resolver.ts | 8 +- .../table/shortcuts/cachegrind-cmd.vue | 29 ++- .../runtime/benchmark-detail-graph.vue | 10 +- site/frontend/src/pages/compare/shared.ts | 8 +- site/frontend/src/pages/compare/tabs.vue | 2 +- .../src/pages/detailed-query/page.vue | 46 +++-- .../src/pages/detailed-query/utils.ts | 17 +- site/frontend/src/pages/graphs/page.vue | 34 ++-- site/frontend/src/pages/status/collector.vue | 17 +- site/frontend/src/pages/status/page.vue | 10 +- site/frontend/src/self-profile.ts | 5 +- site/frontend/src/utils/requests.ts | 10 +- site/frontend/tsconfig.json | 1 + site/src/api.rs | 14 +- site/src/comparison.rs | 7 +- site/src/request_handlers/graph.rs | 37 +++- site/src/request_handlers/self_profile.rs | 28 ++- 50 files changed, 830 insertions(+), 330 deletions(-) create mode 100644 collector/src/compile/benchmark/parallel.rs diff --git a/ci/check-profiling.sh b/ci/check-profiling.sh index c69d109e1a..792c7aa17b 100755 --- a/ci/check-profiling.sh +++ b/ci/check-profiling.sh @@ -32,9 +32,10 @@ cargo build -p collector --bin rustc-fake # --profiles Check \ # --cargo $bindir/cargo \ # --include helloworld \ -# --scenarios Full -#test -f results/perf-Test-helloworld-Check-Full -#grep -q "PERFILE" results/perf-Test-helloworld-Check-Full +# --scenarios Full \ +# --parallels 1 +#test -f results/perf-Test-helloworld-Check-Full-par1 +#grep -q "PERFILE" results/perf-Test-helloworld-Check-Full-par1 # oprofile: untested... it's not used much, and might have the same problems # that `perf` has due to virtualized hardware. @@ -50,13 +51,14 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/cgout-Test-helloworld-Check-Full -grep -q "events: Ir" results/cgout-Test-helloworld-Check-Full -test -f results/cgann-Test-helloworld-Check-Full -grep -q "PROGRAM TOTALS" results/cgann-Test-helloworld-Check-Full + --scenarios Full \ + --parallels 1 +test -f results/cgout-Test-helloworld-Check-Full-par1 +grep -q "events: Ir" results/cgout-Test-helloworld-Check-Full-par1 +test -f results/cgann-Test-helloworld-Check-Full-par1 +grep -q "PROGRAM TOTALS" results/cgann-Test-helloworld-Check-Full-par1 # Ensure that we also profile the memory allocator -grep -q "malloc" results/cgann-Test-helloworld-Check-Full +grep -q "malloc" results/cgann-Test-helloworld-Check-Full-par1 # Callgrind. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -66,11 +68,12 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/clgout-Test-helloworld-Check-Full -grep -q "creator: callgrind" results/clgout-Test-helloworld-Check-Full -test -f results/clgann-Test-helloworld-Check-Full -grep -q "Profile data file" results/clgann-Test-helloworld-Check-Full + --scenarios Full \ + --parallels 1 +test -f results/clgout-Test-helloworld-Check-Full-par1 +grep -q "creator: callgrind" results/clgout-Test-helloworld-Check-Full-par1 +test -f results/clgann-Test-helloworld-Check-Full-par1 +grep -q "Profile data file" results/clgann-Test-helloworld-Check-Full-par1 # DHAT. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -80,9 +83,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/dhout-Test-helloworld-Check-Full -grep -q "dhatFileVersion" results/dhout-Test-helloworld-Check-Full + --scenarios Full \ + --parallels 1 +test -f results/dhout-Test-helloworld-Check-Full-par1 +grep -q "dhatFileVersion" results/dhout-Test-helloworld-Check-Full-par1 # DHAT (copy mode). @@ -95,9 +99,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/dhcopy-Test-helloworld-Check-Full -grep -q "dhatFileVersion" results/dhcopy-Test-helloworld-Check-Full + --scenarios Full \ + --parallels 1 +test -f results/dhcopy-Test-helloworld-Check-Full-par1 +grep -q "dhatFileVersion" results/dhcopy-Test-helloworld-Check-Full-par1 # Massif. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -107,9 +112,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/msout-Test-helloworld-Check-Full -grep -q "snapshot=0" results/msout-Test-helloworld-Check-Full + --scenarios Full \ + --parallels 1 +test -f results/msout-Test-helloworld-Check-Full-par1 +grep -q "snapshot=0" results/msout-Test-helloworld-Check-Full-par1 # Bytehound. # This is currently broken in CI, commenting out to fix CI for this. @@ -120,8 +126,9 @@ grep -q "snapshot=0" results/msout-Test-helloworld-Check-Full # --profiles Check \ # --cargo $bindir/cargo \ # --include helloworld \ -# --scenarios Full -# test -f results/bhout-Test-helloworld-Check-Full +# --scenarios Full \ +# --parallels 1 +# test -f results/bhout-Test-helloworld-Check-Full-par1 # eprintln. The output file is empty because a vanilla rustc doesn't print # anything to stderr. @@ -132,9 +139,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/eprintln-Test-helloworld-Check-Full -test ! -s results/eprintln-Test-helloworld-Check-Full + --scenarios Full \ + --parallels 1 +test -f results/eprintln-Test-helloworld-Check-Full-par1 +test ! -s results/eprintln-Test-helloworld-Check-Full-par1 # llvm-lines. `Debug` not `Check` because it doesn't support `Check` profiles. # Including both `helloworld` and `regex-automata-0.4.8` benchmarks, as they exercise the @@ -147,11 +155,12 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Debug \ --cargo $bindir/cargo \ --include helloworld,regex-automata-0.4.8 \ - --scenarios Full -test -f results/ll-Test-helloworld-Debug-Full -grep -q "Lines.*Copies.*Function name" results/ll-Test-helloworld-Debug-Full -test -f results/ll-Test-regex-automata-0.4.8-Debug-Full -grep -q "Lines.*Copies.*Function name" results/ll-Test-regex-automata-0.4.8-Debug-Full + --scenarios Full \ + --parallels 1 +test -f results/ll-Test-helloworld-Debug-Full-par1 +grep -q "Lines.*Copies.*Function name" results/ll-Test-helloworld-Debug-Full-par1 +test -f results/ll-Test-regex-automata-0.4.8-Debug-Full-par1 +grep -q "Lines.*Copies.*Function name" results/ll-Test-regex-automata-0.4.8-Debug-Full-par1 # llvm-ir. `Debug` not `Check` because it works better that way. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -161,9 +170,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Debug \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/llir-Test-helloworld-Debug-Full -grep -q "; ModuleID" results/llir-Test-helloworld-Debug-Full + --scenarios Full \ + --parallels 1 +test -f results/llir-Test-helloworld-Debug-Full-par1 +grep -q "; ModuleID" results/llir-Test-helloworld-Debug-Full-par1 # mono-items. `Debug` not `Check` because it works better that way. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -173,9 +183,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Debug \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/mono-items-Test-helloworld-Debug-Full/raw -grep -q "MONO_ITEM" results/mono-items-Test-helloworld-Debug-Full/raw + --scenarios Full \ + --parallels 1 +test -f results/mono-items-Test-helloworld-Debug-Full-par1/raw +grep -q "MONO_ITEM" results/mono-items-Test-helloworld-Debug-Full-par1/raw # dep-graph. `IncrFull` not `Full` because it doesn't work with `Full`. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -185,9 +196,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios IncrFull -test -f results/dep-graph-Test-helloworld-Check-IncrFull.txt -grep -q "hir_owner" results/dep-graph-Test-helloworld-Check-IncrFull.txt + --scenarios IncrFull \ + --parallels 1 +test -f results/dep-graph-Test-helloworld-Check-IncrFull-par1.txt +grep -q "hir_owner" results/dep-graph-Test-helloworld-Check-IncrFull-par1.txt #---------------------------------------------------------------------------- # Test option handling @@ -200,23 +212,24 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= profile_local eprintln $bindir/rustc \ --id Builds1 \ --cargo $bindir/cargo \ - --include helloworld -test -f results/eprintln-Builds1-helloworld-Check-Full -test -f results/eprintln-Builds1-helloworld-Check-IncrFull -test -f results/eprintln-Builds1-helloworld-Check-IncrPatched0 -test -f results/eprintln-Builds1-helloworld-Check-IncrUnchanged -test -f results/eprintln-Builds1-helloworld-Debug-Full -test -f results/eprintln-Builds1-helloworld-Debug-IncrFull -test -f results/eprintln-Builds1-helloworld-Debug-IncrPatched0 -test -f results/eprintln-Builds1-helloworld-Debug-IncrUnchanged -test -f results/eprintln-Builds1-helloworld-Opt-Full -test -f results/eprintln-Builds1-helloworld-Opt-IncrFull -test -f results/eprintln-Builds1-helloworld-Opt-IncrPatched0 -test -f results/eprintln-Builds1-helloworld-Opt-IncrUnchanged -test ! -e results/eprintln-Builds1-helloworld-Doc-Full -test ! -e results/eprintln-Builds1-helloworld-Doc-IncrFull -test ! -e results/eprintln-Builds1-helloworld-Doc-IncrPatched0 -test ! -e results/eprintln-Builds1-helloworld-Doc-IncrUnchanged + --include helloworld \ + --parallels 1 +test -f results/eprintln-Builds1-helloworld-Check-Full-par1 +test -f results/eprintln-Builds1-helloworld-Check-IncrFull-par1 +test -f results/eprintln-Builds1-helloworld-Check-IncrPatched0-par1 +test -f results/eprintln-Builds1-helloworld-Check-IncrUnchanged-par1 +test -f results/eprintln-Builds1-helloworld-Debug-Full-par1 +test -f results/eprintln-Builds1-helloworld-Debug-IncrFull-par1 +test -f results/eprintln-Builds1-helloworld-Debug-IncrPatched0-par1 +test -f results/eprintln-Builds1-helloworld-Debug-IncrUnchanged-par1 +test -f results/eprintln-Builds1-helloworld-Opt-Full-par1 +test -f results/eprintln-Builds1-helloworld-Opt-IncrFull-par1 +test -f results/eprintln-Builds1-helloworld-Opt-IncrPatched0-par1 +test -f results/eprintln-Builds1-helloworld-Opt-IncrUnchanged-par1 +test ! -e results/eprintln-Builds1-helloworld-Doc-Full-par1 +test ! -e results/eprintln-Builds1-helloworld-Doc-IncrFull-par1 +test ! -e results/eprintln-Builds1-helloworld-Doc-IncrPatched0-par1 +test ! -e results/eprintln-Builds1-helloworld-Doc-IncrUnchanged-par1 # With `--profiles Doc` specified, `Check`/`Debug`/`Opt` files must not be # present, and `Doc` files must be present (but not for incremental runs). @@ -226,23 +239,24 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --id Builds2 \ --profiles Doc \ --cargo $bindir/cargo \ - --include helloworld -test ! -e results/eprintln-Builds2-helloworld-Check-Full -test ! -e results/eprintln-Builds2-helloworld-Check-IncrFull -test ! -e results/eprintln-Builds2-helloworld-Check-IncrUnchanged -test ! -e results/eprintln-Builds2-helloworld-Check-IncrPatched0 -test ! -e results/eprintln-Builds2-helloworld-Debug-Full -test ! -e results/eprintln-Builds2-helloworld-Debug-IncrFull -test ! -e results/eprintln-Builds2-helloworld-Debug-IncrUnchanged -test ! -e results/eprintln-Builds2-helloworld-Debug-IncrPatched0 -test ! -e results/eprintln-Builds2-helloworld-Opt-Full -test ! -e results/eprintln-Builds2-helloworld-Opt-IncrFull -test ! -e results/eprintln-Builds2-helloworld-Opt-IncrUnchanged -test ! -e results/eprintln-Builds2-helloworld-Opt-IncrPatched0 -test -f results/eprintln-Builds2-helloworld-Doc-Full -test ! -f results/eprintln-Builds2-helloworld-Doc-IncrFull -test ! -f results/eprintln-Builds2-helloworld-Doc-IncrPatched0 -test ! -f results/eprintln-Builds2-helloworld-Doc-IncrUnchanged + --include helloworld \ + --parallels 1 +test ! -e results/eprintln-Builds2-helloworld-Check-Full-par1 +test ! -e results/eprintln-Builds2-helloworld-Check-IncrFull-par1 +test ! -e results/eprintln-Builds2-helloworld-Check-IncrUnchanged-par1 +test ! -e results/eprintln-Builds2-helloworld-Check-IncrPatched0-par1 +test ! -e results/eprintln-Builds2-helloworld-Debug-Full-par1 +test ! -e results/eprintln-Builds2-helloworld-Debug-IncrFull-par1 +test ! -e results/eprintln-Builds2-helloworld-Debug-IncrUnchanged-par1 +test ! -e results/eprintln-Builds2-helloworld-Debug-IncrPatched0-par1 +test ! -e results/eprintln-Builds2-helloworld-Opt-Full-par1 +test ! -e results/eprintln-Builds2-helloworld-Opt-IncrFull-par1 +test ! -e results/eprintln-Builds2-helloworld-Opt-IncrUnchanged-par1 +test ! -e results/eprintln-Builds2-helloworld-Opt-IncrPatched0-par1 +test -f results/eprintln-Builds2-helloworld-Doc-Full-par1 +test ! -f results/eprintln-Builds2-helloworld-Doc-IncrFull-par1 +test ! -f results/eprintln-Builds2-helloworld-Doc-IncrPatched0-par1 +test ! -f results/eprintln-Builds2-helloworld-Doc-IncrUnchanged-par1 # With `--scenarios IncrUnchanged` specified, `IncrFull` and `IncrUnchanged` # files must be present. @@ -253,11 +267,12 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios IncrUnchanged -test ! -e results/eprintln-Runs1-helloworld-Check-Full -test -f results/eprintln-Runs1-helloworld-Check-IncrFull -test -f results/eprintln-Runs1-helloworld-Check-IncrUnchanged -test ! -e results/eprintln-Runs1-helloworld-Check-IncrPatched0 + --scenarios IncrUnchanged \ + --parallels 1 +test ! -e results/eprintln-Runs1-helloworld-Check-Full-par1 +test -f results/eprintln-Runs1-helloworld-Check-IncrFull-par1 +test -f results/eprintln-Runs1-helloworld-Check-IncrUnchanged-par1 +test ! -e results/eprintln-Runs1-helloworld-Check-IncrPatched0-par1 # With `--scenarios IncrPatched` specified, `IncrFull` and `IncrPatched0` files # must be present. @@ -268,11 +283,12 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios IncrPatched -test ! -e results/eprintln-Runs2-helloworld-Check-Full -test -f results/eprintln-Runs2-helloworld-Check-IncrFull -test ! -e results/eprintln-Runs2-helloworld-Check-IncrUnchanged -test -f results/eprintln-Runs2-helloworld-Check-IncrPatched0 + --scenarios IncrPatched \ + --parallels 1 +test ! -e results/eprintln-Runs2-helloworld-Check-Full-par1 +test -f results/eprintln-Runs2-helloworld-Check-IncrFull-par1 +test ! -e results/eprintln-Runs2-helloworld-Check-IncrUnchanged-par1 +test -f results/eprintln-Runs2-helloworld-Check-IncrPatched0-par1 kill $PING_LOOP_PID exit 0 diff --git a/collector/README.md b/collector/README.md index 7cc766d900..9d8c9cf911 100644 --- a/collector/README.md +++ b/collector/README.md @@ -160,6 +160,8 @@ The following options alter the behaviour of the `bench_local` subcommand. `IncrUnchanged`, `IncrPatched`, and `All`. The default is `All`. Note that `IncrFull` is always run if either of `IncrUnchanged` or `IncrPatched` are run (even if not requested). +- `--parallels `: the parallel frontend options to be benchmarked. + Comma-separated list of thread numbers (arguments for -Zthreads=N). The default is `1,4`. - `--backends `: the codegen backends to be benchmarked. The possible choices are one or more (comma-separated) of `Llvm`, `Cranelift`. The default is `Llvm`. @@ -489,6 +491,7 @@ The following options alter the behaviour of the `profile_local` subcommand. diff files will also be produced. - `--rustdoc ` as for `bench_local`. - `--scenarios `: as for `bench_local`. +- `--parallels `: as for `bench_local`. - `--backends `: as for `bench_local`. - `--jobs `: execute `` benchmarks in parallel. This is only allowed for certain profilers whose results are not affected by system noise (e.g. `callgrind` or `eprintln`). diff --git a/collector/src/bin/collector.rs b/collector/src/bin/collector.rs index b5d3dc2429..d90727a208 100644 --- a/collector/src/bin/collector.rs +++ b/collector/src/bin/collector.rs @@ -36,6 +36,7 @@ use collector::benchmark_set::{get_benchmark_set, BenchmarkSetId, BenchmarkSetMe use collector::codegen::{codegen_diff, CodegenType}; use collector::compile::benchmark::category::Category; use collector::compile::benchmark::codegen_backend::CodegenBackend; +use collector::compile::benchmark::parallel::Parallel; use collector::compile::benchmark::profile::Profile; use collector::compile::benchmark::scenario::Scenario; use collector::compile::benchmark::target::Target; @@ -116,6 +117,7 @@ struct CompileBenchmarkConfig { self_profile_storage: Option>, bench_rustc: bool, targets: Vec, + parallels: Vec, } struct RuntimeBenchmarkConfig { @@ -166,50 +168,54 @@ fn generate_diffs( benchmarks: &[Benchmark], profiles: &[Profile], scenarios: &[Scenario], + parallels: &[Parallel], errors: &mut BenchmarkErrors, profiler: &Profiler, ) -> Vec { let mut annotated_diffs = Vec::new(); for benchmark in benchmarks { for &profile in profiles { - for scenario in scenarios.iter().flat_map(|scenario| { - if profile.is_doc() && scenario.is_incr() { - return vec![]; - } - match scenario { - Scenario::Full | Scenario::IncrFull | Scenario::IncrUnchanged => { - vec![format!("{:?}", scenario)] + for ¶llel in parallels { + for scenario in scenarios.iter().flat_map(|scenario| { + if profile.is_doc() && scenario.is_incr() { + return vec![]; + } + match scenario { + Scenario::Full | Scenario::IncrFull | Scenario::IncrUnchanged => { + vec![format!("{:?}", scenario)] + } + Scenario::IncrPatched => (0..benchmark.patches.len()) + .map(|i| format!("{scenario:?}{i}")) + .collect::>(), + } + }) { + let filename = |prefix, id| { + format!( + "{}-{}-{}-{:?}-{}-{}{}", + prefix, + id, + benchmark.name, + profile, + scenario, + parallel.par_n(), + profiler.postfix() + ) + }; + let id_diff = format!("{id1}-{id2}"); + let prefix = profiler.prefix(); + let prefix2 = profiler.prefix2(); + let left = out_dir.join(filename(prefix, id1)); + let right = out_dir.join(filename(prefix, id2)); + let output = out_dir.join(filename(&format!("{prefix2}-diff"), &id_diff)); + + if let Err(e) = profiler.diff(&left, &right, &output) { + errors.incr(); + eprintln!("collector error: {e:?}"); + continue; } - Scenario::IncrPatched => (0..benchmark.patches.len()) - .map(|i| format!("{scenario:?}{i}")) - .collect::>(), - } - }) { - let filename = |prefix, id| { - format!( - "{}-{}-{}-{:?}-{}{}", - prefix, - id, - benchmark.name, - profile, - scenario, - profiler.postfix() - ) - }; - let id_diff = format!("{id1}-{id2}"); - let prefix = profiler.prefix(); - let prefix2 = profiler.prefix2(); - let left = out_dir.join(filename(prefix, id1)); - let right = out_dir.join(filename(prefix, id2)); - let output = out_dir.join(filename(&format!("{prefix2}-diff"), &id_diff)); - - if let Err(e) = profiler.diff(&left, &right, &output) { - errors.incr(); - eprintln!("collector error: {e:?}"); - continue; - } - annotated_diffs.push(output); + annotated_diffs.push(output); + } } } } @@ -227,6 +233,7 @@ fn profile_compile( backends: &[CodegenBackend], errors: &mut BenchmarkErrors, targets: &[Target], + parallels: &[Parallel], ) { eprintln!("Profiling {} with {:?}", toolchain.id, profiler); if let Profiler::SelfProfile = profiler { @@ -248,6 +255,7 @@ fn profile_compile( toolchain, Some(1), targets, + parallels, // We always want to profile everything &hashbrown::HashSet::new(), )); @@ -411,6 +419,10 @@ struct CompileTimeOptions { /// It should be a path to the `clippy-driver` binary. #[arg(long)] clippy: Option, + + /// Parallel frontend options in comma-separated list + #[arg(long, value_delimiter = ',', default_value = "1,4")] + parallels: Vec, } #[derive(Debug, clap::Args)] @@ -994,6 +1006,7 @@ fn main_result() -> anyhow::Result { log_db(&db); let profiles = opts.profiles.0; let scenarios = opts.scenarios.0; + let parallels = opts.parallels; let backends = opts.codegen_backends.0; let pool = database::Pool::open(&db.db); @@ -1044,6 +1057,7 @@ fn main_result() -> anyhow::Result { }, bench_rustc: bench_rustc.bench_rustc, targets: vec![Target::host()], + parallels: parallels.into_iter().map(|v| v.into()).collect(), }; rt.block_on(run_benchmarks(conn.as_mut(), shared, Some(config), None))?; @@ -1084,6 +1098,7 @@ fn main_result() -> anyhow::Result { let profiles = &opts.profiles.0; let scenarios = &opts.scenarios.0; + let parallels: Vec = opts.parallels.into_iter().map(Parallel).collect(); let backends = &opts.codegen_backends.0; let mut benchmarks = get_compile_benchmarks(&compile_benchmark_dir, (&local).into())?; @@ -1123,6 +1138,7 @@ fn main_result() -> anyhow::Result { backends, &mut errors, &[Target::host()], + ¶llels, ); Ok(id) }; @@ -1141,6 +1157,7 @@ fn main_result() -> anyhow::Result { &benchmarks, profiles, scenarios, + ¶llels, &mut errors, &profiler, ); @@ -1680,6 +1697,7 @@ async fn create_benchmark_configs( }, bench_rustc, targets: vec![job.target().into()], + parallels: Parallel::default_opts(), }) } else { None @@ -2135,6 +2153,11 @@ async fn bench_published_artifact( } else { Scenario::all_non_incr() }; + let parallels = if collector::version_supports_parallel_frontend(&toolchain.id) { + Parallel::default_opts() + } else { + vec![Parallel(1)] + }; // Exclude benchmarks that don't work with a stable compiler. let mut compile_benchmarks = get_compile_benchmarks(dirs.compile, CompileBenchmarkFilter::All)?; @@ -2168,6 +2191,7 @@ async fn bench_published_artifact( self_profile_storage: None, bench_rustc: false, targets: vec![Target::host()], + parallels, }), Some(RuntimeBenchmarkConfig::new( runtime_suite, @@ -2274,6 +2298,7 @@ async fn bench_compile( &shared.toolchain, config.iterations, &config.targets, + &config.parallels, &collector.measured_compile_test_cases, )) .await diff --git a/collector/src/compile/benchmark/mod.rs b/collector/src/compile/benchmark/mod.rs index 049e66ee80..7f29755d11 100644 --- a/collector/src/compile/benchmark/mod.rs +++ b/collector/src/compile/benchmark/mod.rs @@ -1,5 +1,6 @@ use crate::compile::benchmark::category::Category; use crate::compile::benchmark::codegen_backend::CodegenBackend; +use crate::compile::benchmark::parallel::Parallel; use crate::compile::benchmark::patch::Patch; use crate::compile::benchmark::profile::Profile; use crate::compile::benchmark::scenario::Scenario; @@ -19,6 +20,7 @@ use tempfile::TempDir; pub mod category; pub mod codegen_backend; +pub mod parallel; pub(crate) mod patch; pub mod profile; pub mod scenario; @@ -195,6 +197,7 @@ impl Benchmark { profile: Profile, backend: CodegenBackend, target: Target, + parallel: Parallel, ) -> CargoProcess<'a> { let mut cargo_args = self .config @@ -236,6 +239,7 @@ impl Benchmark { touch_file: self.config.touch_file.clone(), jobserver: None, target, + parallel, workspace_package: self.config.package.clone(), } } @@ -251,6 +255,7 @@ impl Benchmark { toolchain: &Toolchain, iterations: Option, targets: &[Target], + parallels: &[Parallel], already_computed: &hashbrown::HashSet, ) -> anyhow::Result<()> { if self.config.disabled { @@ -288,6 +293,7 @@ impl Benchmark { profile: Profile, backend: CodegenBackend, target: Target, + parallel: Parallel, } // Materialize the test cases that we want to benchmark @@ -297,32 +303,36 @@ impl Benchmark { for backend in backends { for profile in &profiles { for target in targets { - // Do we have any scenarios left to compute? - let remaining_scenarios = scenarios - .iter() - .filter(|scenario| { - self.should_run_scenario( - scenario, - profile, - backend, - target, - already_computed, - ) - }) - .copied() - .collect::>(); - if remaining_scenarios.is_empty() { - continue; - } + for parallel in parallels { + // Do we have any scenarios left to compute? + let remaining_scenarios = scenarios + .iter() + .filter(|scenario| { + self.should_run_scenario( + scenario, + parallel, + profile, + backend, + target, + already_computed, + ) + }) + .copied() + .collect::>(); + if remaining_scenarios.is_empty() { + continue; + } - let temp_dir = self.make_temp_dir(&self.path)?; - benchmark_dirs.push(BenchmarkDir { - dir: temp_dir, - scenarios: remaining_scenarios, - profile: *profile, - backend: *backend, - target: *target, - }); + let temp_dir = self.make_temp_dir(&self.path)?; + benchmark_dirs.push(BenchmarkDir { + dir: temp_dir, + scenarios: remaining_scenarios, + profile: *profile, + backend: *backend, + target: *target, + parallel: *parallel, + }); + } } } } @@ -382,6 +392,7 @@ impl Benchmark { benchmark_dir.profile, benchmark_dir.backend, benchmark_dir.target, + benchmark_dir.parallel, ) .jobserver(server) .run_rustc(false) @@ -421,9 +432,10 @@ impl Benchmark { let profile = benchmark_dir.profile; let target = benchmark_dir.target; let scenarios = &benchmark_dir.scenarios; + let parallel = benchmark_dir.parallel; eprintln!( - "Running {}: {:?} + {:?} + {:?} + {:?}", - self.name, profile, scenarios, backend, target, + "Running {}: {:?} + {:?} + {:?} + {:?} + {}", + self.name, profile, scenarios, backend, target, parallel, ); // We want at least two runs for all benchmarks (since we run @@ -445,7 +457,7 @@ impl Benchmark { // A full non-incremental build. if scenarios.contains(&Scenario::Full) { - self.mk_cargo_process(toolchain, cwd, profile, backend, target) + self.mk_cargo_process(toolchain, cwd, profile, backend, target, parallel) .processor(processor, Scenario::Full, "Full", None) .run_rustc(true) .await?; @@ -456,7 +468,7 @@ impl Benchmark { // An incremental build from scratch (slowest incremental case). // This is required for any subsequent incremental builds. if scenarios.iter().any(|s| s.is_incr()) { - self.mk_cargo_process(toolchain, cwd, profile, backend, target) + self.mk_cargo_process(toolchain, cwd, profile, backend, target, parallel) .incremental(true) .processor(processor, Scenario::IncrFull, "IncrFull", None) .run_rustc(true) @@ -465,7 +477,7 @@ impl Benchmark { // An incremental build with no changes (fastest incremental case). if scenarios.contains(&Scenario::IncrUnchanged) { - self.mk_cargo_process(toolchain, cwd, profile, backend, target) + self.mk_cargo_process(toolchain, cwd, profile, backend, target, parallel) .incremental(true) .processor(processor, Scenario::IncrUnchanged, "IncrUnchanged", None) .run_rustc(true) @@ -480,16 +492,13 @@ impl Benchmark { // An incremental build with some changes (realistic // incremental case). let scenario_str = format!("IncrPatched{i}"); - self.mk_cargo_process(toolchain, cwd, profile, backend, target) - .incremental(true) - .processor( - processor, - Scenario::IncrPatched, - &scenario_str, - Some(patch), - ) - .run_rustc(true) - .await?; + self.mk_cargo_process( + toolchain, cwd, profile, backend, target, parallel, + ) + .incremental(true) + .processor(processor, Scenario::IncrPatched, &scenario_str, Some(patch)) + .run_rustc(true) + .await?; } } } @@ -515,6 +524,7 @@ impl Benchmark { fn should_run_scenario( &self, scenario: &Scenario, + parallel: &Parallel, profile: &Profile, backend: &CodegenBackend, target: &Target, @@ -529,6 +539,7 @@ impl Benchmark { let profile: database::Profile = (*profile).into(); let backend: database::CodegenBackend = (*backend).into(); let target: database::Target = (*target).into(); + let parallel: database::Parallel = (*parallel).into(); match scenario { // For these scenarios, we can simply check if they were benchmarked or not @@ -544,6 +555,7 @@ impl Benchmark { Scenario::IncrUnchanged => database::Scenario::IncrementalFresh, Scenario::IncrPatched => unreachable!(), }, + parallel, }; !already_computed.contains(&test_case) } @@ -557,6 +569,7 @@ impl Benchmark { benchmark, profile, scenario: database::Scenario::IncrementalPatch(patch.name), + parallel, backend, target, }; diff --git a/collector/src/compile/benchmark/parallel.rs b/collector/src/compile/benchmark/parallel.rs new file mode 100644 index 0000000000..d12c0a490c --- /dev/null +++ b/collector/src/compile/benchmark/parallel.rs @@ -0,0 +1,42 @@ +use std::fmt; + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct Parallel(pub u32); + +impl Parallel { + // Default parallel frontend options + pub fn default_opts() -> Vec { + database::Parallel::default_opts() + .iter() + .map(|v| (*v).into()) + .collect() + } + + pub fn par_n(self) -> String { + database::Parallel(self.0).par_n() + } +} + +impl From for Parallel { + fn from(item: u32) -> Self { + Self(item) + } +} + +impl From for Parallel { + fn from(value: database::Parallel) -> Self { + Parallel(value.0) + } +} + +impl From for database::Parallel { + fn from(value: Parallel) -> Self { + database::Parallel(value.0) + } +} + +impl fmt::Display for Parallel { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "-Zthreads={}", self.0) + } +} diff --git a/collector/src/compile/execute/bencher.rs b/collector/src/compile/execute/bencher.rs index f4b8d4f7bb..d28fe3e0b6 100644 --- a/collector/src/compile/execute/bencher.rs +++ b/collector/src/compile/execute/bencher.rs @@ -27,6 +27,7 @@ pub struct RecordedSelfProfile { profile: database::Profile, codegen_backend: database::CodegenBackend, target: database::Target, + parallel: database::Parallel, files: SelfProfileFiles, } @@ -87,6 +88,7 @@ impl<'a> BenchProcessor<'a> { } } + #[allow(clippy::too_many_arguments)] async fn insert_stats( &mut self, collection: CollectionId, @@ -95,6 +97,7 @@ impl<'a> BenchProcessor<'a> { backend: CodegenBackend, target: Target, stats: Stats, + parallel: database::Parallel, ) { let mut buf = FuturesUnordered::new(); for (stat, value) in stats.iter() { @@ -107,6 +110,7 @@ impl<'a> BenchProcessor<'a> { backend.into(), target.into(), stat, + parallel, value, )); } @@ -168,6 +172,7 @@ impl Processor for BenchProcessor<'_> { execute::store_documentation_size_into_stats(&mut res.0, &doc_dir); } } + let parallel = database::Parallel(data.parallel.0); let scenario = match data.scenario { Scenario::Full => database::Scenario::Empty, @@ -189,6 +194,7 @@ impl Processor for BenchProcessor<'_> { profile, codegen_backend: data.backend.into(), target: data.target.into(), + parallel, files, }); @@ -206,6 +212,7 @@ impl Processor for BenchProcessor<'_> { data.backend, data.target, res.0, + parallel, ) .await; @@ -255,6 +262,7 @@ impl Processor for BenchProcessor<'_> { benchmark: self.benchmark.clone(), profile: profile.profile, scenario: profile.scenario, + parallel: profile.parallel, backend: profile.codegen_backend, target: profile.target, }; diff --git a/collector/src/compile/execute/mod.rs b/collector/src/compile/execute/mod.rs index f54ae246bf..3c3c878188 100644 --- a/collector/src/compile/execute/mod.rs +++ b/collector/src/compile/execute/mod.rs @@ -1,6 +1,7 @@ //! Execute benchmarks. use crate::compile::benchmark::codegen_backend::CodegenBackend; +use crate::compile::benchmark::parallel::Parallel; use crate::compile::benchmark::patch::Patch; use crate::compile::benchmark::profile::Profile; use crate::compile::benchmark::scenario::Scenario; @@ -132,6 +133,7 @@ pub struct CargoProcess<'a> { pub touch_file: Option, pub jobserver: Option, pub target: Target, + pub parallel: Parallel, pub workspace_package: Option, } @@ -358,13 +360,14 @@ impl<'a> CargoProcess<'a> { // really. pub async fn run_rustc(&mut self, needs_final: bool) -> anyhow::Result<()> { log::info!( - "run_rustc with incremental={}, profile={:?}, scenario={:?}, patch={:?}, backend={:?}, target={:?}, phase={}", + "run_rustc with incremental={}, profile={:?}, scenario={:?}, patch={:?}, backend={:?}, target={:?}, parallel={}, phase={}", self.incremental, self.profile, self.processor_etc.as_ref().map(|v| v.1), self.processor_etc.as_ref().and_then(|v| v.3), self.backend, self.target, + self.parallel.0, if needs_final { "benchmark" } else { "dependencies" } ); @@ -406,6 +409,9 @@ impl<'a> CargoProcess<'a> { let mut cmd = self.base_command(self.cwd, cargo_subcommand); cmd.arg("-p").arg(self.get_pkgid(self.cwd)?); + if self.parallel.0 != 1 { + cmd.env("RUSTC_THREAD_COUNT", self.parallel.0.to_string()); + } match self.profile { Profile::Check => { cmd.arg("--profile").arg("check"); @@ -537,6 +543,7 @@ impl<'a> CargoProcess<'a> { patch, backend: self.backend, target: self.target, + parallel: self.parallel, }; match processor.process_output(&data, output).await { Ok(Retry::No) => return Ok(()), @@ -602,6 +609,7 @@ pub struct ProcessOutputData<'a> { patch: Option<&'a Patch>, backend: CodegenBackend, target: Target, + parallel: Parallel, } /// Trait used by `Benchmark::measure()` to provide different kinds of diff --git a/collector/src/compile/execute/profiler.rs b/collector/src/compile/execute/profiler.rs index c276f46861..2e64158ec3 100644 --- a/collector/src/compile/execute/profiler.rs +++ b/collector/src/compile/execute/profiler.rs @@ -131,11 +131,16 @@ impl Processor for ProfileProcessor<'_> { Box::pin(async move { fs::create_dir_all(self.output_dir)?; - // Produce a name of the form $PREFIX-$ID-$BENCHMARK-$PROFILE-$SCENARIO. + // Produce a name of the form $PREFIX-$ID-$BENCHMARK-$PROFILE-$SCENARIO-$PAR. let out_file = |prefix: &str| -> String { format!( - "{}-{}-{}-{:?}-{}", - prefix, self.id, data.name, data.profile, data.scenario_str + "{}-{}-{}-{:?}-{}-{}", + prefix, + self.id, + data.name, + data.profile, + data.scenario_str, + data.parallel.par_n() ) }; diff --git a/collector/src/lib.rs b/collector/src/lib.rs index b0dd797db7..428a98fe77 100644 --- a/collector/src/lib.rs +++ b/collector/src/lib.rs @@ -139,6 +139,15 @@ pub fn version_supports_incremental(version_str: &str) -> bool { } } +pub fn version_supports_parallel_frontend(version_str: &str) -> bool { + if let Ok(version) = version_str.parse::() { + version >= semver::Version::new(1, 33, 0) + } else { + assert!(version_str.starts_with("beta") || version_str.starts_with("master")); + true + } +} + /// Rounds serialized and deserialized floats to 2 decimal places. pub mod round_float { use serde::{Deserialize, Deserializer, Serializer}; diff --git a/collector/src/self_profile.rs b/collector/src/self_profile.rs index ccd4a402c9..2f1b030a0e 100644 --- a/collector/src/self_profile.rs +++ b/collector/src/self_profile.rs @@ -3,7 +3,7 @@ use crate::compile::execute::SelfProfileFiles; use analyzeme::ProfilingData; use anyhow::Context; use database::{ - ArtifactId, ArtifactIdNumber, CodegenBackend, CollectionId, Profile, Scenario, Target, + ArtifactId, ArtifactIdNumber, CodegenBackend, CollectionId, Parallel, Profile, Scenario, Target, }; use reqwest::StatusCode; use std::future::Future; @@ -31,6 +31,7 @@ pub enum SelfProfileId { benchmark: BenchmarkName, profile: Profile, scenario: Scenario, + parallel: Parallel, backend: CodegenBackend, target: Target, }, @@ -53,13 +54,14 @@ impl SelfProfileId { .join(profile.to_string()) .join(scenario.to_id()) .join(format!("self-profile-{collection}.mm_profdata.sz")), - // self-profile////// + // self-profile/////// // /self-profile.mm_profdata.sz SelfProfileId::Simple { artifact_id, benchmark, profile, scenario, + parallel, backend: codegen_backend, target, } => { @@ -74,6 +76,7 @@ impl SelfProfileId { .join(codegen_backend.to_string()) .join(profile.to_string()) .join(scenario.to_id()) + .join(parallel.par_n()) .join("self-profile.mm_profdata.sz") } } diff --git a/database/schema.md b/database/schema.md index 19c334271a..fd63eebc14 100644 --- a/database/schema.md +++ b/database/schema.md @@ -35,7 +35,8 @@ Here is the diagram for compile-time benchmarks: │ scenario │ │ cid │ │ │ backend │ │ value ├───┘ │ metric │ └──────────┘ - │ target │ + │ target | + │ parallel │ └───────────────┘ ``` @@ -101,7 +102,7 @@ Columns: ### pstat_series Describes the parametrization of a compile-time benchmark. Contains a unique combination -of a crate, profile, scenario and the metric being collected. +of a crate, profile, scenario, metric and parallel frontend option being collected. Columns: @@ -110,6 +111,7 @@ Columns: * **scenario** (`text`): Describes how much of the incremental cache is full. An empty incremental cache means that the compiler must do a full build. * **backend** (`text`): Codegen backend used for compilation, for example 'llvm' * **metric** (`text`): the type of metric being collected. +* **parallel** (`text`): Parallel frontend option ('-Zthreads=N'). This corresponds to a [`statistic description`](../docs/glossary.md). diff --git a/database/src/bin/import-sqlite.rs b/database/src/bin/import-sqlite.rs index c30512315f..392dbc7b09 100644 --- a/database/src/bin/import-sqlite.rs +++ b/database/src/bin/import-sqlite.rs @@ -45,7 +45,7 @@ async fn main() { let sqlite_aid = sqlite_conn.artifact_id(&aid).await; let postgres_aid = postgres_conn.artifact_id(&aid).await; - for (&(benchmark, profile, scenario, backend, target, metric), id) in + for (&(benchmark, profile, scenario, backend, target, metric, parallel), id) in sqlite_idx.compile_statistic_descriptions() { if benchmarks.insert(benchmark) { @@ -76,6 +76,7 @@ async fn main() { backend, target, metric.as_str(), + parallel, stat, ) .await; diff --git a/database/src/bin/postgres-to-sqlite.rs b/database/src/bin/postgres-to-sqlite.rs index a779e1cc94..51eb35ea98 100644 --- a/database/src/bin/postgres-to-sqlite.rs +++ b/database/src/bin/postgres-to-sqlite.rs @@ -189,12 +189,12 @@ impl Table for PstatSeries { } fn postgres_select_statement(&self, _since_weeks_ago: Option) -> String { - "select id, crate, profile, scenario, backend, target, metric from ".to_string() + "select id, crate, profile, scenario, backend, target, metric, parallel from ".to_string() + self.name() } fn sqlite_insert_statement(&self) -> &'static str { - "insert into pstat_series (id, crate, profile, scenario, backend, target, metric) VALUES (?, ?, ?, ?, ?, ?, ?)" + "insert into pstat_series (id, crate, profile, scenario, backend, target, metric, parallel) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" } fn sqlite_execute_insert(&self, statement: &mut rusqlite::Statement, row: tokio_postgres::Row) { diff --git a/database/src/bin/sqlite-to-postgres.rs b/database/src/bin/sqlite-to-postgres.rs index 1eb2c6888a..7a56a42515 100644 --- a/database/src/bin/sqlite-to-postgres.rs +++ b/database/src/bin/sqlite-to-postgres.rs @@ -243,6 +243,7 @@ struct PstatSeriesRow<'a> { backend: &'a str, target: &'a str, metric: &'a str, + parallel: u32, } impl Table for PstatSeries { @@ -251,11 +252,11 @@ impl Table for PstatSeries { } fn sqlite_attributes() -> &'static str { - "id, crate, profile, scenario, backend, target, metric" + "id, crate, profile, scenario, backend, target, metric, parallel" } fn postgres_attributes() -> &'static str { - "id, crate, profile, scenario, backend, target, metric" + "id, crate, profile, scenario, backend, target, metric, parallel" } fn postgres_generated_id_attribute() -> Option<&'static str> { @@ -272,6 +273,7 @@ impl Table for PstatSeries { backend: row.get_ref(4).unwrap().as_str().unwrap(), target: row.get_ref(5).unwrap().as_str().unwrap(), metric: row.get_ref(6).unwrap().as_str().unwrap(), + parallel: row.get(7).unwrap(), }) .unwrap(); } diff --git a/database/src/lib.rs b/database/src/lib.rs index 7820e306c3..56a33c4f0e 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -21,6 +21,32 @@ intern!(pub struct Metric); intern!(pub struct Benchmark); intern!(pub struct TargetName); +#[derive(Serialize, Deserialize, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct Parallel(pub u32); + +impl Parallel { + pub fn par_n(self) -> String { + format!("par{}", self.0) + } + // Default parallel frontend options + pub fn default_opts() -> Vec { + vec![Parallel(1), Parallel(4)] + } + pub fn single(self) -> bool { + self.0 == 1 + } +} + +impl std::str::FromStr for Parallel { + type Err = String; + fn from_str(s: &str) -> Result { + let v: u32 = s + .parse() + .map_err(|e: std::num::ParseIntError| e.to_string())?; + Ok(Self(v)) + } +} + pub fn intern_target_name(target: &str) -> TargetName { intern(target) } @@ -562,7 +588,15 @@ pub struct Index { artifacts: Indexed>, /// Id lookup of compile stat description ids /// For legacy reasons called `pstat_series` in the database, and so the name is kept here. - pstat_series: Indexed<(Benchmark, Profile, Scenario, CodegenBackend, Target, Metric)>, + pstat_series: Indexed<( + Benchmark, + Profile, + Scenario, + CodegenBackend, + Target, + Metric, + Parallel, + )>, /// Id lookup of runtime stat description ids runtime_pstat_series: Indexed<(Benchmark, Target, Metric)>, } @@ -685,6 +719,7 @@ pub enum DbLabel { backend: CodegenBackend, target: Target, metric: Metric, + parallel: Parallel, }, } @@ -704,9 +739,10 @@ impl Lookup for DbLabel { backend, metric, target, - } => index - .pstat_series - .get(&(*benchmark, *profile, *scenario, *backend, *target, *metric)), + parallel, + } => index.pstat_series.get(&( + *benchmark, *profile, *scenario, *backend, *target, *metric, *parallel, + )), } } } @@ -722,6 +758,15 @@ impl Lookup for ArtifactId { } pub type StatisticalDescriptionId = u32; +pub type CompileStatisticDescription = ( + Benchmark, + Profile, + Scenario, + CodegenBackend, + Target, + Metric, + Parallel, +); impl Index { pub async fn load(conn: &mut dyn pool::Connection) -> Index { @@ -754,7 +799,7 @@ impl Index { self.pstat_series .map .keys() - .map(|(_, _, _, _, _, metric)| metric) + .map(|(_, _, _, _, _, metric, _)| metric) .collect::>() .into_iter() .map(|s| s.to_string()) @@ -777,7 +822,7 @@ impl Index { self.pstat_series .map .keys() - .map(|(_, _, _, _, target, _)| target) + .map(|(_, _, _, _, target, _, _)| target) .collect::>() .into_iter() .cloned() @@ -790,12 +835,7 @@ impl Index { // for it as keeping indices around would be annoying. pub fn compile_statistic_descriptions( &self, - ) -> impl Iterator< - Item = ( - &(Benchmark, Profile, Scenario, CodegenBackend, Target, Metric), - StatisticalDescriptionId, - ), - > + '_ { + ) -> impl Iterator + '_ { self.pstat_series .map .iter() diff --git a/database/src/pool.rs b/database/src/pool.rs index 5de872082e..97aa04ab9b 100644 --- a/database/src/pool.rs +++ b/database/src/pool.rs @@ -5,7 +5,7 @@ use crate::{ BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectorConfig, CompileBenchmark, PendingBenchmarkRequests, Target, }; -use crate::{CollectionId, Index, Profile, Scenario}; +use crate::{CollectionId, Index, Parallel, Profile, Scenario}; use chrono::{DateTime, Utc}; use hashbrown::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; @@ -59,6 +59,7 @@ pub trait Connection: Send + Sync { backend: CodegenBackend, target: Target, metric: &str, + parallel: Parallel, value: f64, ); async fn record_runtime_statistic( @@ -743,6 +744,7 @@ mod tests { CodegenBackend::Llvm, Target::X86_64UnknownLinuxGnu, Metric::CacheMisses.as_str(), + Parallel(1), 1.0, ) .await; @@ -755,6 +757,7 @@ mod tests { benchmark: "benchmark".into(), profile: Profile::Check, scenario: Scenario::IncrementalFresh, + parallel: Parallel(1), backend: CodegenBackend::Llvm, target: Target::X86_64UnknownLinuxGnu, }]) diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index a537317e19..b2b37c7ad7 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -7,8 +7,8 @@ use crate::{ BenchmarkJobKind, BenchmarkJobStatus, BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestInsertResult, BenchmarkRequestStatus, BenchmarkRequestType, BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectionId, CollectorConfig, - Commit, CommitType, CompileBenchmark, Date, Index, PendingBenchmarkRequests, Profile, Scenario, - Target, BENCHMARK_JOB_STATUS_FAILURE_STR, BENCHMARK_JOB_STATUS_IN_PROGRESS_STR, + Commit, CommitType, CompileBenchmark, Date, Index, Parallel, PendingBenchmarkRequests, Profile, + Scenario, Target, BENCHMARK_JOB_STATUS_FAILURE_STR, BENCHMARK_JOB_STATUS_IN_PROGRESS_STR, BENCHMARK_JOB_STATUS_QUEUED_STR, BENCHMARK_JOB_STATUS_SUCCESS_STR, BENCHMARK_REQUEST_MASTER_STR, BENCHMARK_REQUEST_RELEASE_STR, BENCHMARK_REQUEST_STATUS_ARTIFACTS_READY_STR, BENCHMARK_REQUEST_STATUS_COMPLETED_STR, @@ -441,6 +441,12 @@ static MIGRATIONS: &[&str] = &[ "#, r#"ALTER TABLE job_queue ADD COLUMN is_optional BOOLEAN NOT NULL DEFAULT FALSE"#, r#"ALTER TABLE benchmark_request ADD COLUMN targets TEXT NOT NULL DEFAULT ''"#, + // Add parallel to pstat_series + r#" + alter table pstat_series add parallel int not null default 1; + alter table pstat_series drop constraint test_case; + alter table pstat_series add constraint test_case UNIQUE(crate, profile, scenario, backend, target, metric, parallel); + "#, ]; #[async_trait::async_trait] @@ -633,8 +639,8 @@ impl PostgresConnection { .await .unwrap(), get_error: conn.prepare("select context, message from error where aid = $1").await.unwrap(), - insert_pstat_series: conn.prepare("insert into pstat_series (crate, profile, scenario, backend, target, metric) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(), - select_pstat_series: conn.prepare("select id from pstat_series where crate = $1 and profile = $2 and scenario = $3 and backend = $4 and target = $5 and metric = $6").await.unwrap(), + insert_pstat_series: conn.prepare("insert into pstat_series (crate, profile, scenario, backend, target, metric, parallel) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(), + select_pstat_series: conn.prepare("select id from pstat_series where crate = $1 and profile = $2 and scenario = $3 and backend = $4 and target = $5 and metric = $6 and parallel = $7").await.unwrap(), collection_id: conn.prepare("insert into collection (perf_commit) VALUES ($1) returning id").await.unwrap(), get_benchmarks: conn.prepare(" select name, category @@ -686,7 +692,7 @@ impl PostgresConnection { WHERE tag IS NOT NULL ").await.unwrap(), get_compile_test_cases_with_measurements: conn.prepare(" - SELECT DISTINCT crate, profile, scenario, backend, target + SELECT DISTINCT crate, profile, scenario, backend, target, parallel FROM pstat_series WHERE id IN ( SELECT DISTINCT series @@ -885,7 +891,7 @@ where pstat_series: self .conn() .query( - "select id, crate, profile, scenario, backend, target, metric from pstat_series;", + "select id, crate, profile, scenario, backend, target, metric, parallel from pstat_series;", &[], ) .await @@ -901,6 +907,7 @@ where CodegenBackend::from_str(row.get::<_, String>(4).as_str()).unwrap(), Target::from_str(row.get::<_, String>(5).as_str()).unwrap(), row.get::<_, String>(6).as_str().into(), + Parallel(row.get::<_, i32>(7) as u32), ), ) }) @@ -1021,17 +1028,21 @@ where backend: CodegenBackend, target: Target, metric: &str, + parallel: Parallel, stat: f64, ) { let profile = profile.to_string(); let scenario = scenario.to_string(); let backend = backend.to_string(); let target = target.to_string(); + let parallel = parallel.0 as i32; let sid = self .conn() .query_opt( &self.statements().select_pstat_series, - &[&benchmark, &profile, &scenario, &backend, &target, &metric], + &[ + &benchmark, &profile, &scenario, &backend, &target, &metric, ¶llel, + ], ) .await .unwrap(); @@ -1041,14 +1052,18 @@ where self.conn() .query_opt( &self.statements().insert_pstat_series, - &[&benchmark, &profile, &scenario, &backend, &target, &metric], + &[ + &benchmark, &profile, &scenario, &backend, &target, &metric, ¶llel, + ], ) .await .unwrap(); self.conn() .query_one( &self.statements().select_pstat_series, - &[&benchmark, &profile, &scenario, &backend, &target, &metric], + &[ + &benchmark, &profile, &scenario, &backend, &target, &metric, ¶llel, + ], ) .await .unwrap() @@ -1625,6 +1640,7 @@ where scenario: row.get::<_, &str>(2).parse().unwrap(), backend: CodegenBackend::from_str(row.get::<_, &str>(3)).unwrap(), target: Target::from_str(row.get::<_, &str>(4)).unwrap(), + parallel: Parallel(row.get::<_, i32>(5) as u32), }) .collect()) } diff --git a/database/src/pool/sqlite.rs b/database/src/pool/sqlite.rs index cd9115d80f..1ea997459d 100644 --- a/database/src/pool/sqlite.rs +++ b/database/src/pool/sqlite.rs @@ -6,7 +6,8 @@ use crate::{ ArtifactId, Benchmark, BenchmarkJob, BenchmarkJobConclusion, BenchmarkJobKind, BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestInsertResult, BenchmarkRequestStatus, BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectionId, CollectorConfig, - Commit, CommitType, CompileBenchmark, Date, PendingBenchmarkRequests, Profile, Target, + Commit, CommitType, CompileBenchmark, Date, Parallel, PendingBenchmarkRequests, Profile, + Target, }; use crate::{ArtifactIdNumber, Index}; use chrono::{DateTime, TimeZone, Utc}; @@ -448,6 +449,25 @@ static MIGRATIONS: &[Migration] = &[ ALTER TABLE runtime_pstat_series_with_target RENAME TO runtime_pstat_series; "#, ), + // Add parallel as an unique constraint, defaulting to '1' + Migration::without_foreign_key_constraints( + r#" + create table pstat_series_with_parallel( + id integer primary key not null, + crate text not null references benchmark(name) on delete cascade on update cascade, + profile text not null, + scenario text not null, + backend text not null, + target text not null default 'x86_64-unknown-linux-gnu', + metric text not null, + parallel integer not null default 1, + UNIQUE(crate, profile, scenario, backend, target, metric, parallel) + ); + insert into pstat_series_with_parallel select id, crate, profile, scenario, backend, 'x86_64-unknown-linux-gnu', metric, 1 from pstat_series; + drop table pstat_series; + alter table pstat_series_with_parallel rename to pstat_series; + "#, + ), ]; #[async_trait::async_trait] @@ -575,7 +595,7 @@ impl Connection for SqliteConnection { let pstat_series = self .raw() .prepare( - "select id, crate, profile, scenario, backend, target, metric from pstat_series;", + "select id, crate, profile, scenario, backend, target, metric, parallel from pstat_series;", ) .unwrap() .query_map(params![], |row| { @@ -588,6 +608,7 @@ impl Connection for SqliteConnection { CodegenBackend::from_str(row.get::<_, String>(4)?.as_str()).unwrap(), Target::from_str(row.get::<_, String>(5)?.as_str()).unwrap(), row.get::<_, String>(6)?.as_str().into(), + Parallel(row.get::<_, i32>(7)? as u32), ), )) }) @@ -724,27 +745,30 @@ impl Connection for SqliteConnection { backend: CodegenBackend, target: Target, metric: &str, + parallel: Parallel, value: f64, ) { let profile = profile.to_string(); let scenario = scenario.to_string(); let backend = backend.to_string(); let target = target.to_string(); - self.raw_ref().execute("insert or ignore into pstat_series (crate, profile, scenario, backend, target, metric) VALUES (?, ?, ?, ?, ?, ?)", params![ + self.raw_ref().execute("insert or ignore into pstat_series (crate, profile, scenario, backend, target, metric, parallel) VALUES (?, ?, ?, ?, ?, ?, ?)", params![ &benchmark, &profile, &scenario, &backend, &target, &metric, + ¶llel.0, ]).unwrap(); - let sid: i32 = self.raw_ref().query_row("select id from pstat_series where crate = ? and profile = ? and scenario = ? and backend = ? and target = ? and metric = ?", params![ + let sid: i32 = self.raw_ref().query_row("select id from pstat_series where crate = ? and profile = ? and scenario = ? and backend = ? and target = ? and metric = ? and parallel = ?", params![ &benchmark, &profile, &scenario, &backend, &target, &metric, + ¶llel.0, ], |r| r.get(0)).unwrap(); self.raw_ref() .execute( @@ -1082,7 +1106,7 @@ impl Connection for SqliteConnection { Ok(self .raw_ref() .prepare_cached( - "SELECT DISTINCT crate, profile, scenario, backend, target + "SELECT DISTINCT crate, profile, scenario, backend, target, parallel FROM pstat_series WHERE id IN ( SELECT DISTINCT series @@ -1097,6 +1121,7 @@ impl Connection for SqliteConnection { scenario: row.get::<_, String>(2)?.parse().unwrap(), backend: row.get::<_, String>(3)?.parse().unwrap(), target: row.get::<_, String>(4)?.parse().unwrap(), + parallel: Parallel(row.get::<_, i32>(5)? as u32), }) })? .collect::>()?) diff --git a/database/src/selector.rs b/database/src/selector.rs index 042dbf932a..3d4881c507 100644 --- a/database/src/selector.rs +++ b/database/src/selector.rs @@ -29,7 +29,7 @@ use std::{ use crate::{ interpolate::Interpolate, metric::Metric, ArtifactId, ArtifactIdIter, Benchmark, - CodegenBackend, Connection, Index, Lookup, Profile, Scenario, Target, + CodegenBackend, Connection, Index, Lookup, Parallel, Profile, Scenario, Target, }; #[derive(Debug)] @@ -193,6 +193,7 @@ pub struct CompileBenchmarkQuery { profile: Selector, backend: Selector, metric: Selector, + parallel: Selector, target: Selector, } @@ -217,6 +218,11 @@ impl CompileBenchmarkQuery { self } + pub fn parallel(mut self, selector: Selector) -> Self { + self.parallel = selector; + self + } + pub fn target(mut self, selector: Selector) -> Self { self.target = selector; self @@ -233,6 +239,7 @@ impl CompileBenchmarkQuery { profile: Selector::All, scenario: Selector::All, backend: Selector::All, + parallel: Selector::All, metric: Selector::One(metric.as_str().into()), target: Selector::All, } @@ -246,6 +253,7 @@ impl Default for CompileBenchmarkQuery { scenario: Selector::All, profile: Selector::All, backend: Selector::All, + parallel: Selector::All, metric: Selector::All, target: Selector::All, } @@ -263,21 +271,23 @@ impl BenchmarkQuery for CompileBenchmarkQuery { ) -> Result>, String> { let mut statistic_descriptions: Vec<_> = index .compile_statistic_descriptions() - .filter(|(&(b, p, s, backend, target, metric), _)| { + .filter(|(&(b, p, s, backend, target, metric, parallel), _)| { self.benchmark.matches(b) && self.profile.matches(p) && self.scenario.matches(s) && self.backend.matches(backend) && self.target.matches(target) && self.metric.matches(metric) + && self.parallel.matches(parallel) }) .map( - |(&(benchmark, profile, scenario, backend, target, metric), sid)| { + |(&(benchmark, profile, scenario, backend, target, metric, parallel), sid)| { ( CompileTestCase { benchmark, profile, scenario, + parallel, backend, target, }, @@ -334,6 +344,7 @@ pub struct CompileTestCase { pub benchmark: Benchmark, pub profile: Profile, pub scenario: Scenario, + pub parallel: Parallel, pub backend: CodegenBackend, pub target: Target, } diff --git a/site/frontend/src/components/perfetto-link.vue b/site/frontend/src/components/perfetto-link.vue index db30a7fa95..6ef79b92f8 100644 --- a/site/frontend/src/components/perfetto-link.vue +++ b/site/frontend/src/components/perfetto-link.vue @@ -15,6 +15,7 @@ const link = computed(() => { props.artifact.commit, `${props.testCase.benchmark}`, props.testCase.scenario, + props.testCase.parallel.toString(), props.testCase.profile.toLowerCase(), props.testCase.backend, props.testCase.target diff --git a/site/frontend/src/graph/api.ts b/site/frontend/src/graph/api.ts index f4c1a16ca3..8881990b28 100644 --- a/site/frontend/src/graph/api.ts +++ b/site/frontend/src/graph/api.ts @@ -12,9 +12,19 @@ export async function loadGraphs( stat: selector.stat, benchmark: selector.benchmark, scenario: selector.scenario, + parallel: selector.parallel, profile: selector.profile, backend: selector.backend, target: selector.target, }; - return await getJson(GRAPH_DATA_URL, params); + const dict: Dict = Object.entries(params).reduce( + (acc, [key, value]) => { + if (value !== null && value !== undefined) { + acc[key] = value; + } + return acc; + }, + {} as Dict + ); + return await getJson(GRAPH_DATA_URL, dict); } diff --git a/site/frontend/src/graph/data.ts b/site/frontend/src/graph/data.ts index 8180f94c8f..16be80b8de 100644 --- a/site/frontend/src/graph/data.ts +++ b/site/frontend/src/graph/data.ts @@ -8,6 +8,7 @@ export interface GraphsSelector { stat: string; benchmark: string | null; scenario: string | null; + parallel: string | null; profile: string | null; backend: string | null; target: string | null; @@ -21,8 +22,8 @@ export interface Series { // Graph data received from the server export interface CompileGraphData { commits: Array<[number, string]>; - // benchmark -> profile -> scenario -> series - benchmarks: Dict>>; + // benchmark -> profile -> parallel -> scenario -> series + benchmarks: Dict>>>; } export interface RuntimeGraphData { diff --git a/site/frontend/src/graph/render.ts b/site/frontend/src/graph/render.ts index 61fda201f0..1dc2624399 100644 --- a/site/frontend/src/graph/render.ts +++ b/site/frontend/src/graph/render.ts @@ -376,10 +376,14 @@ function genPlotOpts({ function normalizeData(data: CompileGraphData) { function optInterpolated(profile) { - for (const scenario in profile) { - profile[scenario].interpolated_indices = new Set( - profile[scenario].interpolated_indices - ); + for (const parallel in profile) { + let par = profile[parallel]; + for (const scenario in par) { + par[scenario].interpolated_indices = new Set( + par[scenario].interpolated_indices + ); + } + profile[parallel] = par; } return profile; @@ -430,10 +434,7 @@ export function renderPlots( let i = 0; for (let profile in profiles) { - let scenarios = profiles[profile]; - let scenarioNames = Object.keys(scenarios); - scenarioNames.sort(); - + let parallels = profiles[profile]; let yAxis = selector.stat; let yAxisUnit = null; @@ -486,24 +487,34 @@ export function renderPlots( let plotData = [xVals]; let otherColorIdx = 0; + let indices = null; - for (let scenarioName of scenarioNames) { - let yVals = scenarios[scenarioName].points; - let color = - commonCacheStateColors[scenarioName] || - otherCacheStateColors[otherColorIdx++]; - - plotData.push(yVals); + for (let parallel in parallels) { + let scenarios = parallels[parallel]; + let scenarioNames = Object.keys(scenarios); + scenarioNames.sort(); - seriesOpts.push({ - label: scenarioName, - width: devicePixelRatio, - stroke: color, - }); + for (let scenarioName of scenarioNames) { + let yVals = scenarios[scenarioName].points; + if (indices === null) { + indices = scenarios[scenarioName].interpolated_indices; + } + let color = + parallel == "1" + ? commonCacheStateColors[scenarioName] || + otherCacheStateColors[otherColorIdx++] + : otherCacheStateColors[otherColorIdx++]; + + plotData.push(yVals); + + seriesOpts.push({ + label: scenarioName + "_par" + parallel, + width: devicePixelRatio, + stroke: color, + }); + } } - let indices = scenarios[Object.keys(scenarios)[0]].interpolated_indices; - let plotOpts = genPlotOpts({ width, height, diff --git a/site/frontend/src/pages/bootstrap/data-selector.vue b/site/frontend/src/pages/bootstrap/data-selector.vue index 1e9a723c53..1ec5180e72 100644 --- a/site/frontend/src/pages/bootstrap/data-selector.vue +++ b/site/frontend/src/pages/bootstrap/data-selector.vue @@ -16,13 +16,13 @@ const startRef = ref(null); const endRef = ref(null); onMounted(() => { - startRef.value.value = props.start; - endRef.value.value = props.end; + if (startRef.value) startRef.value.value = props.start; + if (endRef.value) endRef.value.value = props.end; }); function submitSettings() { - const start = startRef.value.value; - const end = endRef.value.value; + const start = startRef.value?.value ?? ""; + const end = endRef.value?.value ?? ""; const params = {start, end}; emit("change", params); diff --git a/site/frontend/src/pages/compare/compile/aggregations.vue b/site/frontend/src/pages/compare/compile/aggregations.vue index 0bcf1e2a6c..3d067ac2dc 100644 --- a/site/frontend/src/pages/compare/compile/aggregations.vue +++ b/site/frontend/src/pages/compare/compile/aggregations.vue @@ -88,6 +88,18 @@ const totalAfter = computed(() => +
+
Parallel
+
+
+
par{{ parallel }}
+ +
+
+
Category
diff --git a/site/frontend/src/pages/compare/compile/benchmarks.vue b/site/frontend/src/pages/compare/compile/benchmarks.vue index 975b4e56a1..26be9cc9b9 100644 --- a/site/frontend/src/pages/compare/compile/benchmarks.vue +++ b/site/frontend/src/pages/compare/compile/benchmarks.vue @@ -1,5 +1,5 @@