ci: Run tests only for the changed packages and their dependants (#48380)

We're still gonna run a full test suite on main and in non-PR workflows
exercising tests


Release Notes:

- N/A

.github/workflows/run_tests.yml

@@ -46,12 +46,58 @@ jobs:
             echo "${output_name}=false" >> "$GITHUB_OUTPUT"
         }
 
+        # Check for changes that require full rebuild (no filter)
+        # Direct pushes to main/stable/preview always run full suite
+        if [ -z "$GITHUB_BASE_REF" ]; then
+          echo "Not a PR, running full test suite"
+          echo "changed_packages=" >> "$GITHUB_OUTPUT"
+        elif echo "$CHANGED_FILES" | grep -qP '^(rust-toolchain\.toml|\.cargo/|\.github/)'; then
+          echo "Toolchain, .github or cargo config changed, will run all tests"
+          echo "changed_packages=" >> "$GITHUB_OUTPUT"
+        else
+          # Extract changed packages from file paths
+          FILE_CHANGED_PKGS=$(echo "$CHANGED_FILES" | \
+            grep -oP '^(crates|tooling)/\K[^/]+' | \
+            sort -u || true)
+
+          # If assets/ changed, add crates that depend on those assets
+          if echo "$CHANGED_FILES" | grep -qP '^assets/'; then
+            FILE_CHANGED_PKGS=$(printf '%s\n%s\n%s\n%s' "$FILE_CHANGED_PKGS" "settings" "storybook" "assets" | sort -u)
+          fi
+
+          # Parse Cargo.lock diff for added/changed crates
+          LOCK_CHANGED_PKGS=""
+          if echo "$CHANGED_FILES" | grep -qP '^Cargo\.lock$'; then
+            echo "Cargo.lock changed, analyzing diff..."
+            LOCK_CHANGED_PKGS=$(git diff "$COMPARE_REV" "${{ github.sha }}" -- Cargo.lock | \
+              grep -oP '^[+-]name = "\K[^"]+' | \
+              sort -u || true)
+          fi
+
+          # Combine all changed packages
+          ALL_CHANGED_PKGS=$(printf '%s\n%s' "$FILE_CHANGED_PKGS" "$LOCK_CHANGED_PKGS" | sort -u | grep -v '^$' || true)
+
+          if [ -z "$ALL_CHANGED_PKGS" ]; then
+            echo "No package changes detected, will run all tests"
+            echo "changed_packages=" >> "$GITHUB_OUTPUT"
+          else
+            # Build nextest filterset with rdeps for each package
+            FILTERSET=$(echo "$ALL_CHANGED_PKGS" | \
+              sed 's/.*/rdeps(&)/' | \
+              tr '\n' '|' | \
+              sed 's/|$//')
+            echo "Changed packages filterset: $FILTERSET"
+            echo "changed_packages=$FILTERSET" >> "$GITHUB_OUTPUT"
+          fi
+        fi
+
         check_pattern "run_action_checks" '^\.github/(workflows/|actions/|actionlint.yml)|tooling/xtask|script/' -qP
         check_pattern "run_docs" '^(docs/|crates/.*\.rs)' -qP
         check_pattern "run_licenses" '^(Cargo.lock|script/.*licenses)' -qP
         check_pattern "run_nix" '^(nix/|flake\.|Cargo\.|rust-toolchain.toml|\.cargo/config.toml)' -qP
         check_pattern "run_tests" '^(docs/|script/update_top_ranking_issues/|\.github/(ISSUE_TEMPLATE|workflows/(?!run_tests)))' -qvP
     outputs:
+      changed_packages: ${{ steps.filter.outputs.changed_packages }}
       run_action_checks: ${{ steps.filter.outputs.run_action_checks }}
       run_docs: ${{ steps.filter.outputs.run_docs }}
       run_licenses: ${{ steps.filter.outputs.run_licenses }}
@@ -179,7 +225,7 @@ jobs:
       run: ./script/clear-target-dir-if-larger-than.ps1 250
       shell: pwsh
     - name: steps::cargo_nextest
-      run: cargo nextest run --workspace --no-fail-fast
+      run: cargo nextest run --workspace --no-fail-fast${{ needs.orchestrate.outputs.changed_packages && format(' -E "{0}"', needs.orchestrate.outputs.changed_packages) || '' }}
       shell: pwsh
     - name: steps::cleanup_cargo_config
       if: always()
@@ -221,7 +267,7 @@ jobs:
     - name: steps::clear_target_dir_if_large
       run: ./script/clear-target-dir-if-larger-than 250
     - name: steps::cargo_nextest
-      run: cargo nextest run --workspace --no-fail-fast
+      run: cargo nextest run --workspace --no-fail-fast${{ needs.orchestrate.outputs.changed_packages && format(' -E "{0}"', needs.orchestrate.outputs.changed_packages) || '' }}
     - name: steps::cleanup_cargo_config
       if: always()
       run: |
@@ -263,7 +309,7 @@ jobs:
     - name: steps::clear_target_dir_if_large
       run: ./script/clear-target-dir-if-larger-than 300
     - name: steps::cargo_nextest
-      run: cargo nextest run --workspace --no-fail-fast
+      run: cargo nextest run --workspace --no-fail-fast${{ needs.orchestrate.outputs.changed_packages && format(' -E "{0}"', needs.orchestrate.outputs.changed_packages) || '' }}
     - name: steps::cleanup_cargo_config
       if: always()
       run: |

tooling/xtask/src/tasks/workflows/extension_tests.rs

@@ -2,7 +2,7 @@ use gh_workflow::*;
 use indoc::indoc;
 
 use crate::tasks::workflows::{
-    run_tests::{orchestrate, tests_pass},
+    run_tests::{orchestrate_without_package_filter, tests_pass},
     runners,
     steps::{self, CommonJobConditions, FluentBuilder, NamedJob, named},
     vars::{PathCondition, StepOutput, one_workflow_per_non_main_branch},
@@ -18,7 +18,8 @@ pub(crate) fn extension_tests() -> Workflow {
     let should_check_rust = PathCondition::new("check_rust", r"^(Cargo.lock|Cargo.toml|.*\.rs)$");
     let should_check_extension = PathCondition::new("check_extension", r"^.*\.scm$");
 
-    let orchestrate = orchestrate(&[&should_check_rust, &should_check_extension]);
+    let orchestrate =
+        orchestrate_without_package_filter(&[&should_check_rust, &should_check_extension]);
 
     let jobs = [
         orchestrate,

tooling/xtask/src/tasks/workflows/release.rs

@@ -13,9 +13,9 @@ const CURRENT_ACTION_RUN_URL: &str =
     "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}";
 
 pub(crate) fn release() -> Workflow {
-    let macos_tests = run_tests::run_platform_tests(Platform::Mac);
-    let linux_tests = run_tests::run_platform_tests(Platform::Linux);
-    let windows_tests = run_tests::run_platform_tests(Platform::Windows);
+    let macos_tests = run_tests::run_platform_tests_no_filter(Platform::Mac);
+    let linux_tests = run_tests::run_platform_tests_no_filter(Platform::Linux);
+    let windows_tests = run_tests::run_platform_tests_no_filter(Platform::Windows);
     let macos_clippy = run_tests::clippy(Platform::Mac);
     let linux_clippy = run_tests::clippy(Platform::Linux);
     let windows_clippy = run_tests::clippy(Platform::Windows);

tooling/xtask/src/tasks/workflows/release_nightly.rs

@@ -5,7 +5,7 @@ use crate::tasks::workflows::{
         prep_release_artifacts,
     },
     run_bundling::{bundle_linux, bundle_mac, bundle_windows},
-    run_tests::{clippy, run_platform_tests},
+    run_tests::{clippy, run_platform_tests_no_filter},
     runners::{Arch, Platform, ReleaseChannel},
     steps::{CommonJobConditions, FluentBuilder, NamedJob},
 };
@@ -17,7 +17,7 @@ use gh_workflow::*;
 pub fn release_nightly() -> Workflow {
     let style = check_style();
     // run only on windows as that's our fastest platform right now.
-    let tests = run_platform_tests(Platform::Windows);
+    let tests = run_platform_tests_no_filter(Platform::Windows);
     let clippy_job = clippy(Platform::Windows);
     let nightly = Some(ReleaseChannel::Nightly);
 

tooling/xtask/src/tasks/workflows/run_tests.rs

@@ -115,6 +115,14 @@ pub(crate) fn run_tests() -> Workflow {
 // Generates a bash script that checks changed files against regex patterns
 // and sets GitHub output variables accordingly
 pub fn orchestrate(rules: &[&PathCondition]) -> NamedJob {
+    orchestrate_impl(rules, true)
+}
+
+pub fn orchestrate_without_package_filter(rules: &[&PathCondition]) -> NamedJob {
+    orchestrate_impl(rules, false)
+}
+
+fn orchestrate_impl(rules: &[&PathCondition], include_package_filter: bool) -> NamedJob {
     let name = "orchestrate".to_owned();
     let step_name = "filter".to_owned();
     let mut script = String::new();
@@ -144,6 +152,61 @@ pub fn orchestrate(rules: &[&PathCondition]) -> NamedJob {
 
     let mut outputs = IndexMap::new();
 
+    if include_package_filter {
+        script.push_str(indoc::indoc! {r#"
+        # Check for changes that require full rebuild (no filter)
+        # Direct pushes to main/stable/preview always run full suite
+        if [ -z "$GITHUB_BASE_REF" ]; then
+          echo "Not a PR, running full test suite"
+          echo "changed_packages=" >> "$GITHUB_OUTPUT"
+        elif echo "$CHANGED_FILES" | grep -qP '^(rust-toolchain\.toml|\.cargo/|\.github/)'; then
+          echo "Toolchain, .github or cargo config changed, will run all tests"
+          echo "changed_packages=" >> "$GITHUB_OUTPUT"
+        else
+          # Extract changed packages from file paths
+          FILE_CHANGED_PKGS=$(echo "$CHANGED_FILES" | \
+            grep -oP '^(crates|tooling)/\K[^/]+' | \
+            sort -u || true)
+
+          # If assets/ changed, add crates that depend on those assets
+          if echo "$CHANGED_FILES" | grep -qP '^assets/'; then
+            FILE_CHANGED_PKGS=$(printf '%s\n%s\n%s\n%s' "$FILE_CHANGED_PKGS" "settings" "storybook" "assets" | sort -u)
+          fi
+
+          # Parse Cargo.lock diff for added/changed crates
+          LOCK_CHANGED_PKGS=""
+          if echo "$CHANGED_FILES" | grep -qP '^Cargo\.lock$'; then
+            echo "Cargo.lock changed, analyzing diff..."
+            LOCK_CHANGED_PKGS=$(git diff "$COMPARE_REV" "${{ github.sha }}" -- Cargo.lock | \
+              grep -oP '^[+-]name = "\K[^"]+' | \
+              sort -u || true)
+          fi
+
+          # Combine all changed packages
+          ALL_CHANGED_PKGS=$(printf '%s\n%s' "$FILE_CHANGED_PKGS" "$LOCK_CHANGED_PKGS" | sort -u | grep -v '^$' || true)
+
+          if [ -z "$ALL_CHANGED_PKGS" ]; then
+            echo "No package changes detected, will run all tests"
+            echo "changed_packages=" >> "$GITHUB_OUTPUT"
+          else
+            # Build nextest filterset with rdeps for each package
+            FILTERSET=$(echo "$ALL_CHANGED_PKGS" | \
+              sed 's/.*/rdeps(&)/' | \
+              tr '\n' '|' | \
+              sed 's/|$//')
+            echo "Changed packages filterset: $FILTERSET"
+            echo "changed_packages=$FILTERSET" >> "$GITHUB_OUTPUT"
+          fi
+        fi
+
+    "#});
+
+        outputs.insert(
+            "changed_packages".to_owned(),
+            format!("${{{{ steps.{}.outputs.changed_packages }}}}", step_name),
+        );
+    }
+
     for rule in rules {
         assert!(
             rule.set_by_step
@@ -328,6 +391,14 @@ pub(crate) fn clippy(platform: Platform) -> NamedJob {
 }
 
 pub(crate) fn run_platform_tests(platform: Platform) -> NamedJob {
+    run_platform_tests_impl(platform, true)
+}
+
+pub(crate) fn run_platform_tests_no_filter(platform: Platform) -> NamedJob {
+    run_platform_tests_impl(platform, false)
+}
+
+fn run_platform_tests_impl(platform: Platform, filter_packages: bool) -> NamedJob {
     let runner = match platform {
         Platform::Windows => runners::WINDOWS_DEFAULT,
         Platform::Linux => runners::LINUX_DEFAULT,
@@ -367,7 +438,14 @@ pub(crate) fn run_platform_tests(platform: Platform) -> NamedJob {
                 |job| job.add_step(steps::cargo_install_nextest()),
             )
             .add_step(steps::clear_target_dir_if_large(platform))
-            .add_step(steps::cargo_nextest(platform))
+            .when(filter_packages, |job| {
+                job.add_step(
+                    steps::cargo_nextest(platform).with_changed_packages_filter("orchestrate"),
+                )
+            })
+            .when(!filter_packages, |job| {
+                job.add_step(steps::cargo_nextest(platform))
+            })
             .add_step(steps::cleanup_cargo_config(platform)),
     }
 }

tooling/xtask/src/tasks/workflows/steps.rs

@@ -22,6 +22,23 @@ impl Nextest {
         }
         self.into()
     }
+
+    #[allow(dead_code)]
+    pub(crate) fn with_filter_expr(mut self, filter_expr: &str) -> Self {
+        if let Some(nextest_command) = self.0.value.run.as_mut() {
+            nextest_command.push_str(&format!(r#" -E "{filter_expr}""#));
+        }
+        self
+    }
+
+    pub(crate) fn with_changed_packages_filter(mut self, orchestrate_job: &str) -> Self {
+        if let Some(nextest_command) = self.0.value.run.as_mut() {
+            nextest_command.push_str(&format!(
+                r#"${{{{ needs.{orchestrate_job}.outputs.changed_packages && format(' -E "{{0}}"', needs.{orchestrate_job}.outputs.changed_packages) || '' }}}}"#
+            ));
+        }
+        self
+    }
 }
 
 impl From<Nextest> for Step<Run> {