From 665558bd897988935de08c56f0506b33cca8c0f0 Mon Sep 17 00:00:00 2001 From: emmaoke-w Date: Thu, 21 May 2026 12:36:07 +0200 Subject: [PATCH 1/2] WPB-25800 support multiple test case ids per UI test --- .../com/wire/android/tests/support/suite/AllureLabelsRule.kt | 4 +++- .../com/wire/android/tests/support/suite/TaggedFilter.kt | 2 +- .../kotlin/com/wire/android/tests/support/tags/TestCaseId.kt | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/AllureLabelsRule.kt b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/AllureLabelsRule.kt index 784f19c01e3..35c72ec8bcd 100644 --- a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/AllureLabelsRule.kt +++ b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/AllureLabelsRule.kt @@ -46,7 +46,9 @@ class AllureLabelsRule : TestRule { .filterIsInstance() .firstOrNull() ?.let { anno -> - Allure.label("tag", anno.value) + anno.value.forEach { testCaseId -> + Allure.label("tag", testCaseId) + } } // ---- Category → tag: criticalFlow, regression, ... ---- diff --git a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt index 52d0c69e5d2..b4c098d17a6 100644 --- a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt +++ b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt @@ -171,7 +171,7 @@ class TaggedFilter : Filter() { // 1) TestCaseId filterTestCaseId?.let { wantedId -> val testCaseAnno = annotations.filterIsInstance().firstOrNull() - if (testCaseAnno == null || testCaseAnno.value != wantedId) { + if (testCaseAnno == null || !testCaseAnno.value.contains(wantedId)) { return false } } diff --git a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/tags/TestCaseId.kt b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/tags/TestCaseId.kt index ccc71147a27..4d001ae7599 100644 --- a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/tags/TestCaseId.kt +++ b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/tags/TestCaseId.kt @@ -18,9 +18,9 @@ package com.wire.android.tests.support.tags /** - * Annotation used to assign a unique Test Case ID to a test. + * Annotation used to assign one or more Test Case IDs to a test. * Example: @TestCaseId("TC-4265") */ @Retention(AnnotationRetention.RUNTIME) @Target(AnnotationTarget.FUNCTION, AnnotationTarget.CLASS) -annotation class TestCaseId(val value: String) +annotation class TestCaseId(vararg val value: String) From 49af684c160652adee52febca37231cd05d204e3 Mon Sep 17 00:00:00 2001 From: emmaoke-w Date: Thu, 21 May 2026 12:36:39 +0200 Subject: [PATCH 2/2] fix upgrade test selection for UI workflow categories --- .../qa-android-critical-flow-tests.yml | 15 ++++++-- scripts/qa_android_ui_tests/README.md | 34 +++++++++++++++++++ scripts/qa_android_ui_tests/run_ui_tests.sh | 3 ++ .../tests/support/suite/TaggedFilter.kt | 20 +++++++++-- .../tests/support/suite/TaggedTestRunner.kt | 2 ++ 5 files changed, 68 insertions(+), 6 deletions(-) diff --git a/.github/workflows/qa-android-critical-flow-tests.yml b/.github/workflows/qa-android-critical-flow-tests.yml index b1f675f0764..efe2ee32b6b 100644 --- a/.github/workflows/qa-android-critical-flow-tests.yml +++ b/.github/workflows/qa-android-critical-flow-tests.yml @@ -310,7 +310,7 @@ jobs: bash scripts/qa_android_ui_tests/run_ui_tests.sh elif [[ "${IS_UPGRADE}" == "true" && -z "${RESOLVED_TESTCASE_ID}" ]]; then # For broad upgrade runs, run the selected normal tests on the new APK first. - # Then run only upgrade tests afterward, starting them from the old APK. + # Then run only the selected upgrade subset afterward, starting from the old APK. upgrade_device="${DEVICE_LIST%% *}" normal_category="${RESOLVED_CATEGORY}" normal_status=0 @@ -326,7 +326,8 @@ jobs: echo "Running upgrade tests last with old APK installed before each attempt." DEVICE_LIST="${upgrade_device}" \ DEVICE_COUNT="1" \ - RESOLVED_CATEGORY="upgrade" \ + RESOLVED_CATEGORY="${normal_category}" \ + REQUIRED_CATEGORY="upgrade" \ APP_APK_BEFORE_ATTEMPT="${OLD_APK_DEVICE_PATH:-/data/local/tmp/Wire.old.apk}" \ RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-upgrade" \ ALLURE_RESULTS_ROOT="${RUNNER_TEMP}/allure-results/upgrade" \ @@ -343,7 +344,15 @@ jobs: exit "${upgrade_status}" fi else - bash scripts/qa_android_ui_tests/run_ui_tests.sh + if [[ "${IS_UPGRADE}" != "true" && -n "${RESOLVED_CATEGORY}" ]]; then + # When upgrade mode is off, keep category runs in their selected + # scope but skip upgrade-tagged tests instead of executing them + # in the normal phase. + EXCLUDE_CATEGORY="upgrade" \ + bash scripts/qa_android_ui_tests/run_ui_tests.sh + else + bash scripts/qa_android_ui_tests/run_ui_tests.sh + fi fi # Export one standard artifact so future manual deflake runs can reuse diff --git a/scripts/qa_android_ui_tests/README.md b/scripts/qa_android_ui_tests/README.md index d8adbb89cac..cfcdd15eac0 100644 --- a/scripts/qa_android_ui_tests/README.md +++ b/scripts/qa_android_ui_tests/README.md @@ -17,6 +17,7 @@ Main critical-flow workflow. - Runs nightly on `schedule` at `04:00 UTC`. - Uses built-in defaults for non-manual runs: - latest APK + - upgrade mode enabled - `internal release candidate` - `@criticalFlow` - auto device selection @@ -108,6 +109,39 @@ Reporting behavior: - `passed_on_rerun` - `failed_after_retries` +## Selector And Upgrade Behavior + +The selector decides which tests are in scope. Upgrade mode decides how the +selected upgrade-tagged subset is executed. + +- `@criticalFlow` runs only `criticalFlow` tests. +- `@regression` runs only `regression` tests. +- `@TC-1234` runs only that test case. + +When `isUpgrade=true` and the selector is a broad category run: + +1. the workflow runs the selected category first on the new APK and excludes tests tagged `upgrade` +2. the workflow then runs only the selected `upgrade` subset last on one device, starting from the old APK + +Examples: + +- `@criticalFlow` + `isUpgrade=true`: + - normal phase: `criticalFlow` tests except `criticalFlow + upgrade` + - upgrade phase: only `criticalFlow + upgrade` +- `@regression` + `isUpgrade=true`: + - normal phase: `regression` tests except `regression + upgrade` + - upgrade phase: only `regression + upgrade` + +When `isUpgrade=false` and the selector is a category run: + +- the workflow keeps the selected category scope +- tests also tagged `upgrade` are skipped instead of running in the normal phase + +When the selector is left empty and `isUpgrade=true`: + +- the workflow runs all non-upgrade tests first +- then runs all upgrade-tagged tests last + ## Manual Deflake Flow 1. Critical flow or an earlier manual deflake run uploads `android-ui-test-deflake-input`. diff --git a/scripts/qa_android_ui_tests/run_ui_tests.sh b/scripts/qa_android_ui_tests/run_ui_tests.sh index adbf96836a9..7c9462bac22 100755 --- a/scripts/qa_android_ui_tests/run_ui_tests.sh +++ b/scripts/qa_android_ui_tests/run_ui_tests.sh @@ -418,6 +418,9 @@ run_attempt_on_devices() { if [[ -n "${EXCLUDE_CATEGORY:-}" ]]; then args+=(-e excludeCategory "${EXCLUDE_CATEGORY}") fi + if [[ -n "${REQUIRED_CATEGORY:-}" ]]; then + args+=(-e requiredCategory "${REQUIRED_CATEGORY}") + fi else local retry_list_file local retry_test_count diff --git a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt index b4c098d17a6..dd6c70f60f2 100644 --- a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt +++ b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt @@ -33,9 +33,11 @@ import java.io.File /** * JUnit filter used by AndroidJUnitRunner / AllureAndroidJUnitRunner. * - * Supports normal selector args (@TestCaseId, @Category, @Tag, excludeCategory), - * and rerun selector args (Class#method). excludeCategory skips tests that also - * have that category, for example normal test phases skip upgrade tests. + * Supports normal selector args (@TestCaseId, @Category, @Tag, excludeCategory, + * requiredCategory), and rerun selector args (Class#method). excludeCategory + * skips tests that also have that category, for example normal test phases skip + * upgrade tests. requiredCategory lets CI keep the original selector and add one + * more category requirement for a second pass, for example regression + upgrade. */ class TaggedFilter : Filter() { private val args = InstrumentationRegistry.getArguments() @@ -43,6 +45,7 @@ class TaggedFilter : Filter() { private val filterTestCaseId: String? = args.getString("testCaseId") private val filterCategory: String? = args.getString("category") private val excludeCategory: String? = args.getString("excludeCategory") + private val requiredCategory: String? = args.getString("requiredCategory") private val filterTagKey: String? = args.getString("tagKey") private val filterTagValue: String? = args.getString("tagValue") @@ -67,6 +70,7 @@ class TaggedFilter : Filter() { if (filterTestCaseId == null && filterCategory == null && excludeCategory == null && + requiredCategory == null && filterTagKey == null && filterTagValue == null ) { @@ -186,6 +190,15 @@ class TaggedFilter : Filter() { if (!matchesCat) return false } + requiredCategory?.let { requiredCat -> + if (categories.isEmpty()) return false + + val matchesRequiredCat = categories.any { catAnno -> + catAnno.value.contains(requiredCat) + } + if (!matchesRequiredCat) return false + } + // 3) Tag (key + value) if (filterTagKey != null || filterTagValue != null) { val tags = annotations.filterIsInstance() @@ -205,6 +218,7 @@ class TaggedFilter : Filter() { override fun describe(): String { return "TaggedFilter(testCaseId=$filterTestCaseId, " + "category=$filterCategory, excludeCategory=$excludeCategory, " + + "requiredCategory=$requiredCategory, " + "tagKey=$filterTagKey, tagValue=$filterTagValue, " + "rerunModeEnabled=$rerunModeEnabled, rerunAttempt=$rerunAttempt)" } diff --git a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedTestRunner.kt b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedTestRunner.kt index c04c5a5f297..32158809a37 100644 --- a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedTestRunner.kt +++ b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedTestRunner.kt @@ -35,6 +35,7 @@ class TaggedTestRunner : AllureAndroidJUnitRunner() { val filterId = arguments.getString("testCaseId") val category = arguments.getString("category") val excludeCategory = arguments.getString("excludeCategory") + val requiredCategory = arguments.getString("requiredCategory") val tagKey = arguments.getString("tagKey") val tagValue = arguments.getString("tagValue") val rerunMode = arguments.getString(RetryContract.ARG_ENABLE_RERUN_MODE) @@ -50,6 +51,7 @@ class TaggedTestRunner : AllureAndroidJUnitRunner() { "TaggedTestRunner", "onCreate called. " + "testCaseId=$filterId, category=$category, excludeCategory=$excludeCategory, " + + "requiredCategory=$requiredCategory, " + "tagKey=$tagKey, tagValue=$tagValue, " + "rerunMode=$rerunMode, rerunAttempt=$rerunAttempt, " + "rerunListPath=$rerunListPath, rerunListInlineLength=${rerunListInline?.length ?: 0}, " +